nwsserver-2.0.0/0000755000175000017500000000000011261701137013166 5ustar westonwestonnwsserver-2.0.0/PKG-INFO0000644000175000017500000000263711261701137014273 0ustar westonwestonMetadata-Version: 1.0 Name: nwsserver Version: 2.0.0 Summary: Python NetWorkSpaces Server Home-page: http://nws-py.sourceforge.net/ Author: REvolution Computing, Inc. Author-email: sbweston@users.sourceforge.net License: GPL version 2 or later Description: NetWorkSpaces (NWS) is a system that makes it very easy for different scripts and programs running (potentially) on different machines to communicate and coordinate with one another. The requirements for the NWS server are: Python 2.2 or later on Linux, Mac OS X, and other Unix systems. Python 2.4 or later on Windows. Twisted 2.1 and Twisted-Web 0.5 or later. Twisted is available from: http://www.twistedmatrix.com/ Twisted itself requires: Zope Interfaces 3.0.1 (http://zope.org/Products/ZopeInterface) Platform: any Classifier: Topic :: System :: Clustering Classifier: Topic :: System :: Distributed Computing Classifier: License :: OSI Approved :: GNU General Public License (GPL) Classifier: Development Status :: 5 - Production/Stable Classifier: Environment :: Console Classifier: Intended Audience :: Developers Classifier: Intended Audience :: Science/Research Classifier: Intended Audience :: System Administrators Classifier: Operating System :: OS Independent Classifier: Programming Language :: Python Classifier: Natural Language :: English nwsserver-2.0.0/nwss/0000755000175000017500000000000011261701137014160 5ustar westonwestonnwsserver-2.0.0/nwss/util.py0000644000175000017500000000747311214274547015532 0ustar westonweston# # Copyright (c) 2005-2009, REvolution Computing, Inc. # # NetWorkSpaces is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License as published # by the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 # USA # """System-level utilities.""" import sys, os import string #pylint: disable-msg=W0402 __all__ = ['which', 'msc_quote', 'msc_argv2str'] IS_WINDOWS = sys.platform.startswith('win') WIN_EXTS = os.environ.get('PATHEXT', '').split(os.pathsep) NEED_QUOTING = string.whitespace + '"' def _isexec(path): """Check if a path refers to an executable file.""" return os.path.isfile(path) and os.access(path, os.X_OK) def _exenames(name): """Generate the list of executable names to which ``name`` might refer.""" names = [name] # only expand on Windows if IS_WINDOWS: # only expand if not extension is specified base, ext = os.path.splitext(name) if not ext: for ext in WIN_EXTS: names.append(base + ext) return names def which(cmd, path=None): """Find the command ``cmd`` in the path, similar to shell-builtin 'which' """ if os.path.split(cmd)[0]: cmd = os.path.abspath(cmd) # print "Checking for", cmd matches = [x for x in _exenames(cmd) if _isexec(x)] else: matches = [] if not path: path = os.environ.get('PATH', os.defpath).split(os.pathsep) if IS_WINDOWS: path.insert(0, os.curdir) for pelem in path: full = os.path.join(os.path.normpath(pelem), cmd) # print "Checking for", full matches += [x for x in _exenames(full) if _isexec(x)] return matches def msc_quote(cmd): """Quote a command-line argument as per MSC quoting rules. This is useful in conjunction with os.spawnv on Windows. For example:: os.spawnv(os.P_WAIT, argv[0], [msc_quote(a) for a in argv]) This is only useful on Windows. """ if not [char for char in cmd if char in NEED_QUOTING]: return cmd quoted = '"' nbs = 0 for char in cmd: if char == '\\': quoted += char nbs += 1 elif char == '"': quoted += (nbs + 1) * '\\' + char nbs = 0 else: quoted += char nbs = 0 quoted += nbs * '\\' + '"' return quoted def msc_argv2str(argv): """Quote a command-line as per MSC quoting rules. This is useful in conjunction with win32process.CreateProcess and os.system on Windows. For example:: os.system(msc_argv2str(argv)) This is only useful on Windows. """ return ' '.join([msc_quote(arg) for arg in argv]) if __name__ == '__main__': def _qtest(argv): """Simple test for msc_argv2str quoting.""" print "quoted command string:", msc_argv2str(argv) def _wtest(argv): """Simple test for ``which`` functionality.""" for arg in argv[1:]: plist = which(arg) if not plist: print >> sys.stderr, \ "error: no matches found for", arg continue if len(plist) > 1: print >> sys.stderr, \ "warning: more than one match found for", arg print plist[0] _qtest(sys.argv) nwsserver-2.0.0/nwss/protoutils.py0000644000175000017500000004343611214274547017000 0ustar westonweston# # Copyright (c) 2005-2009, REvolution Computing, Inc. # # NetWorkSpaces is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License as published # by the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 # USA # """Miscellaneous utilities and building blocks for the NWS protocol.""" from twisted.python import log import nwss import os _MIN_LONG_VALUE_SIZE = 64 _BUFFER_SIZE = 16 * 1024 _DEBUG = nwss.config.is_debug_enabled('NWS:protoutils') try: set() #pylint: disable-msg=W8302 except NameError: from sets import Set as set #pylint: disable-msg=W0622 class WsNameMap(object): """'External-to-internal' name mapping for workspaces. This was separated into a mixin so that it can be used in DummyConnection as well.""" def __init__(self): """Create a new ext-to-int workspace name mapping.""" self.__ext_to_int_names = {} def get(self, ext_name): """Lookup the internal name for an external workspace name. Returns None if the named workspace doesn't exist.""" return self.__ext_to_int_names.get(ext_name) def set(self, ext_name, int_name): """Set the internal name for an external workspace name.""" self.__ext_to_int_names[ext_name] = int_name def remove(self, ext_name): """Clear the mapping for an internal name for a given external workspace name. No error is raised if the name didn't exist.""" try: del self.__ext_to_int_names[ext_name] except KeyError: pass class WsTracker(object): """List of 'owned' workspaces, tracked by internal name. Separated into a mixin so that it can be used in DummyConnection as well.""" def __init__(self): """Create a new owned workspace list.""" self.__owned_workspaces = set() def add(self, int_name): """Add a workspace to our ownership list.""" self.__owned_workspaces.add(int_name) def remove(self, int_name): """Remove a workspace from our ownership list.""" try: self.__owned_workspaces.remove(int_name) return True except KeyError: return False def contains(self, int_name): """Check if we claim ownership of a given workspace.""" return (int_name in self.__owned_workspaces) def as_list(self): """Get a list of workspaces owned by us.""" return list(self.__owned_workspaces) def get_count(self): """Get a count of the workspaces owned by us.""" return len(self.__owned_workspaces) count = property(get_count) def clear(self): """Clear all workspaces from our list.""" self.__owned_workspaces.clear() class CountedReceiver(object): """Protocol helper class for protocol atoms which consist of a fixed-length ASCII decimal byte count followed by raw data. Most data in the NWS protocol adheres to this format with either a 4-byte or 20-byte count. Generally, this class is used from a protocol object as: return CountedReceiver(self, self.set_my_value).start, 4 """ def __init__(self, conn, target, start_count=4): """Create a new CountedReceiver protocol atom. Parameters: conn - the protocol object on whose behalf we act target - function to receive the data start_count - size of count (in bytes) [default: 4] """ self.__conn = conn self.__target = target self.start_count = start_count def get_target(self): """Get the target of this protocol helper.""" return self.__target target = property(get_target) def get_connection(self): """Get the Protocol object for this protocol helper.""" return self.__conn connection = property(get_connection) def start(self, data): """The start state for this atom. This is the method to pass to twisted as the handler for the chunk of data on a newly created atom. """ try: length = int(data) if length < 0: raise ValueError('negative argument length') except ValueError: log.msg("error: got bad data from client") self.__conn.transport.loseConnection() return None return self.__target, length class CountedReceiverLong(CountedReceiver): """Specialized protocol helper class for protocol atoms identical to those supported by CountedReceiver, with a count occupying 20 bytes. The important difference is that this atom supports saving large data directly to a file. As a result, the 'target' must support an optional boolean argument 'long_data'. If True, the data passed to the target will be a filename, rather than the data itself. Generally, this class is used from a protocol object as: return CountedReceiverLong(self, self.set_my_value).start, 20 """ def __init__(self, conn, target): """Create a new CountedReceiverLong protocol atom. Parameters: conn - the protocol object on whose behalf we act target - function to receive the data """ CountedReceiver.__init__(self, conn, target, 20) self.__target = target self.__conn = conn self.__file = None self.__filename = None self.__remain_length = 0 def start(self, data): """The start state for this atom. This is the method to pass to twisted as the handler for the chunk of data on a newly created atom. """ base_next = CountedReceiver.start(self, data) if base_next is None: return None _, length = base_next # Compute the threshold for treating the value as long data threshold = max(_MIN_LONG_VALUE_SIZE, nwss.config.nwsLongValueSize) if length >= threshold: # Set up the streaming transfer self.__remain_length = length tmpfile = self.__conn.new_long_arg_file() if tmpfile == None: # Failed to create the file, but still need to ride out the # transfer. self.__file = None else: self.__file, self.__filename = tmpfile return self.long_data, min(_BUFFER_SIZE, length) else: return base_next def long_data(self, data): """The streaming data state for this atom. This state is entered only if the length of the data exceeds the long-data threshold, and will be re-entered repeatedly until all data has been read. """ self.__remain_length -= len(data) if self.__file != None: self.__file.write(data) if self.__remain_length <= 0: self.__file.close() return self.__target(self.__filename, long_data=True) else: if self.__remain_length <= 0: self.__conn.send_error('Failed to read long data from the ' + 'filesystem.') self.__conn.transport.loseConnection() return None return self.long_data, min(_BUFFER_SIZE, self.__remain_length) class NameValueReceiver(object): """Specialized protocol helper class for protocol elements consisting of two consecutive CountedReceiver atoms representing names and values. The target will receive a (name, value) tuple. Generally, this class is used from a protocol object as: return NameValueReceiver(self, self.add_entry).start, 4 """ start_count = 4 def __init__(self, conn, target): """Create a new NameValueReceiver protocol atom. Parameters: conn - the protocol object on whose behalf we act target - function to receive the data """ self.__conn = conn self.__target = target self.__name = None def get_start(self): """Get the start state for this atom. Note: unlike the preceding helpers, this is NOT a handler to be passed to twisted, as it needs to delegate to a counted receiver instead. Thanks to the 'start' property defined below, however, the notation is the same. """ return CountedReceiver(self.__conn, self.name).start start = property(get_start) def name(self, data): """Target to receive name of name/value pair.""" self.__name = data return CountedReceiver(self.__conn, self.value).start, 4 def value(self, data): """Target to receive value of name/value pair.""" return self.__target((self.__name, data)) class ListReceiver(object): """Specialized protocol helper class for protocol elements consisting of a 4-byte ASCII decimal count followed by a series of protocol elements of a given format. Two callback functions should be provided. One will be called for each item in the list, and the other will be called when all items have been received. Generally, this class is used from a protocol object as: return ListReceiver(self, ItemReceiver, self.add_item, self.last_item).start, 4 """ start_count = 4 def __init__(self, conn, item, target, done): """Create a new ListReceiver protocol element. Parameters: conn - the protocol object on whose behalf we act item - item type to read for list target - function to receive each item done - function called when all items have been received """ self.__conn = conn self.__target = target self.__done = done self.__item_type = item self.__remaining = 0 def start(self, data): """The start state for this atom. This is the method to pass to twisted as the handler for the chunk of data on a newly created atom. """ # Read the length try: length = int(data) if length < 0: raise ValueError('Negative count for item list.') except ValueError, exc: log.msg('Malformed protocol message: %s' % exc.args[0]) self.__conn.transport.loseConnection() return None # Start reading the first item self.__remaining = length if self.__remaining == 0: return self.__done() else: item = self.__item_type(self.__conn, self.item) return item.start, item.start_count def item(self, data, **kwargs): """The callback for each item.""" self.__remaining -= 1 self.__target(data, **kwargs) if self.__remaining == 0: return self.__done() else: item = self.__item_type(self.__conn, self.item) return item.start, item.start_count class DictReceiver(ListReceiver): """Specialized protocol helper class for protocol elements consisting of a stream of name-value pairs. The format is that of a ListReceiver receiving NameValueReceiver elements. A callback function will receive a dictionary of all name-value pairs once they have all been read in. Generally, this class is used from a protocol object as: return DictReceiver(self, self.set_dictionary).start, 4 or: dr = DictReceiver(self, self.__receive_connection_options) return dr.start, dr.start_count """ def __init__(self, conn, target): """Create a new DictReceiver protocol element. Parameters: conn - the protocol object on whose behalf we act target - function to receive the complete dictionary """ ListReceiver.__init__(self, conn, NameValueReceiver, self.entry, self.finished) self.__target = target self.__entries = {} def entry(self, data): """Callback to receive each name, value pair.""" name, value = data self.__entries[name] = value def finished(self): """Callback upon completion.""" return self.__target(self.__entries) class ArgTupleReceiver(ListReceiver): """Specialized protocol helper class for protocol elements consisting of a stream of arguments. The format is that of a ListReceiver receiving CountedReceiverLong elements. A callback function will receive a list of all arguments once they have all been read in. The elements of the list will be strings and, for "long" items, tuples of filename and content length. Generally, this class is used from a protocol object as: return ArgTupleReceiver(self, self.set_args).start, 4 or: atr = ArgTupleReceiver(self, self.set_args) return atr.start, atr.start_count """ def __init__(self, conn, target, metadata): """Create a new ArgTupleReceiver protocol element. Parameters: conn - the protocol object on whose behalf we act target - function to receive the complete argument list metadata - metadata attached to this arg tuple, if any """ ListReceiver.__init__(self, conn, CountedReceiverLong, self.next_arg, self.finished) self.__target = target self.__args = [] self.__metadata = metadata def next_arg(self, data, long_data=False): """Callback to receive each argument. If long_data is True, the data contains a filename rather than directly containing the data.""" if long_data: dlen = os.stat(data).st_size data = (data, dlen) self.__args.append(data) def finished(self): """Callback upon completion.""" return self.__target(self.__args, self.__metadata) class FileProducer(object): """Twisted "producer" to allow drawing data directly from a file on disk.""" def __init__(self, value, transport): """Create a new file producer for a given value object. Parameters: value - the value transport - consumer of data """ self.__value = value self.__file = value.get_file() self.__buffer_size = _BUFFER_SIZE self.__transport = transport self.__finished = False def stopProducing(self): #pylint: disable-msg=C0103 """Implementation of IPushProducer.stopProducing method from Twisted. Called to abort production of data from the file.""" if _DEBUG: log.msg('stopProducing called') if not self.__finished: if _DEBUG: log.msg('stopProducing unregistering producer') try: self.__file.close() except (KeyboardInterrupt, SystemExit): raise except Exception: #pylint: disable-msg=W0703 pass self.__transport.unregisterProducer() self.__finished = True self.__value.access_complete() else: log.msg("error: stopProducing called even though I finished") def resumeProducing(self): #pylint: disable-msg=C0103 """Implementation of IPushProducer.resumeProducing method from Twisted. Called to resume production of data from the file after it has been paused.""" # read some more data from the file, a write it to the transport if _DEBUG: log.msg('resumeProducing called') if not self.__finished: if _DEBUG: log.msg('resumeProducing reading file') data = self.__file.read(self.__buffer_size) if not data: if _DEBUG: log.msg('resumeProducing unregistering producer') try: self.__file.close() except (KeyboardInterrupt, SystemExit): raise except Exception: #pylint: disable-msg=W0703 pass self.__transport.unregisterProducer() self.__finished = True self.__value.access_complete() else: if _DEBUG: log.msg('resumeProducing sending data to client ' + data) self.__transport.write(data) else: log.msg("error: resumeProducing called even though I unregistered") def pauseProducing(self): #pylint: disable-msg=C0103,R0201 """Implementation of IPushProducer.pauseProducing method from Twisted. Called to resume production of data from the file after it has been paused. Even though this method does nothing, it must be present in order for this class to implement the IPushProducer interface.""" if _DEBUG: log.msg('pauseProducing called') def map_proto_generator(data): """Utility which turns a map into a stream of name value pairs in the correct protocol form for metadata maps.""" return [('%04d%s%04d%s' % (len(k), k, len(v), v)) for k, v in data.items()] nwsserver-2.0.0/nwss/local.py0000644000175000017500000000727011214274547015642 0ustar westonweston# # Copyright (c) 2007-2009, REvolution Computing, Inc. # # NetWorkSpaces is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License as published # by the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 # USA # """Implementation of NWS server startup as an interpreter-local thread.""" import threading, time from twisted.internet import reactor from nwss.server import NwsService __all__ = ['NwsLocalServerException', 'NwsLocalServer'] class NwsLocalServerException(Exception): """Exception thrown if an error occurs while starting the local server.""" pass class NwsLocalServer(threading.Thread): """Utility to start the NWS server as a thread within a Python interpreter.""" def __init__(self, port=0, interface='', daemon=True, name='NwsLocalServer', **kw): threading.Thread.__init__(self, name=name, **kw) self.__desired_port = port self.__port = None self.__interface = interface self.__started = False self.__condition = threading.Condition() self.setDaemon(daemon) self.start() def get_port(self): """Get the actual port number to which we bound.""" return self.__port._realPortNumber #pylint: disable-msg=W0212 port = property(get_port) def shutdown(self, timeout=None): """Request the shutdown of the server, waiting at most 'timeout' seconds for the server thread to stop.""" reactor.callFromThread(reactor.stop) #pylint: disable-msg=E1101 self.join(timeout=timeout) def wait_until_started(self, timeout=30): """Wait for the server thread to finish initializing. Wait for at most 'timeout' seconds for the server to start, throwing NwsLocalServerException if it has not started by the time the timer expires.""" start_time = time.time() timeout_remain = timeout self.__condition.acquire() try: while not self.__started: self.__condition.wait(timeout=timeout_remain) if timeout is not None: timeout_remain = start_time + timeout - time.time() if timeout_remain <= 0: break if not self.__started: raise NwsLocalServerException( 'local server timeout expired: %d' % timeout) finally: self.__condition.release() def run(self): """Main loop of NWS local server thread.""" srv = NwsService() srv.startService() #pylint: disable-msg=E1101 try: #pylint: disable-msg=E1101,W0212 self.__port = reactor.listenTCP(self.__desired_port, srv._factory) reactor.callWhenRunning(self.__set_started) reactor.run(installSignalHandlers=0) finally: srv.stopService() #pylint: disable-msg=E1101 def __set_started(self): """Callback from twisted indicating successful startup of the server.""" self.__condition.acquire() try: self.__started = True self.__condition.notifyAll() finally: self.__condition.release() nwsserver-2.0.0/nwss/__init__.py0000644000175000017500000001171511214274547016306 0ustar westonweston# # Copyright (c) 2005-2009, REvolution Computing, Inc. # # NetWorkSpaces is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License as published # by the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 # USA # """This module implements the core of the NWS server.""" __version__ = '2.0.0' __all__ = ['config', 'server', 'util', 'mock', 'workspace', 'pyutils', 'protocol' ] def _cp_int(parser, name, default): """Utility to encapsulate loading an integer setting from a config file.""" from ConfigParser import NoOptionError try: return parser.getint('NWS Server', name) except NoOptionError: return default def _cp_str(parser, name, default): """Utility to encapsulate loading a string setting from a config file.""" from ConfigParser import NoOptionError try: return parser.get('NWS Server', name) except NoOptionError: return default class Config(object): #pylint: disable-msg=R0902,R0903 """Configuration wrapper object.""" __slots__ = [ # General server settings 'serverport', 'tmpdir', 'longvaluesize', # web settings 'webport', 'webserveddir', # SSL settings 'serversslcert', 'serversslkey', # Plugin settings 'plugindirs', # Debug settings 'debug' ] def __init__(self): import nwss.config as cfg self.serverport = cfg.nwsServerPort self.tmpdir = cfg.nwsTmpDir self.longvaluesize = cfg.nwsLongValueSize self.webport = cfg.nwsWebPort self.webserveddir = cfg.nwsWebServedDir self.serversslcert = cfg.nwsServerSslCert self.serversslkey = cfg.nwsServerSslKey self.plugindirs = cfg.nwsPluginDirs if hasattr(cfg, 'debug'): self.debug = cfg.debug else: self.debug = [] def is_debug_enabled(self, opt): """Check if debugging is enabled for a certain section of the code.""" optsplit = opt.split(':') for scope in range(0, len(optsplit) + 1): if scope != 0: name_specific = ':'.join(optsplit[0:scope]) if name_specific in self.debug: return True name_all = ':'.join(optsplit[0:scope] + ['ALL']) if name_all in self.debug: return True return False def load_from_file(self, filename): """Load the configuration from a config file. Parameters: filename - name of the config file """ from ConfigParser import ConfigParser import os parser = ConfigParser() parser.read(filename) self.serverport = _cp_int(parser, 'serverPort', self.serverport) self.tmpdir = _cp_str(parser, 'tmpDir', self.tmpdir) self.longvaluesize = _cp_int(parser, 'longValueSize', self.longvaluesize) self.webport = _cp_int(parser, 'webPort', self.webport) self.webserveddir = _cp_str(parser, 'webClientCode', self.webserveddir) self.serversslcert = _cp_str(parser, 'sslCertificate', self.serversslcert) self.serversslkey = _cp_str(parser, 'sslKey', self.serversslkey) plugins = os.pathsep.join(self.plugindirs) plugins = _cp_str(parser, 'pluginsPath', plugins) self.plugindirs = plugins.split(os.pathsep) debug = ','.join(self.debug) debug = _cp_str(parser, 'debug', debug) self.debug = [opt.strip() for opt in debug.split(',') if opt.strip() != ''] def __getattr__(self, name): name = name.lower() if name.startswith('nws'): name = name[3:] return getattr(self, name) def __setattr__(self, name, value): name = name.lower() if name.startswith('nws'): name = name[3:] return object.__setattr__(self, name, value) config = Config() #pylint: disable-msg=C0103 nwsserver-2.0.0/nwss/stdvars.py0000644000175000017500000007471211217710530016231 0ustar westonweston# # Copyright (c) 2005-2009, REvolution Computing, Inc. # # NetWorkSpaces is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License as published # by the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 # USA # """ Core NetWorkSpaces server - variables implementation. """ from __future__ import generators import time from twisted.python import log from nwss.pyutils import new_list, remove_first, clear_list from nwss.base import BadModeException from nwss.base import WorkspaceFailure from nwss.base import Response, Value import nwss _DEBUG = nwss.config.is_debug_enabled('NWS:stdvars') class BaseVar(object): """Base class for variables to simplify implementation of different variable types. """ def __init__(self, name): """Constructor for BaseVar objects. Parameters: name - user-readable name for var """ self.__name = name self.vid = None self.fetchers = [] self.finders = [] def __get_name(self): """Get the name of this container.""" return self.__name name = property(__get_name) def __get_num_fetchers(self): """Accessor for fetcher count property.""" return len(self.fetchers) num_fetchers = property(__get_num_fetchers) def __get_num_finders(self): """Accessor for finder count property.""" return len(self.finders) num_finders = property(__get_num_finders) def add_fetcher(self, fetcher): """Add a fetcher to this variable. Arguments: fetcher -- the fetcher to add """ self.fetchers.append(fetcher) fetcher.set_blocking_var(self.__name, self.fetchers) def add_finder(self, finder): """Add a finder to this variable. Arguments: finder -- the finder to add """ self.finders.append(finder) finder.set_blocking_var(self.__name, self.finders) def new_value(self, val_index, val, metadata): """Announce the appearance of a new value. If there are prior finders, the value will be distributed to them. If there are prior fetchers, the value will be distributed to the first of them in line, and False will be returned. Arguments: val_index - index of value being stored (for iterated finds, etc) val - newly stored value metadata - metadata stored with value """ # Not consumed unless there was a fetcher consumed = False # Build response resp = Response(metadata, val) resp.iterstate = (self.vid, val_index) # feed the finders for client in self.finders: if _DEBUG: log.msg('calling finder session %d with val_index %d' % (client.transport.sessionno, val_index)) client.send_long_response(resp) # clear the finders list del self.finders[:] # give it to a fetcher if there is one if self.fetchers: client = self.fetchers.pop(0) if _DEBUG: log.msg('calling fetcher session %d with val_index %d' % (client.transport.sessionno, val_index)) client.send_long_response(resp) consumed = True return consumed def fail_waiters(self, reason): """Cause all waiters to fail, typically because this variable has been destroyed.""" for client in self.fetchers: if _DEBUG: log.msg('sending error value to fetcher session %d' % client.transport.sessionno) client.send_error(reason, long_reply=True) for client in self.finders: if _DEBUG: log.msg('sending error value to finder session %d' % client.transport.sessionno) client.send_error(reason, long_reply=True) del self.fetchers[:] del self.finders[:] class Fifo(BaseVar): """Variable class for FIFO-type variables.""" def __init__(self, name): """Constructor for FIFO-type variables. Arguments: name - user-readable name for var """ BaseVar.__init__(self, name) self._contents = new_list() self._metadata = new_list() self._index = 0 def __len__(self): return len(self._contents) def __iter__(self): return iter(self._contents) def store(self, client, value, metadata): #pylint: disable-msg=W0613 """Handle a store request on this variable. For a FIFO variable, this adds a value to the tail of the values queue. Arguments: client -- client for whom to perform store value -- value to store in FIFO """ # compute the index of this incoming value in case we # need to give it to any waiting clients. # it isn't used for anything else val_index = self._index + len(self._contents) if self.new_value(val_index, value, metadata): # value was consumed, so increment index self._index += 1 else: # value wasn't consumed, so save it self._contents.append(value) self._metadata.append(metadata) def fetch(self, client, blocking, val_index, metadata): #pylint: disable-msg=W0613 """Handle a fetch request on this variable. Arguments: client - client for whom to perform fetch blocking - is this a blocking fetch? val_index - index of value to fetch (unused here) """ fetch_location = max(val_index - self._index + 1, 0) if fetch_location > 0: raise WorkspaceFailure( 'ifetch* only supported at beginning of FIFO') try: value = remove_first(self._contents) var_metadata = remove_first(self._metadata) value.consumed() response = Response(var_metadata, value) response.iterstate = (self.vid, self._index) self._index += 1 return response except IndexError: if blocking: self.add_fetcher(client) return None else: raise WorkspaceFailure('no value available') def find(self, client, blocking, val_index, metadata): #pylint: disable-msg=W0613 """Handle a find request on this variable. Arguments: client - client for whom to perform find blocking - is this a blocking find? val_index - index of value to find (for iterated find) """ try: find_location = max(val_index - self._index + 1, 0) response = Response(self._metadata[find_location], self._contents[find_location]) response.iterstate = self.vid, self._index + find_location return response except IndexError: if blocking: self.add_finder(client) return None else: raise WorkspaceFailure('no value available') def purge(self): """Purge this variable from the workspace, causing any clients waiting for a value to fail. """ self.fail_waiters('Variable purged.') for val in self._contents: if isinstance(val, Value): val.close() clear_list(self._contents) clear_list(self._metadata) class Lifo(BaseVar): """Variable class for LIFO-type variables.""" def __init__(self, name): """Constructor for FIFO-type variables. Arguments: name - user-readable name for var """ BaseVar.__init__(self, name) self._contents = [] self._metadata = [] def __len__(self): return len(self._contents) def __iter__(self): return iter(self._contents) def store(self, client, value, metadata): #pylint: disable-msg=W0613 """Handle a store request on this variable. For a LIFO variable, this adds a value to the tail of the values stack. Arguments: client -- client for whom to perform store value -- value to store in LIFO """ if not self.new_value(0, value, metadata): self._contents.append(value) self._metadata.append(metadata) def fetch(self, client, blocking, val_index, metadata): #pylint: disable-msg=W0613 """Handle a fetch request on this variable. Arguments: client - client for whom to perform fetch blocking - is this a blocking fetch? val_index - index of value to fetch (unused here) """ if val_index >= 0: raise WorkspaceFailure('ifetch* not supported on LIFO') try: value = self._contents.pop() var_metadata = self._metadata.pop() value.consumed() return Response(var_metadata, value) except IndexError: if blocking: self.add_fetcher(client) return None else: raise WorkspaceFailure('no value available') def find(self, client, blocking, val_index, metadata): #pylint: disable-msg=W0613 """Handle a find request on this variable. Arguments: client - client for whom to perform find blocking - is this a blocking find? val_index - index of value to find (for iterated find) """ if val_index >= 0: raise WorkspaceFailure('ifind* not supported on LIFO') try: # ignore val_index since we don't allow iterators return Response(self._metadata[-1], self._contents[-1]) except IndexError: if blocking: self.add_finder(client) return None else: raise WorkspaceFailure('no value available') def purge(self): """Purge this variable from the workspace, causing any clients waiting for a value to fail. """ self.fail_waiters('Variable purged.') for val in self._contents: val.close() del self._contents[:] del self._metadata[:] class Single(BaseVar): """Variable class for Single-type variables.""" def __init__(self, name): """Constructor for Single-type variables. Arguments: name - user-readable name for var """ BaseVar.__init__(self, name) self._contents = [] self._metadata = None self._index = 0 def __len__(self): return self._contents and 1 or 0 def __iter__(self): return iter(self._contents) def store(self, client, value, metadata): #pylint: disable-msg=W0613 """Handle a store request on this variable. For a Single variable, this sets the value or feeds it to the first waiting fetcher. Arguments: client -- client for whom to perform store value -- value to store in Single """ # compute the index of this incoming value in case we # need to give it to any waiting clients. # it isn't used for anything else val_index = self._index + len(self) if self.new_value(val_index, value, metadata): # value was consumed, so increment index self._index += 1 else: # value wasn't consumed, so save it if self._contents: self._contents[0].close() self._contents[0] = value self._index += 1 else: self._contents.append(value) self._metadata = metadata def fetch(self, client, blocking, val_index, metadata): #pylint: disable-msg=W0613 """Handle a fetch request on this variable. Arguments: client -- client for whom to perform fetch blocking -- is this a blocking fetch? val_index -- index of value to fetch (unused here) """ try: fetch_location = max(val_index - self._index + 1, 0) value = self._contents.pop(fetch_location) value.consumed() response = Response(self._metadata, value) response.iterstate = (self.vid, self._index) self._metadata = None self._index += 1 return response except IndexError: if blocking: self.add_fetcher(client) return None else: raise WorkspaceFailure('no value available') def find(self, client, blocking, val_index, metadata): #pylint: disable-msg=W0613 """Handle a find request on this variable. Arguments: client -- client for whom to perform find blocking -- is this a blocking find? val_index -- index of value to find (for iterated find) """ try: find_location = max(val_index - self._index + 1, 0) response = Response(self._metadata, self._contents[find_location]) response.iterstate = (self.vid, self._index + find_location) return response except IndexError: if blocking: if _DEBUG: log.msg('queueing up blocking find request') self.add_finder(client) return None else: if _DEBUG: log.msg('returning unsuccessful reply') raise WorkspaceFailure('no value available') def purge(self): """Purge this variable from the workspace, causing any clients waiting for a value to fail. """ self.fail_waiters('Variable purged.') if self._contents: self._contents[0].close() del self._contents[:] self._metadata = None class SimpleAttribute(BaseVar): """Container type to hold a constant value, ignoring store requests and always allowing fetch/find requests to succeed. """ def __init__(self, name, value_func, metadata=None): BaseVar.__init__(self, name) self.__value_func = value_func if metadata is None: self.__metadata = {} else: self.__metadata = metadata def __len__(self): return 1 def __iter__(self): def singleton(): """Single item iterator.""" yield self.__value_func() return singleton() def new_value(self, val_index, val, metadata): """Announce the appearance of a new value. This should never happen for this variable type. Arguments: val_index - index of value being stored (for iterated finds, etc) val - newly stored value metadata - metadata stored with value """ assert self.num_finders == 0 assert self.num_fetchers == 0 return False def store(self, client, value, metadata): #pylint: disable-msg=W0613,R0201 """Handle a store request on this variable. For a constant, a store request is always an error. Arguments: client - client for whom to perform store value - value to store metadata - metadata for store operation """ raise WorkspaceFailure('Store is not supported for this variable.') def fetch(self, client, blocking, val_index, metadata): #pylint: disable-msg=W0613 """Handle a fetch request on this variable. For a constant, a fetch request always returns the same value. Arguments: client - client for whom to perform fetch blocking - is this a blocking operation val_index - value index to fetch (irrelevant) metadata - metadata for fetch operation """ value = self.__value_func() return Response(self.__metadata, value) find = fetch def purge(self): """Purge this variable from the workspace, causing any clients waiting for a value to fail. There should never be clients waiting on this variable. """ self.fail_waiters('Variable purged.') # Time, Constant variable types are now single-line implementations Constant = lambda name, val: SimpleAttribute(name, lambda: val) #pylint: disable-msg=C0103,C0301,E0601 Time = lambda name: SimpleAttribute(name, time.asctime) #pylint: disable-msg=C0103,C0301 class Barrier(BaseVar): """ Variable type encapsulating the concept of group membership. A client joins the group by storing any value into the variable. It uses fetch to leave the group. XXX: If a client loses its connection without leaving the group, it corrupts the group. Therefore, I need to add a mechanism that lets clients automatically leave the group when it loses its connection. This might be implemented via the __client_disconnected method of the NwsService class, or perhaps only via the NwsProtocol class. """ def __init__(self, name): """Create a new barrier variable. Arguments: name - user-readable name for var """ BaseVar.__init__(self, name) self._members = {} def __len__(self): # Keep this in sync with the __iter__ method. Presently, always # returns 3. When iterating over this variable, it will yield, in # order: # # * number of members # * list of members # * number of members at barrier return 3 def __iter__(self): # Keep __len__ in sync with this method. Presently, always returns the # following 3 items in order: # # * number of members # * list of members # * number of members at barrier members = ' '.join([str(m) for m in self._members]) if not members: members = '' def gen(): """Helper generator function.""" yield 'Number of members: %d' % len(self._members) yield 'List of members: %s' % members yield 'Number at barrier: %d' % self.num_finders return gen() def store(self, client, value, metadata): #pylint: disable-msg=W0613 """Handle a store request for a Barrier variable. The effect of this is to add the storing client to the group represented by this barrier. Arguments: client -- client performing the store value -- value to store (unused) """ num_members = len(self._members) assert self.num_finders == 0 or self.num_finders < num_members if client.transport.sessionno in self._members: # Client is trying to join the group a second time raise WorkspaceFailure('Client attempting to join barrier ' + 'group, but is already a member') self._members[client.transport.sessionno] = 1 def fetch(self, client, blocking, val_index, metadata): #pylint: disable-msg=W0613 """Execute a fetch on this variable. The effect of this is to remove the fetching client from the group represented by this Barrier. Arguments: client -- client which is performing the fetch blocking -- is this a blocking fetch? val_index -- index of value to fetch """ num_members = len(self._members) assert self.num_finders == 0 or self.num_finders < num_members try: del self._members[client.transport.sessionno] except (KeyError, AttributeError): raise WorkspaceFailure('Client has not joined this barrier group.') if self.num_finders >= len(self._members): consumed = self.new_value(0, str(num_members), {}) assert not consumed return Response(value='') def find(self, client, blocking, val_index, metadata): #pylint: disable-msg=W0613 """Execute a find on this variable. The effect of this is to check if the fetching client is in the group represented by this Barrier. Arguments: client -- client which is performing the find blocking -- is this a blocking find? val_index -- index of value to find """ num_members = len(self._members) assert self.num_finders == 0 or self.num_finders < num_members if blocking: if client.transport.sessionno in self._members: if self.num_finders == num_members - 1: response = Response(value=str(num_members)) consumed = self.new_value(0, response.value, {}) assert not consumed return response else: self.add_finder(client) return None else: # return an error because they're not a member raise WorkspaceFailure('Client has not joined this barrier ' + 'group.') else: return Response(value='%d out of %d at barrier' % (self.num_finders, num_members)) def purge(self): """Handle a purge request on this variable.""" self.fail_waiters('Variable purged.') class Unknown(BaseVar): """Placeholder variable class for variables which have not been stored to yet. An Unknown variable will exist from the time the first fetcher or finder accesses the variable until someone performs a or declares the variable, at which time, the appropriate variable type is substituted. """ def __init__(self, name): """Create a new Unknown variable. Arguments: name - user-readable name for var """ BaseVar.__init__(self, name) def __len__(self): """Implementation of iterator protocol for Unknown-type variables.""" return 0 def __iter__(self): """Implementation of iterator protocol for Unknown-type variables.""" return self def next(self): #pylint: disable-msg=R0201 """Implementation of iterator protocol for Unknown-type variables.""" raise StopIteration() def store(self, client, value, metadata): #pylint: disable-msg=R0201,W0613 """Handle a store request for an Unknown variable. This is an error, and should never occur. Arguments: client -- client performing the store value -- value to store (unused) """ # this should never be called raise WorkspaceFailure('store called on a variable of unknown mode') def fetch(self, client, blocking, val_index, metadata): #pylint: disable-msg=W0613 """Handle a fetch request on this variable. For an Unknown variable, this always fails or blocks immediately. Arguments: client -- client for whom to perform fetch blocking -- is this a blocking fetch? val_index -- index of value to fetch (unused here) """ if blocking: if _DEBUG: log.msg('queueing up blocking fetch request') self.add_fetcher(client) return None else: if _DEBUG: log.msg('returning unsuccessful reply') raise WorkspaceFailure('no value available') def find(self, client, blocking, val_index, metadata): #pylint: disable-msg=W0613 """Handle a find request on this variable. For an Unknown variable, this always fails or blocks immediately. Arguments: client -- client for whom to perform find blocking -- is this a blocking find? val_index -- index of value to find (unused here) """ if blocking: if _DEBUG: log.msg('queueing up blocking find request') self.add_finder(client) return None else: if _DEBUG: log.msg('returning unsuccessful reply') raise WorkspaceFailure('no value available') def purge(self): """Handle a purge request on this variable, causing all waiters to fail.""" self.fail_waiters('Variable purged.') CONTAINER_TYPES = {'fifo': Fifo, 'lifo': Lifo, 'single': Single, 'multi': Lifo, # XXX: fix multi '__time': Time, '__barrier': Barrier} class Variable(object): """Class representing a variable in a workspace. This class is a wrapper around the "container" class to hold the value (Single, Fifo, Lifo, etc.). """ def __init__(self, var_name, hidden): """Initializer for a variable. Parameters: var_name - variable name hidden - should this var be hidden from the web UI? """ self.__name = var_name self.__mode = 'unknown' self.__id = None self.__container = Unknown(self.__name) self.__hidden = hidden def __str__(self): return 'Variable[%s]' % self.__name def __get_var_id(self): """Get the unique 'id' for this variable.""" return self.__id def __set_var_id(self, var_id): """Set the unique 'id' for this variable.""" self.__id = var_id self.__container.vid = var_id vid = property(__get_var_id, __set_var_id) def get_name(self): """Get the variable name for this variable.""" return self.__name name = property(get_name) def __get_hidden(self): """Hide this variable from the Web UI?""" return self.__hidden hidden = property(__get_hidden) # called from the web interface def mode(self): """Get the mode of this variable.""" return self.__mode # called from the web interface def values(self): """Get the container of the values in this variable.""" # this must return an iterable return self.__container def __get_num_fetchers(self): """Get the count of fetchers in this variable.""" return self.__container.num_fetchers num_fetchers = property(__get_num_fetchers) def __get_num_finders(self): """Get the count of finders in this variable.""" return self.__container.num_finders num_finders = property(__get_num_finders) def __get_num_values(self): """Get the count of values in this variable.""" return len(self.__container) num_values = property(__get_num_values) def set_mode(self, mode): """Set the mode of this variable. This is called when a variable is declared. It sets the type of container used for the variable. Parameters: mode -- the variable mode (for instance, 'single', 'lifo', 'fifo') """ if _DEBUG: log.msg('set_mode(%s, %s)' % (str(self), mode)) if self.__mode == 'unknown': finders = self.__container.finders fetchers = self.__container.fetchers try: cont_type = CONTAINER_TYPES[mode] self.__container = cont_type(self.__name) self.__container.vid = self.vid self.__container.finders = finders self.__container.fetchers = fetchers self.__mode = mode if _DEBUG: log.msg('set_mode(%s, %s): new container type = %s' % (str(self), mode, str(type(self.__container)))) except KeyError: raise BadModeException("illegal mode specified") elif self.__mode != mode: raise BadModeException("mode is already set to incompatible value") def set_container(self, cont): """Specialized version of set_mode to be used from plugins to create variables using custom container classes.""" self.__mode = 'custom' self.__container = cont def new_value(self, val_index, val, metadata): """Publish a new value to the appropriate waiters.""" self.__container.new_value(val_index, val, metadata) def purge(self): """Purge this variable from the workspace.""" self.__container.purge() def format(self): """Format this variable for the 'list vars' command.""" return '%s\t%d\t%d\t%d\t%s' % (self.name, len(self.__container), self.__container.num_fetchers, self.__container.num_finders, self.__mode) def store(self, client, val, metadata): """Set the value of this variable, converting it to FIFO type if it is Unknown. Arguments: client -- client for whom to store val -- value to store """ if self.__mode == 'unknown': self.set_mode('fifo') self.__container.store(client, val, metadata) def fetch(self, client, is_blocking, val_index, metadata): """Do a fetch operation on this variable. Arguments: client - client for whom to fetch/find is_blocking - is this a blocking operation? val_index - value index, if this is an iterated operation metadata - metadata, if any """ return self.__container.fetch(client, is_blocking, val_index, metadata) def find(self, client, is_blocking, val_index, metadata): """Do a find operation on this variable. Arguments: client - client for whom to fetch/find is_blocking - is this a blocking operation? val_index - value index, if this is an iterated operation metadata - metadata, if any """ return self.__container.find(client, is_blocking, val_index, metadata) nwsserver-2.0.0/nwss/protocol.py0000644000175000017500000005501311214274547016407 0ustar westonweston# # Copyright (c) 2005-2009, REvolution Computing, Inc. # # NetWorkSpaces is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License as published # by the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 # USA # """Protocol implementation for NetWorkSpaces server.""" # TODO: In some cases, the client is expecting a long response and in some # cases, a short response. In many error conditions, we send only a # short response... Wacky Hijinks (TM) ensue. import os, time from tempfile import mkstemp from twisted.protocols import stateful from twisted.python import log from twisted.internet import reactor from nwss.base import Value, DIRECT_STRING, Response, ERROR_VALUE from nwss.protoutils import DictReceiver, ArgTupleReceiver, FileProducer from nwss.protoutils import map_proto_generator import nwss try: from twisted.internet.ssl import DefaultOpenSSLContextFactory except ImportError: DefaultOpenSSLContextFactory = None #pylint: disable-msg=C0103 _DEBUG = nwss.config.is_debug_enabled('NWS:protocol') def server_configured_ssl(): """Return True if the server has configured SSL support.""" return (nwss.config.serversslkey is not None and nwss.config.serversslcert is not None) def clear_server_ssl_config(): """Clear the server's SSL configuration to avoid spamming the logs.""" nwss.config.serversslkey = None nwss.config.serversslcert = None def ssl_is_available(): """Return True if the PyOpenSSL library is available on the server.""" return DefaultOpenSSLContextFactory is not None if server_configured_ssl() and not ssl_is_available(): log.msg("Failed to import PyOpenSSL. SSL support is disabled.") clear_server_ssl_config() class WsSessionStats(object): """Counter for some informational statistics about a given NWS session.""" def __init__(self): """Create a new session statistics counter.""" self.__num_operations = 0 self.__last_operation = '' self.__last_operation_time = '' self.__num_long_values = 0 def mark_operation(self, opname): """Mark the occurrence of an operation, updating all relevant statistics.""" self.__last_operation = opname self.__last_operation_time = time.asctime() self.__num_operations += 1 def mark_new_long_value(self): """Mark the creation of a new long value, updating all relevant statistics.""" self.__num_long_values += 1 def __get_num_operations(self): """Get the number of operations we've performed since this session started.""" return self.__num_operations num_operations = property(__get_num_operations) def __get_last_operation(self): """Get a tuple of the name and time of the last operation performed. For instance: ("store", "Sun Apr 19 20:04:42 PDT 2009") """ return self.__last_operation, self.__last_operation_time last_operation = property(__get_last_operation) def __get_num_long_values(self): """Get the number of values which have been stored through this connection which have resulted in the creation of "long value" files. """ return self.__num_long_values num_long_values = property(__get_num_long_values) class WsBlockingInfo(object): """Manager for the 'blocking' state of a protocol object. Responsible for ensuring that the protocol is on at most 1 waiter list, and is removed at the appropriate time.""" def __init__(self): self.__blocking = False self.__waiter_list = [] # Var's blocked clients list containing us self.__var = None # Var for which this conn. is blocking def block(self): """Put this session into the "blocking" state if it is not blocking already.""" if self.__blocking: return False self.__blocking = True return True def clear(self): """If this connection was blocking, mark it as no longer so. This will not remove the connection from any "waiter" lists, so if we may still be on a waiter list, we need to call remove_from_waiter_list first. """ self.__blocking = False self.__waiter_list = [] self.__var = None def remove(self, proto): """Remove us from whichever waiter list we appear in, if, indeed, we appear in a waiter list.""" try: self.__waiter_list.remove(proto) except ValueError: if _DEBUG: log.msg("Blocked client was not in blocked clients list.") self.clear() def set_var(self, var, waiter_list): """Set the variable into whose waiter list this connection has been entered, as well as the waiter list itself. When a value becomes availble, this information will be used to remove this connection from the appropriate waiter list.""" self.__var = var self.__waiter_list = waiter_list def __is_blocking(self): """Check if this connection is currently waiting on a response from the server code. This flag is briefly true for any operation, but may be true for an extended period of time if we are performing a blocking operation for a value not yet available.""" return self.__blocking blocking = property(__is_blocking) def __get_var(self): """Get the variable in whose waiter lists we appear. Used for monitoring purposes.""" return self.__var var = property(__get_var) def coerce_status(status): """Utility to coerce status to a 4-byte string in appropriate format for inclusion in the protocol. This is primarily to handle the integer -> string case.""" if isinstance(status, int): status = '%04d' % status elif not isinstance(status, str): log.msg('Internal error: status of %s [%s] returned to client' % (str(status), str(type(status)))) log.msg('Converting to string.') status = str(status) if len(status) != 4: log.msg('Internal error: status of %s returned to client' % status) log.msg('Truncating to 4 bytes.') status = status[0:4] return status class NwsProtocol(object, stateful.StatefulProtocol): #pylint: disable-msg=R0901,R0902 """Twisted protocol object for NetWorkSpaces server. The generic structure of the protocol consists mostly of fixed length ASCII sequences. The core of a NWS message consists of a 4-digit 0-padded ASCII decimal count of the elements of a tuple, which must always have at least one argument. The tuple elements are each serialized as a 20-digit 0-padded ASCII decimal length followed by raw bytes. The first element of the tuple is always a command-name, of which a dozen or so valid values exist. A 20-digit count is used to allow accommodation of any 64-bit value. The interpretation of the remainder of the elements in the tuple depend on the specific command used. """ DEFAULT_OPTIONS = { 'MetadataToServer': '', 'MetadataFromServer': '', 'KillServerOnClose': '', } if server_configured_ssl(): DEFAULT_OPTIONS['SSL'] = '' def __init__(self): # Twisted will initialize 'factory' to point at the NwsService self.factory = None # Twisted will initialize 'peer' to point at details about the client # self.peer = None # Twisted stashes a unique id here for this client self.__protokey = -1 # Connection options self.__metadata_receive = False self.__metadata_send = False self.__deadman = False self.__reply_long_preamble = self.__reply_long_preamble_nocookie # Session statistics self.__statistics = WsSessionStats() # Blocking info self.__blocking_state = WsBlockingInfo() def __str__(self): if hasattr(self, 'peer'): return 'NwsProtocol[%s]' % self.peer else: return 'NwsProtocol[not connected]' ####################################################### # Overrides for Twisted methods ####################################################### def connectionMade(self): #pylint: disable-msg=C0103 """Callback from Twisted after a new connection is made. Note that protocol objects are not reused, so the only real purpose of this method is to initialize state which requires access to the factory and transport objects.""" self.transport.setTcpNoDelay(1) self.transport.setTcpKeepAlive(1) # HACK: dig through the factory for the web port, add it to the # advertised options. if hasattr(self.factory, 'nwsWebPort'): port = str(self.factory.nwsWebPort()) self.DEFAULT_OPTIONS['NwsWebPort'] = str(port) def connectionLost(self, reason): #pylint: disable-msg=C0103,W0222 """Callback from Twisted to indicate that this connection has been shutdown. """ if _DEBUG: log.msg('connectionLost called') self.factory.goodbye(self) if self.__deadman: log.msg('stopping the server due to deadman switch') #pylint: disable-msg=E1101 reactor.stop() def getInitialState(self): #pylint: disable-msg=C0103 """Callback from Twisted to find the start state for this protocol. The NWS protocol always begins with a 4-byte handshake.""" return (self.__receive_handshake_request, 4) ####################################################### # Interface exposed to server ####################################################### def __get_protocol_key(self): """Attribute accessor for the "protocol key", which is actually a unique id for this client connection. (One of several, since twisted also assigns a unique sessionid...)""" return self.__protokey def __set_protocol_key(self, key): """Attribute mutator for the "protocol key".""" self.__protokey = key protokey = property(__get_protocol_key, __set_protocol_key) def get_peer(self): """Get a semi-human-readable textual identifier for the host on the other side of the connection. Generally something containing the IP address and port number for the remote side.""" return str(self.transport.getPeer()) peer = property(get_peer) def __get_num_operations(self): """Get the number of operations we've performed since connection creation.""" return self.__statistics.num_operations num_operations = property(__get_num_operations) def __get_last_operation(self): """Get a tuple of the name and time of the last operation performed. For instance: ("store", "Sun Apr 19 20:04:42 PDT 2009") """ return self.__statistics.last_operation last_operation = property(__get_last_operation) def __get_num_long_values(self): """Get the number of values which have been stored through this connection which have resulted in the creation of "long value" files. """ return self.__statistics.num_long_values num_long_values = property(__get_num_long_values) def __is_blocking(self): """Check if this connection is currently waiting on a response from the server code.""" return self.__blocking_state.blocking blocking = property(__is_blocking) def set_blocking_var(self, var, waiter_list): """Set the variable into whose waiter list this connection has been entered, as well as the waiter list itself.""" self.__blocking_state.set_var(var, waiter_list) def __get_blocking_var(self): """Get the variable in whose waiter lists we appear. Used for monitoring purposes.""" return self.__blocking_state.var blocking_var = property(__get_blocking_var) def remove_from_waiter_list(self): """Remove us from whichever waiter list we appear in, if, indeed, we appear in a waiter list.""" self.__blocking_state.remove(self) def mark_for_death(self): """Mark this connection as a deadman connection. When this connection is closed, it will stop the reactor, resulting in the shutdown of the server.""" self.__deadman = True def new_long_arg_file(self): """Allocate a new file for a long argument. The file will be uniquely named and securely created in the NWS temporary directory.""" self.__statistics.mark_new_long_value() try: filedesc, tmpname = mkstemp(prefix='__nwss', suffix='.dat', dir=nwss.config.tmpdir) return os.fdopen(filedesc, 'w+b'), tmpname except OSError, exc: log.msg('error creating temporary file: ' + str(exc)) return None ####################################################### # Generic protocol utilities ####################################################### def __send_dictionary(self, dictionary): """Marshal and write the contents of a dictionary to the transport in the canonical form, as interpreted by the DictReceiver utility.""" maplen = len(dictionary) self.transport.write('%04d' % maplen) #pylint: disable-msg=W0141 map(self.transport.write, map_proto_generator(dictionary)) ####################################################### # Handshake protocol machinery ####################################################### def __receive_handshake_request(self, data): """Receive a handshake request from the client-side. This is the entry point to the NWS protocol.""" if _DEBUG: log.msg('handshake initiated with: ' + repr(data)) # New-style handshake if data.startswith('X'): self.__reply_long_preamble = self.__reply_long_preamble_cookie self.__send_options_advertise(self.DEFAULT_OPTIONS) return (self.__receive_options_request, 4) # Old-style handshake if data not in ['0000', '1111']: self.__reply_long_preamble = self.__reply_long_preamble_cookie self.transport.write('2223') # Beginning of the protocol proper. return self.__get_command_state() def __send_options_advertise(self, opts): """Send an options advertisement to the client with a list of the options the server supports as well as required or forbidden options.""" self.transport.write('P000') self.__send_dictionary(opts) def __receive_options_request(self, data): """Receive a "connection options" request packet from the client or terminate the connection.""" if data != 'R000': log.msg('Client send invalid handshake response.') self.transport.loseConnection() receiver = DictReceiver(self, self.__receive_connection_options) return receiver.start, receiver.start_count def __receive_connection_options(self, options): """Callback from the protocol handlers when we have a handshake options negotiation request.""" if self.__validate_connection_options(options): self.__process_connection_options(options) if options.get('SSL') == '1': if ssl_is_available(): key = nwss.config.serversslkey cert = nwss.config.serversslcert ctx = DefaultOpenSSLContextFactory(key, cert) self.__send_accept_connection() self.transport.startTLS(ctx) else: # SSL requested, but not available server side log.msg('Internal error: SSL not available.') self.__send_deny_connection() return None else: self.__send_accept_connection() return self.__get_command_state() else: self.__send_deny_connection() return None def __validate_connection_options(self, options): """Check that the requested connection options are compatible with our advertised options.""" for opt, val in options.items(): if not self.DEFAULT_OPTIONS.has_key(opt): return False elif (self.DEFAULT_OPTIONS[opt] != '' and self.DEFAULT_OPTIONS[opt] != val): return False return True def __process_connection_options(self, options): """Read through the connection options, pulling out options which are of interest to us.""" if options.get("KillServerOnClose") == "1": self.__deadman = True if options.get("MetadataToServer") == "1": self.__metadata_receive = True if options.get("MetadataFromServer") == "1": self.__metadata_send = True def __send_deny_connection(self): """Deny the client's connection request and shut down the connection.""" self.transport.write('F000') self.transport.loseConnection() log.msg('Dropped connection after handshake: invalid option requested') def __send_accept_connection(self): """Accept the client's connection request.""" self.transport.write('A000') ####################################################### # Command protocol machinery ####################################################### def __get_command_state(self): """Get the protocol state for the start of a new command request. This varies depending on whether metadata from the client is enabled.""" if self.__metadata_receive: return DictReceiver(self, self.__receive_metadata).start, 4 else: return self.__get_args_state(metadata={}) def __get_args_state(self, metadata): """Get the protocol state for the start of the arguments proper in a command request, populating the state with the specified metadata map.""" return ArgTupleReceiver(self, self.__handle_command, metadata).start, 4 def __receive_metadata(self, metadata): """Receive a metadata map from the client side and advance the protocol to the argument list state.""" return self.__get_args_state(metadata) def __handle_command(self, args, metadata): """Handle a command from the client.""" if not self.__blocking_state.block(): self.send_error('Received a request while already blocking on a ' + 'command.') self.transport.loseConnection() if len(args) < 1: log.msg('Empty argument list') self.send_error('Received an empty argument list.') self.transport.loseConnection() return None #pylint: disable-msg=W0142 self.factory.handle_command(self, metadata, *args) self.__statistics.mark_operation(args[0]) return self.__get_command_state() def __reply_long_preamble_cookie(self, response): """Send the "cookie protocol" version of a long reply preamble.""" self.transport.write('%s%020d%-20.20s%020d%020d' % (response.status, response.value.type_descriptor, response.iterstate[0], response.iterstate[1], response.value.length())) def __reply_long_preamble_nocookie(self, response): #pylint: disable-msg=W0613 """Send the no-"cookie protocol" version of a long reply preamble.""" self.transport.write('%s%020d%020d' % (response.status, response.value.type_descriptor, response.value.length())) def send_error(self, reason, status=1, long_reply=False): """Utility to send an error reply.""" metadata = { 'nwsReason': reason } response = Response(metadata) response.status = status if long_reply: response.value = ERROR_VALUE self.send_long_response(response) else: self.send_short_response(response) def send_short_response(self, response=None): """Send a response to a query which expects a "short" response.""" if response is None: response = Response() assert response.value is None assert response.iterstate is None # This operation is obviously no longer blocking self.__blocking_state.clear() # Coerce the status to a 4-digit string response.status = coerce_status(response.status) # Send the metadata if self.__metadata_send: self.__send_dictionary(response.metadata) # Send the reply self.transport.write(response.status) def send_long_response(self, response=None): """Send a response to a query which expects a "long" response.""" if response is None: response = Response(value=ERROR_VALUE) assert response.value is not None if response.iterstate is None: response.iterstate = ('', 0) # This operation is obviously no longer blocking self.__blocking_state.clear() # Coerce the response to a Value if not isinstance(response.value, Value): response.value = str(response.value) if isinstance(response.value, str): response.value = Value(DIRECT_STRING, response.value) # Coerce the status to a 4-digit string response.status = coerce_status(response.status) # Send the metadata if self.__metadata_send: self.__send_dictionary(response.metadata) # Send the reply itself self.__reply_long_preamble(response) if response.value.is_large(): if _DEBUG: log.msg("using long value protocol") producer = FileProducer(response.value, self.transport) self.transport.registerProducer(producer, None) else: self.transport.write(response.value.val()) nwsserver-2.0.0/nwss/workspace.py0000644000175000017500000005366011217710530016540 0ustar westonweston# # Copyright (c) 2005-2009, REvolution Computing, Inc. # # NetWorkSpaces is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License as published # by the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 # USA # """ Core NetWorkSpaces server - workspace implementation. """ from __future__ import generators from random import randint from twisted.internet import reactor, defer, task from twisted.python import log from nwss.base import ServerException, NoSuchVariableException from nwss.base import WorkspaceFailure from nwss.base import Response from nwss.stdvars import Variable, BaseVar import nwss _DEBUG = nwss.config.is_debug_enabled('NWS:workspace') class OperationFailure(ServerException): """Used to signal the failure of an operation inside a plugin and communicate back an error message to the user. """ def __init__(self, message, code=1): ServerException.__init__(self, message) self.return_code = code def call_after_delay(delay, func, *args, **kw): """Utility to schedule a particular call to happen at a specific point in the future. Utility for plugin development. Parameters: delay - delay before call (in seconds) func - function to call ... - arguments to function """ deferred = defer.Deferred() deferred.addCallback(lambda ignored: func(*args, **kw)) #pylint: disable-msg=W0142,C0301 reactor.callLater(delay, deferred.callback, None) #pylint: disable-msg=E1101,C0301 class WorkSpace(object): #pylint: disable-msg=R0902 """Server-side object representing a workspace in the NetWorkSpace server.""" def __init__(self, name): """Initializer for a WorkSpace object. Parameters: name - name for workspace """ self.__bindings = {} # str (name) -> Variable self.__vars_by_id = {} # str (id) -> Variable self.name = name self.owner = '' self.persistent = False self.__periodic_tasks = [] self.__have_hidden = False def __is_owned(self): """Synthetic 'owned' attribute.""" return (self.owner != '') owned = property(__is_owned) def __str__(self): return 'WorkSpace[%s]' % self.name ############################################################### # Private interface exposed to NwsService ############################################################### def _started(self, metadata): """Callback indicating that this workspace has been created and is active. """ self.__hook('created_ws', metadata) def _stopped(self): """Callback indicating that this workspace has been shutdown and will be destroyed. """ for periodic_task in self.__periodic_tasks: periodic_task.stop() self.__hook('destroyed_ws') def _declare_var(self, name, mode, metadata): #pylint: disable-msg=W0613 """Declare a variable to be of a particular mode. Currently defined modes are 'lifo', 'fifo', 'single', '__time', and '__barrier'. Parameters: name - name of the variable mode - 'fifo', 'lifo', etc. metadata - metadata passed in from the client """ var = self.__get_var_object(name) var.set_mode(mode) def _fetch_var(self, name, client, is_blocking, iterstate, metadata): #pylint: disable-msg=R0913 """Fetch a value from a variable. This encompasses all of the different {,i}fetch{Try,} variants of the operation. Parameters: name - name of the variable client - protocol object from whom request originated is_blocking - True if blocking, False if a *Try variant iterstate - iteration state - ('', -1) for non-iterated metadata - metadata passed in from the client """ var = self.__get_var_object(name) # Check for variable id mismatch on an iterated op if iterstate[0]: if iterstate[0] != var.vid: raise WorkspaceFailure('Variable id mismatch.') # Run the initial hook self.__hook('fetch_pre', var, iterstate[1], is_blocking, metadata) # Fetch the next value, if possible response = var.fetch(client, is_blocking, iterstate[1], metadata) if response is None: return None # Stash the iter state if the variable hasn't written its own if response.iterstate is None: response.iterstate = var.vid, max(0, iterstate[1]) return response def _find_var(self, name, client, is_blocking, iterstate, metadata): #pylint: disable-msg=R0913 """Find a value from a variable. This encompasses all of the different {,i}find{Try,} variants of the operation. Parameters: name - name of the variable client - protocol object from whom request originated is_blocking - True if blocking, False if a *Try variant iterstate - iteration state - ('', -1) for non-iterated metadata - metadata passed in from the client """ var = self.__get_var_object(name) # Check for variable id mismatch on an iterated op if iterstate[0]: if iterstate[0] != var.vid: raise WorkspaceFailure('Variable id mismatch.') # Run the initial hook self.__hook('find_pre', var, iterstate[1], is_blocking, metadata) # Find the value, if possible response = var.find(client, is_blocking, iterstate[1], metadata) if response is None: return None # Stash the iter state if the variable hasn't written its own if response.iterstate is None: response.iterstate = var.vid, max(0, iterstate[1]) return response def _set_var(self, name, client, val, metadata): """Store a value into a variable. Parameters: name - name of the variable client - protocol object from whom request originated val - value to store metadata - metadata passed in from the client """ var = self.__get_var_object(name) self.__hook('store_pre', var, val, metadata) var.store(client, val, metadata) self.__hook('store_post', var, val, metadata) def _delete_var(self, name, metadata): """Delete a variable. Parameters: name - name of the variable metadata - metadata passed in from the client """ self.__hook('delete_pre', name, metadata) self.__delete_var_object(name) self.__hook('delete_post', name, metadata) return 0 def _get_bindings(self, hide=False): """Get the set of all variable bindings in this workspace. Parameters: hide - do we exclude 'hidden' bindings? """ if hide and not self.__have_hidden: hide = False if not hide: return self.__bindings else: bindings = {} for key, value in self.__bindings.items(): if not value.hidden: bindings[key] = value return bindings def _get_binding(self, name, hide=False): """Get a particular variable binding. Parameters: name - name of the variable hide - do we exclude 'hidden' bindings? """ var = self.__bindings[name] if hide and var.hidden: return None return var def _set_owner_info(self, owner, persistent, metadata): """Set the ownership of this workspace if it isn't owned yet. If the workspace is unowned, it will take on the owner and persistence properties of whichever client caused this to be called. This is called when someone opens a workspace that they are willing to assert ownership of, if the workspace exists and noone has previously asserted ownership of the space. Parameters: owner - owner of client for whom this method is called persistent - request the space to be persistent """ status = False if not self.owned: self.__hook('setowner_pre', owner, persistent, metadata) self.owner = owner self.persistent = persistent status = True self.__hook('setowner_post', owner, persistent, metadata) return status ############################################################### # Private implementation ############################################################### def __has_hook(self, name): """Check for a named hook function. Parameters: name - hook name """ hookname = 'hook_' + name return hasattr(self, hookname) def __hook(self, name, *args): """Call a named hook function. Parameters: name - hook name args - arguments to pass to hook """ hookname = 'hook_' + name if hasattr(self, hookname): hook = getattr(self, hookname) return hook(*args) return None def __allocate_var_id(self): """Allocate a unique id for a variable.""" for _ in range(1000): new_id = '%020u' % randint(0, 999999999) if new_id not in self.__vars_by_id: return new_id else: raise ServerException('Internal error: Failed to allocate a ' + 'unique variable id.') def __get_var_object(self, name, create=True): """Get and/or create a variable in this workspace. If the variable has not been created and 'create' is true, it will be created as a simple variable. Parameters: name - name of variable create - flag indicating whether the variable may be created """ try: var = self.__bindings[name] except KeyError: if not create: var = None else: var = Variable(name, False) self.add_variable(var) return var def __delete_var_object(self, name): """Remove a variable from this workspace. Parameters: name - name of variable """ if not self.remove_variable(name): raise NoSuchVariableException('no variable named %s' % name) ############################################################### # Interface exposed to plugin classes ############################################################### def call_hook(self, name, *args): """Interface to allow plugins to use the same 'hook' mechanism as the base workspace. Parameters: name - hook name (method will be hook_) args - args to pass to hook """ return self.__hook(name, *args) def add_periodic_task(self, period, periodic_task): """Add a periodic task to this workspace which will be called every 'period' seconds. This is intended for use by the plugins. Parameters: period - periodicity of task in seconds periodic_task - 0-args function to call periodically """ call = task.LoopingCall(periodic_task) self.__periodic_tasks.append(call) call.start(period) def create_standard_var(self, varname, mode, hidden=True): """Utility to simplify the creation of standard variables. Parameters: varname - name of variable mode - variable mode hidden - should the var be hidden from the web UI? """ var = Variable(varname, hidden) var.set_mode(mode) self.add_variable(var) def create_var(self, container, hidden=False): """Utility to simplify the creation of custom variable types. Parameters: container - custom container for variable hidden - should the var be hidden from the web UI? """ var = Variable(container.name, hidden) var.set_container(container) self.add_variable(var) def add_variable(self, var): """Add a new variable to this workspace, setting its unique ID and adding it to all appropriate indices. Parameters: var - variable to add """ var_id = self.__allocate_var_id() var.vid = var_id self.__vars_by_id[var_id] = var self.__bindings[var.name] = var if var.hidden: self.__have_hidden = True def remove_variable(self, name): """Remove a variable from this workspace by name. Parameters: name - variable name to remove """ try: return self.remove_variable_by_id(self.__bindings[name].vid) except KeyError: return False def remove_variable_by_id(self, var_id): """Remove a variable from this workspace by unique id. Parameters: var_id - variable id to remove """ try: var = self.__vars_by_id.pop(var_id) var.purge() try: del self.__bindings[var.name] except KeyError: pass return True except KeyError: return False def get_variable(self, name): """Fetch a variable from this workspace by name. Parameters: name - variable name to fetch """ return self.__bindings.get(name) def get_variable_by_id(self, var_id): """Fetch a variable from this workspace by unique id. Parameters: var_id - variable id to fetch """ return self.__vars_by_id.get(var_id) def purge(self, metadata=None): """Purge all variables in this workspace. Parameters: metadata - metadata passed in from the client """ if metadata is None: metadata = {} self.__hook('purge_pre', metadata) names = self.__bindings.keys() for name in names: var = self.__bindings.pop(name) var.purge() self.__vars_by_id.clear() self.__hook('purge_post', metadata) class GetRequest(object): #pylint: disable-msg=R0903 """Encapsulation of a fetch/find request used by the simplified plugin system. """ def __init__(self, name, metadata): self.name = name self.blocking = True self.remove = True self.iterstate = None self.metadata = metadata class SetRequest(object): #pylint: disable-msg=R0903 """Encapsulation of a store request used by the simplified plugin system. """ def __init__(self, name, value, metadata): self.name = name self.value = value self.metadata = metadata def constant_fail(message, status=1): """Utility to create functions which fail with a custom message and status code. Useful for plugins. """ def handler(*args): #pylint: disable-msg=W0613 """Generic exception-thrower.""" raise WorkspaceFailure(message, status) return handler def constant_return(value): """Utility to create functions which return a constant value. Useful for writing plugins. """ def handler(*args): #pylint: disable-msg=W0613 """Generic constant function.""" return value return handler def function_return(func): """Utility to create functions which return a value from a no-args function call. Useful for writing plugins. """ def handler(*args): #pylint: disable-msg=W0613 """Generic 0-arg function caller.""" return func() return handler def singleton_iter_provider(func): """Utility to implement iteration functionality on a new style plugin variable. """ def handler(*args): #pylint: disable-msg=W0613 """Generic singleton iterator functionality.""" val = func() if val is None: return [], 0 else: return [val], 1 return handler class MetaVariable(BaseVar): """Utility variable "container" type to simplify implementation of custom variables in workspace plugins. """ def __init__(self, name, get_handler, iter_provider=None, cb_handler=None): BaseVar.__init__(self, name) self.get_handler = get_handler if iter_provider is None: self.iter_handler = constant_return(([], 0)) else: self.iter_handler = iter_provider if cb_handler is None: self.cb_handler = constant_return(False) else: self.cb_handler = cb_handler self.set_handler = constant_fail('Store is not supported for this ' + 'variable.') self.unwrap_stores = True def __len__(self): _, count = self.iter_handler() return count def __iter__(self): iterable, _ = self.iter_handler() return iter(iterable) def store(self, client, value, metadata): """Handle a store request for a metavariable.""" if self.unwrap_stores: request = SetRequest(self.name, value.val(), metadata) else: request = SetRequest(self.name, value, metadata) self.set_handler(client, request) def __value_callback(self, client, request, value): """Callback to announce the unblocking of a fetch/find operation.""" if request.remove: try: self.fetchers.remove(client) except ValueError: log.msg('Blocking fetcher was not in the fetchers list for ' + self.name) return else: try: self.finders.remove(client) except ValueError: log.msg('Blocking finder was not in the finders list for ' + self.name) return if isinstance(value, OperationFailure): client.send_error(value.args[0], value.return_code, True) return if not isinstance(value, Response): data = str(value.value) metadata = value.metadata if request.iterstate is None: val_index = 0 else: val_index = request.iterstate[1] value = Response(metadata, data) value.iterstate = self.vid, val_index client.send_long_response(value) def __handle_get(self, client, request): """Generic fetch/find implementation.""" try: value = self.get_handler(client, request) if value is not None: if not isinstance(value, Response): value = Response(value=value) if request.iterstate is not None: val_index = request.iterstate[1] + 1 value.iterstate = request.iterstate[0], val_index return value elif not request.blocking: raise OperationFailure('no value available.') else: thunk = lambda val: self.__value_callback(client, request, val) if request.remove: self.add_fetcher(client) else: self.add_finder(client) if not self.cb_handler(client, request, thunk): raise OperationFailure( 'Metavariable did not add callback.') else: return None except OperationFailure, fail: if fail.return_code == 0: fail.return_code = 1 raise WorkspaceFailure(fail.args[0], fail.return_code) def fetch(self, client, blocking, val_index, metadata): """Fetch implementation for metavariables.""" request = GetRequest(self.name, metadata) request.blocking = blocking request.remove = True if val_index is not None: request.iterstate = (self.vid, val_index) return self.__handle_get(client, request) def find(self, client, blocking, val_index, metadata): """Find implementation for metavariables.""" request = GetRequest(self.name, metadata) request.blocking = blocking request.remove = False if val_index is not None: request.iterstate = (self.vid, val_index) return self.__handle_get(client, request) def purge(self): """Purge this variable from the workspace, causing any clients waiting for a value to fail. """ self.fail_waiters('Variable purged.') def simple_metavariable(name, func): """Utility to create a simple metavariable which does not accept store requests, on which fetch and find always succeed, and on which the value returned from a fetch/find is taken from a 0-arg function. """ return MetaVariable(name, function_return(func), singleton_iter_provider(func)) nwsserver-2.0.0/nwss/engines.py0000644000175000017500000000443611214274547016201 0ustar westonweston# # Copyright (c) 2005-2009, REvolution Computing, Inc. # # NetWorkSpaces is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License as published # by the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 # USA # """ Static configuration of Babelfish translators for various scripting languages. """ __all__ = ['BABEL_ENGINES', 'MONITOR_ENGINES'] def matlab_translation(deferred, val): """Translation callback for Matlab removes trailing newline.""" deferred.callback(val[:-2]) # strip new lines. def passthrough_translation(deferred, val): """Default callback passes data to the callback verbatim.""" deferred.callback(val) BABEL_ENGINES = {1: ('Python babelfish', passthrough_translation), 2: ('Matlab babelfish', matlab_translation), 3: ('R babelfish', passthrough_translation), 4: ('Perl babelfish', passthrough_translation), 5: ('Ruby babelfish', passthrough_translation), 6: ('Octave babelfish', passthrough_translation), 7: ('Java babelfish', passthrough_translation), 8: ('CSharp babelfish', passthrough_translation), 9: ('ObjC babelfish', passthrough_translation), 10: ('Elisp babelfish', passthrough_translation)} MONITOR_ENGINES = ( ('Sleigh Monitor', 'Sleigh Monitor', ('nodeList', 'totalTasks', 'rankCount', 'workerCount'), ('imagefile',) ), ('Nws Utility', 'Nws Utility', ('enableNwsUtility',), ('varName', 'value') ), ('Nws Configurator', 'Nws Configurator', ('enableNwsConfigurator',), ('varName', 'value') ), ('chat example', 'Chat Service', ('chat',), ('msg', 'from') ), ) nwsserver-2.0.0/nwss/server.py0000644000175000017500000007747411261677130016070 0ustar westonweston# # Copyright (c) 2005-2009, REvolution Computing, Inc. # # NetWorkSpaces is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License as published # by the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 # USA # # server for NetWorkSpaces. """ Core NetWorkSpaces server. """ import os, traceback from tempfile import mkstemp from twisted.internet import protocol from twisted.python import log from nwss.protocol import NwsProtocol from nwss.protoutils import WsTracker, WsNameMap try: from nwss.web import NwsWeb except ImportError: # Disable web interface -- couldn't load nwss.web NwsWeb = None #pylint: disable-msg=C0103 from nwss.base import BadModeException from nwss.base import NoSuchVariableException from nwss.base import WorkspaceFailure from nwss.base import Value from nwss.base import Response from nwss.workspace import WorkSpace import nwss _DEBUG = nwss.config.is_debug_enabled('NWS:server') def var_list_csv(bindings): """Produce a variable list from the given dictionary of bindings. Arguments: bindings -- the dictionary """ k = bindings.keys() k.sort() return ','.join(k) def get_int_ws_name(client, ext_name, long_reply): """Get the internal name for a workspace. Arguments: client - the client state in which to look up the name ext_name - the user-visible workspace name to look up long_reply - send a long reply in case of an error? """ int_name = client.workspace_names.get(ext_name) if int_name is None: client.send_error('Workspace %s has not been opened.' % ext_name, 2001, long_reply) return int_name def plugin_score_function(plug): """Plugins are ordered by their PRIORITY class fields, with an omitted priority counting as a 0.""" if hasattr(plug, 'PRIORITY'): return plug.PRIORITY else: return 0 def find_best_plugin(plugins): """Order the list of plugins by the scoring metric specified by the plugin_score_function. Technically, we aren't ordering the list, but simply selecting the best item.""" best_pri = None best_plugin = None for plugin in plugins: pri = plugin_score_function(plugin) if best_pri is None or pri > best_pri: best_plugin = plugin best_pri = pri return best_plugin PLUGINS_ENABLED = True try: from pkg_resources import Environment, working_set except ImportError: PLUGINS_ENABLED = False ALL_PLUGINS = None def load_plugin(name): """Plugin loader. This is responsible for interfacing to the EGG loader and finding the best plugin for a given plugin type.""" if not PLUGINS_ENABLED: log.msg('Plugin requested, but pkg_resources is not installed.') # Replace the body of this function so we don't get a lot of noise. dummy_load_plugin = lambda name: None #pylint: disable-msg=W0612 load_plugin.func_code = dummy_load_plugin.func_code return None if _DEBUG: log.msg('Loading plugin "%s"' % name) global ALL_PLUGINS #pylint: disable-msg=W0603 if ALL_PLUGINS is None: if _DEBUG: log.msg('Scanning plugin dirs') ALL_PLUGINS = working_set for i in nwss.config.nwsPluginDirs: if _DEBUG: log.msg('Scanning plugin dir "%s"' % i) ALL_PLUGINS.add_entry(i) env = Environment(nwss.config.nwsPluginDirs) ALL_PLUGINS.require(*[i for i in env]) entry_points = ALL_PLUGINS.iter_entry_points('nws.workspace', name) plugins = [i.load() for i in entry_points] if _DEBUG: log.msg('Found %d matching plugins' % len(plugins)) if len(plugins) == 0: log.msg('Request for plugin type "%s" found no plugins' % name) return None return find_best_plugin(plugins) def create_space(ext_name, metadata): """Create a workspace object with the given name, and metadata. Parameters: ext_name - name for the new space metadata - metadata to control space creation """ if _DEBUG: log.msg('metadata = ' + str(metadata)) # Find the right constructor if metadata.has_key('wstype'): ws_type = metadata['wstype'] log.msg('Creating plugin workspace type "%s" (name "%s")' % (ws_type, ext_name)) ctor = load_plugin(ws_type) if ctor is None: log.msg("ERROR: Failed to create workspace - " + "couldn't load plugin.") raise WorkspaceFailure("Failed to create workspace: " + "couldn't load plugin.") else: log.msg('Creating standard workspace (name "%s")' % ext_name) ctor = WorkSpace # Create the space space = ctor(ext_name) space._started(metadata) #pylint: disable-msg=W0212 return space class NwsService(protocol.ServerFactory): #pylint: disable-msg=W0212 """The NWS Service itself, in suitable form to attach to a Twisted server. """ def __init__(self): """Initialize an NWS service.""" # keep a list of the protocol objects that we've created self.protocols = {} self.__protokey = 0 self.__tmp_filename = None self.ws_basename = None # Create default space default_ext_name = '__default' default_space = WorkSpace(default_ext_name) default_space._set_owner_info('[system]', True, None) default_int_name = default_ext_name, 0 default_space.internal_name = default_int_name self.__ext_to_int_ws_name = { default_ext_name : default_int_name } self.spaces = { default_int_name : default_space } self.__ws_counter = 1 #################################################### # Twisted interface #################################################### # Twisted uses this to determine which protocol handler to instantiate protocol = NwsProtocol def buildProtocol(self, addr): #pylint: disable-msg=C0103 """Build a protocol object for a NWS connection. This is called from the Twisted framework and expects us to return a newly created protocol object. Arguments: addr -- address of remote side """ proto = protocol.Factory.buildProtocol(self, addr) proto.protokey = self.__protokey self.protocols[proto.protokey] = proto self.__protokey += 1 if _DEBUG: log.msg('built a new protocol[%d]: %s' % (proto.protokey, str(self.protocols))) proto.owned_workspaces = WsTracker() proto.workspace_names = WsNameMap() return proto def startFactory(self): #pylint: disable-msg=C0103 """Callback when this factory is started.""" log.msg('NetWorkSpaces Server version %s' % nwss.__version__) if NwsWeb is None: log.msg('WARNING: The web interface is not available, ' 'probably because twisted.web is not installed') tmpdir = nwss.config.nwsTmpDir log.msg('using temp directory ' + tmpdir) # XXX: Why do we create this temp file? Creating a unique random # workspace name can be done more easily without resorting to # this. tmpfile, self.__tmp_filename = mkstemp(prefix='__nwss', dir=tmpdir) self.ws_basename = os.path.basename(self.__tmp_filename) try: os.close(tmpfile) except OSError: pass def stopFactory(self): #pylint: disable-msg=C0103 """Callback when this factory is stopped.""" log.msg('stopping NwsService') try: os.remove(self.__tmp_filename) except OSError: pass # purge all WorkSpace objects, which will remove the temp files # currently in use for int_name, space in self.spaces.items(): try: space.purge() space._stopped() except (KeyboardInterrupt, SystemExit): raise except Exception: #pylint: disable-msg=W0703 log.msg("error while purging workspace %s" % int_name[0]) traceback.print_exc() log.msg('stopping complete') #################################################### # Interface to web UI #################################################### def get_ext_to_int_mapping(self): """Get the external-to-internal name mapping for all current workspaces.""" return self.__ext_to_int_ws_name #################################################### # Mechanics of workspace creation/destruction #################################################### def __reference_space(self, ext_name, client, can_create, metadata): """Reference a workspace, causing it to spring into existence if it does not exist and the client is willing to create the workspace. Arguments: ext_name - the workspace name client - client connection can_create - can we create the workspace? metadata - metadata to use for space creation """ if metadata is None: metadata = {} if not self.__ext_to_int_ws_name.has_key(ext_name): # return an error if the workspace shouldn't be created if not can_create: return None # we use a separate internal name that allows us to track # instances. e.g., workspace 'foo' is created, deleted and created # again. a connection using the first may map 'foo' to the internal # "name" the tuple '(foo, 1)' while a connection using the second # may map 'foo' to '(foo, 7)'. Here, we build the internal name. int_name = (ext_name, self.__ws_counter) self.__ws_counter += 1 # Create the workspace space = create_space(ext_name, metadata) space.internal_name = int_name # Store the space in a two-level lookup: # ext_name -> int_name -> space self.spaces[int_name] = space self.__ext_to_int_ws_name[ext_name] = int_name else: # Look up the space in a two-level lookup: # ext_name -> int_name -> space int_name = self.__ext_to_int_ws_name[ext_name] space = self.spaces[int_name] # Debug mode only: Check for and log the case where the client had an # out of date external -> internal mapping for the workspace name. if _DEBUG: old_int_name = client.workspace_names.get(ext_name) if old_int_name is not None and old_int_name != int_name: log.msg('connection has new reference (%s, %s)' % (old_int_name, int_name)) # Update the client's external -> internal name mapping client.workspace_names.set(ext_name, int_name) return space def goodbye(self, client): """Signal the closure of a given client connection. Arguments: client - client connection """ self.__client_disconnected(client) # this method is called by the NwsProtocol object when # it loses it's connection def __client_disconnected(self, client): """Handle the closure (intentional or accidental) of a client connection. Arguments: client - client connection """ self.__purge_workspaces_for_client(client) if client.blocking: client.remove_from_waiter_list() try: del client.factory.protocols[client.protokey] if _DEBUG: log.msg("after removing protocol: " + str(client.factory.protocols)) except KeyError: log.msg("Internal error: Protocol lost connection, " + "but was not in the connection map.") traceback.print_exc() def __purge_workspaces_for_client(self, client): """Purge all workspaces opened by a given client connection. Arguments: client -- client connection """ if _DEBUG: log.msg('purging owned workspaces') for int_name in client.owned_workspaces.as_list(): if _DEBUG: log.msg('purging %s' % str(int_name[0])) try: if not self.spaces[int_name].persistent: space = self.spaces.pop(int_name) space.purge({}) space._stopped() try: self.__ext_to_int_ws_name.pop(int_name[0]) except KeyError: log.msg('WARNING: workspace name "%s" is not known', int_name[0]) except KeyError: log.msg('workspace no longer exists: %s' % str(int_name)) except Exception: log.msg("unexpected error while purging client's workspaces") traceback.print_exc() client.owned_workspaces.clear() #################################################### # Mechanics of command dispatch #################################################### def __find_workspace(self, client, ext_name, long_reply=False): """Look up a given workspace, or produce an error. Arguments: client - client connection for whom to access the workspace ext_name - user-visible workspace name long_reply - send a long reply in case of an error? """ int_name = get_int_ws_name(client, ext_name, long_reply) if int_name is None: return None try: workspace = self.spaces[int_name] except KeyError: # this probably means the workspace was deleted by another client client.send_error('No such workspace.', 100, long_reply) return None return workspace def handle_command(self, client, metadata, *args): """Perform the requested workspace operation. Arguments: client -- client connection, prepopulated with the requested workspace operation """ # dispatch try: self.OPERATIONS[args[0]](self, client, metadata=dict(metadata), *args) except KeyError: client.send_error('Unknown verb "%s"' % args[0]) except Exception: log.msg('ignoring unexpected exception') log.msg('protocol arguments: ' + str(args)) traceback.print_exc() ####### Command handler: "declare var" def cmd_declare_var(self, client, op_name, ext_name, var_name, mode, metadata=None): #pylint: disable-msg=W0613,R0913 """NWS Command handler: Declare a variable, creating it if it did not exist, or setting its mode if it was formerly unknown. Arguments: client - client connection op_name - operation name (unused here) ext_name - the workspace name var_name - the variable name to declare mode - the mode for the created variable """ # convert null metadata to empty metadata if metadata is None: metadata = {} # find the workspace workspace = self.__find_workspace(client, ext_name) if workspace is None: return # declare the variable try: workspace._declare_var(var_name, mode, metadata) client.send_short_response() except BadModeException: client.send_error('Cannot change variable mode to "%s".' % mode) except Exception, exc: client.send_error('Internal error: "%s".' % str(exc), 2000) raise ####### Command handler: "delete ws" def cmd_delete_workspace(self, client, op_name, ext_name, metadata=None): #pylint: disable-msg=W0613 """NWS Command handler: Delete the workspace. Arguments: client - client connection op_name - operation name (unused here) ext_name - the workspace name """ # convert null metadata to empty metadata if metadata is None: metadata = {} # delete the workspace try: int_name = self.__ext_to_int_ws_name.pop(ext_name) space = self.spaces.pop(int_name) space.purge(metadata) space._stopped() client.workspace_names.remove(ext_name) client.owned_workspaces.remove(int_name) client.send_short_response() except KeyError: log.msg('workspace "%s" does not exist.' % ext_name) client.send_error('Workspace "%s" does not exist.' % ext_name) except Exception, exc: client.send_error('Internal error: "%s".' % str(exc), 2000) raise ####### Command handler: "delete var" def cmd_delete_var(self, client, op_name, ext_name, var_name, metadata=None): #pylint: disable-msg=W0613,R0913 """NWS Command handler: Delete the variable. Arguments: client - client connection op_name - operation name (unused here) ext_name - the workspace name var_name - the variable name metadata - metadata for this operation """ # convert null metadata to empty metadata if metadata is None: metadata = {} # find the workspace workspace = self.__find_workspace(client, ext_name) if workspace is None: return # delete the variable try: workspace._delete_var(var_name, metadata) client.send_short_response() except NoSuchVariableException: client.send_error('Variable "%s" does not exist in workspace "%s".' % (var_name, ext_name)) except Exception, exc: client.send_error('Internal error: "%s".' % str(exc), 2000) raise # tuples here encode the properties remove, block, and iterate. GET_OP_PROPERTIES = { #### name remove block iterate 'fetch': (True, True, False), 'fetchTry': (True, False, False), 'find': (False, True, False), 'findTry': (False, False, False), 'ifetch': (True, True, True), 'ifetchTry': (True, False, True), 'ifind': (False, True, True), 'ifindTry': (False, False, True), } ####### Command handler: "fetch", "fetchTry", "find", "findTry" ####### Command handler: "ifetch", "ifetchTry", "ifind", "ifindTry" def cmd_get(self, client, op_name, ext_name, var_name, var_id='', val_index='-999', metadata=None): #pylint: disable-msg=R0913 """NWS Command handler: Get a value from the variable. Arguments: client - client connection op_name - operation name (fetch, find, findtry, ifind, etc.) ext_name - the workspace name var_name - the variable name var_id - the variable id (to detect var. re-creation) val_index - the index of the value to read (for iterated operations) """ # Get the operation properties props = self.GET_OP_PROPERTIES[op_name] # convert null metadata to empty metadata if metadata is None: metadata = {} # Trim whitespace from variable id. We do this because the variable id # is sent in a fixed-width space-padded field. var_id = var_id.strip() # If this isn't an iterated op, var_id and val_index should be cleared if not props[2]: var_id = '' val_index = -1 # Convert val_index to an int (always -1 if we have no var_id) if var_id: val_index = int(val_index) else: val_index = -1 # Find the workspace workspace = self.__find_workspace(client, ext_name, long_reply=True) if workspace is None: return # Perform the operation try: iterstate = (var_id, val_index) if props[0]: response = workspace._fetch_var(var_name, client, props[1], iterstate, metadata) else: response = workspace._find_var(var_name, client, props[1], iterstate, metadata) # If the return from the workspace is None, we are blocked, and we # are not presently responsible for sending a reply; instead, a # reply will be triggered by a later store or by special # functionality in a workspace plugin. if response is not None: client.send_long_response(response) except WorkspaceFailure, exc: client.send_error(exc.args[0], exc.status, long_reply=True) except Exception, exc: client.send_error('Internal error: "%s".' % str(exc), 2000, long_reply=True) raise ####### Command handler: "list vars" def cmd_list_vars(self, client, op_name, ext_name, metadata=None): """NWS Command handler: List the variables in a workspace. Arguments: client - client connection op_name - operation name (not used here) ext_name - the workspace name """ #pylint: disable-msg=W0613 # convert null metadata to empty metadata if metadata is None: metadata = {} # find the workspace workspace = self.__find_workspace(client, ext_name, long_reply=True) if workspace is None: return # list the variables try: bindings = workspace._get_bindings() varkeys = bindings.keys() varkeys.sort() var_listing = '\n'.join([bindings[var_name].format() for var_name in varkeys]) client.send_long_response(Response(value=var_listing)) except Exception, exc: client.send_error('Internal error: "%s".' % str(exc), 2000, long_reply=True) raise ####### Command handler: "list wss" def cmd_list_workspaces(self, client, op_name, ext_name_wanted=None, metadata=None): #pylint: disable-msg=W0613 """NWS Command handler: List the workspaces in this server. Arguments: client - client connection op_name - operation name (not used here) ext_name_wanted - the workspace name, or None to list all workspaces """ # Collect the relevant spaces spaces = [] if not ext_name_wanted: all_ext_names = self.__ext_to_int_ws_name.keys() all_ext_names.sort() for ext_name in all_ext_names: # Translate to internal name int_name = self.__ext_to_int_ws_name.get(ext_name) if int_name is None: continue # Get the space space = self.spaces.get(int_name) if space is None: continue spaces.append(space) else: try: int_name = self.__ext_to_int_ws_name[ext_name_wanted] space = self.spaces[int_name] spaces.append(space) except KeyError: pass # Format each space space_list = [] for space in spaces: int_name = self.__ext_to_int_ws_name[space.name] i_own_this = client.owned_workspaces.contains(int_name) bindings = space._get_bindings() space_list.append('%s%s\t%s\t%s\t%d\t%s' % (' >'[i_own_this], int_name[0], space.owner, space.persistent, len(bindings), var_list_csv(bindings))) # Send on the response space_list = '\n'.join(space_list) + '\n' client.send_long_response(Response(value=space_list)) ####### Command handler: "mktemp ws" def cmd_make_temp_workspace(self, client, op_name, template='__ws__%d', metadata=None): #pylint: disable-msg=W0613 """NWS Command handler: Make a temporary, uniquely named workspace. Arguments: client - client connection op_name - operation name (not used here) template - sprintf-style format taking a single integer argument for workspace name """ if metadata is None: metadata = {} # step the counter on every attempt. for _ in range(1000): my_count = self.__ws_counter self.__ws_counter += 1 # Build the name try: new_name = (template % my_count) + self.ws_basename except (ValueError, OverflowError, TypeError): msg = 'mktemp: bad template "%s".' % template log.msg(msg) client.send_error(msg, long_reply=True) return # If we've found a unique name, we're done. if not self.__ext_to_int_ws_name.has_key(new_name): break else: new_name = None # If we didn't succeed after 100 tries... if new_name is None: msg = 'mktemp: failed to generate unique name using "%s".' % \ template log.msg(msg) client.send_error(msg, long_reply=True) return # make a non-owning reference (triggering existence). try: space = self.__reference_space(new_name, # ext_name client, # client True, # can_create metadata) # metadata assert space is not None client.send_long_response(Response(value=new_name)) except WorkspaceFailure, fail: log.msg('Internal error: __reference_space failed unexpectedly.') log.msg(' %s' % fail.args[0]) client.send_error('Internal error: reference_space failed', long_reply=True) ####### Command handler: "open ws", "use ws" def cmd_open_workspace(self, client, op_name, ext_name, owner_label, persistent_str, create_str='yes', metadata=None): #pylint: disable-msg=R0913 """NWS Command handler: Open a workspace. Arguments: client - client connection op_name - operation name ext_name - the user-visible workspace name owner_label - extra label for the owner string of a workspace persistent_str - should the workspace be persistent? ('yes'/'no') create_str - should we create the workspace? ('yes'/'no') """ if metadata is None: metadata = {} create = create_str == 'yes' persistent = persistent_str == 'yes' try: space = self.__reference_space(ext_name, client, create, metadata) # Maybe claim ownership if op_name == 'open ws' and space is not None: owner = '%s (%s)' % (client.peer, owner_label) space._set_owner_info(owner, persistent, metadata) client.owned_workspaces.add(space.internal_name) # If space is None, we weren't allowed to create ws. if space is None: client.send_error('No such workspace.', 100) # HACK: return failure value to be used by web ui return -1 else: client.send_short_response() # HACK: return success value to be used by web ui return 0 except WorkspaceFailure, fail: client.send_error(fail.args[0], fail.status) # HACK: return failure value to be used by web ui return -1 ####### Command handler: "store" def cmd_store(self, client, op_name, ext_name, var_name, type_desc, data, metadata=None): #pylint: disable-msg=W0613,R0913 """NWS Command handler: Perform a store operation on a given variable. Arguments: client - client connection op_name - operation name (unused) ext_name - workspace name var_name - variable name type_desc - value type descriptor (in string form) data - data to store to the variable """ # convert null metadata to empty metadata if metadata is None: metadata = {} # find the workspace workspace = self.__find_workspace(client, ext_name) if workspace is None: return # store the value try: value = Value(int(type_desc), data) workspace._set_var(var_name, client, value, metadata) client.send_short_response() except WorkspaceFailure, fail: client.send_error(fail.args[0], fail.status) except Exception, exc: client.send_error('Internal error: "%s".' % str(exc), 2000) raise ####### Command handler: "deadman" def cmd_deadman(self, client, op_name, metadata=None): #pylint: disable-msg=W0613,R0201 """NWS Command handler: Deadman operation, signalling the shutdown of the server. Arguments: client - client connection op_name - operation name """ if metadata is None: metadata = {} client.mark_for_death() client.send_short_response() ####### Command dispatch map OPERATIONS = { 'declare var': cmd_declare_var, 'delete ws': cmd_delete_workspace, 'delete var': cmd_delete_var, 'fetch': cmd_get, 'fetchTry': cmd_get, 'find': cmd_get, 'findTry': cmd_get, 'ifetch': cmd_get, 'ifetchTry': cmd_get, 'ifind': cmd_get, 'ifindTry': cmd_get, 'list vars': cmd_list_vars, 'list wss': cmd_list_workspaces, 'mktemp ws': cmd_make_temp_workspace, 'open ws': cmd_open_workspace, 'store': cmd_store, 'use ws': cmd_open_workspace, 'deadman': cmd_deadman, } nwsserver-2.0.0/nwss/config.py0000644000175000017500000000211211214274547016003 0ustar westonweston# # Copyright (c) 2005-2009, REvolution Computing, Inc. # # NetWorkSpaces is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License as published # by the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 # USA # #pylint: disable-msg=C0103 """Configuration settings used throughout the NWS server.""" import tempfile nwsServerPort = 8765 nwsWebPort = 8766 nwsWebServedDir = 'clientCode' nwsTmpDir = tempfile.gettempdir() nwsLongValueSize = 16 * 1024 * 1024 nwsServerSslCert = None nwsServerSslKey = None nwsPluginDirs = ['./plugins'] debug = [] nwsserver-2.0.0/nwss/web.py0000644000175000017500000010315611261677175015333 0ustar westonweston# # Copyright (c) 2005-2009, REvolution Computing, Inc. # # NetWorkSpaces is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License as published # by the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 # USA # # TODO: uniformly cope with get encodings # TODO: rework Translator/Monitor """NWS Web UI implementation.""" import traceback import os from cgi import escape from urllib import quote_plus from twisted.web import resource, server as webserver, static from twisted.internet import defer from twisted.python import log from nwss.mock import DummyConnection from nwss.engines import MONITOR_ENGINES from nwss.engines import BABEL_ENGINES import nwss from nwss.base import Value, DIRECT_STRING from nwss.webtemplates import * #pylint: disable-msg=W0401 PYTHON_ENVIRONMENT = 0x01000000 _DEBUG = nwss.config.is_debug_enabled('NWS:web') def translate_value(server, value, callback, *extra_args): """Translate a value by sending it to the appropriate babelfish, sending on the translated value to the supplied callback. Arguments: server - server whose babelfish to attach to value - value to translate callback - callback to receive babelfish-translated value extra_args - arguments for callback """ if not isinstance(value, Value): callback(str(value), *extra_args) return if value.is_large(): callback('', *extra_args) return elif value.type_descriptor & DIRECT_STRING: callback(value.val(), *extra_args) return deferred = defer.Deferred() deferred.addCallback(callback, *extra_args) environment_id = (value.type_descriptor >> 24) & 0xFF try: babelfish_ws_name, val_callback = BABEL_ENGINES[environment_id] def send_reply(status, metadata, value): #pylint: disable-msg=W0613 """Callback for data from the babelfish.""" log.msg('Value.val(): %s' % repr(value.val())) val_callback(deferred, value.val()) client = DummyConnection(send_reply) status = use_workspace(server, client, babelfish_ws_name, False) if status == 0: server.cmd_store(client, 'store', babelfish_ws_name, 'food', value.type_descriptor, value.val()) server.cmd_get(client, 'fetch', babelfish_ws_name, 'doof') else: deferred.callback('[error: %s not running]' % babelfish_ws_name) except KeyError: deferred.callback('[error: unknown babel engine]') def get_binding(space, var_name): """Get the variable binding ``var_name`` for the workspace ``space``, or None if there is no such binding. Parameters: space - the workspace object var_name - the variable binding to get """ #pylint: disable-msg=W0212 return space._get_binding(var_name, True) def get_bindings(space): """Get all variable bindings for the workspace ``space``. Parameters: space - the workspace object """ #pylint: disable-msg=W0212 return space._get_bindings(True) def use_workspace(server, conn, ws_name, create=True): """Open the workspace ``ws_name`` for the given connection to the NWS server. Parameters: server - the NWS server conn - the connection to the server ws_name - the workspace name """ if create: return server.cmd_open_workspace(conn, 'use ws', ws_name, '', 'no') else: return server.cmd_open_workspace(conn, 'use ws', ws_name, '', 'no', 'no') def has_all_keys(dictionary, klist): """Check if ``dictionary`` has every key in the iterable ``klist``. Parameters: dictionary - the dictionary to check klist - the iterable containing the keys """ for key in klist: if not dictionary.has_key(key): return False return True # here are a number of functions that generate html for error messages def errpage_no_workspace(ws_name): """Build an HTML error page indicating that the workspace ``ws_name`` was not found. Parameters: ws_name - missing workspace name. """ message = 'There is currently no workspace named %s.' % escape(ws_name) return make_errpage(message) def errpage_monitor_not_running(mon_name): """Build an HTML error page indicating that the monitor ``mon_name`` was not running. Parameters: mon_name - missing monitor name. """ message = '%s not running.' % escape(mon_name) return make_errpage(message) def errpage_invalid_ws_for_monitor(ws_name): """Build an HTML error page indicating that the workspace ``ws_name`` was invalid for the requested monitor. Parameters: ws_name - invalid workspace """ message = 'Invalid workspace specified for monitor: %s' % escape(ws_name) return make_errpage(message) def errpage_variable_not_found(var_name, ws_name): """Build an HTML error page indicating that the variable ``var_name`` was not found in the workspace ``ws_name``. Parameters: ws_name - the workspace var_name - the variable """ message = 'No variable named %s currently in "%s"' % \ (escape(var_name), escape(ws_name)) return make_errpage(message, wsname=ws_name) def errpage_no_monitor(mon_name): """Build an HTML error page indicating that the monitor ``mon_name`` was not found. Parameters: mon_name - missing monitor name. """ message = 'No monitor found for %s' % escape(mon_name) return make_errpage(message) def errpage_malformed_request(): """Build an HTML error page indicating that the last request was somehow malformed. """ return make_errpage('Malformed request.') def errpage_monitor_error(msg): """Build an HTML error page indicating some error has occurred inside the monitor for a workspace. Parameters: msg - description of the error """ message = 'Error monitoring workspace: %s' % msg return make_errpage(message) def errpage_static_server_error(dirname): """Build an HTML error page indicating a problem with the static content directory ``dirname``. Parameters: dirname - the static content dir which was invalid """ message = 'Cannot serve files from directory "%s".' % dirname return make_errpage(message) def infopage_ws_deleted(ws_name): """Build an HTML info page indicating that the workspace ``ws_name`` has been successfully deleted. Parameters: ws_name - the workspace which was deleted """ message = 'NetWorkSpace "%s" was deleted.' % escape(ws_name) return make_infopage(message, 'Workspace Deleted', refresh_url='doit?op=listWss') def infopage_var_deleted(ws_name, var_name): """Build an HTML info page indicating that the variable ``var_name`` has been successfully deleted from the workspace ``ws_name``. Parameters: ws_name - the workspace name var_name - the variable which was deleted """ message = 'Variable "%s" in "%s" was deleted.' % \ (escape(var_name), escape(ws_name)) refresh_url = 'doit?op=listVars&wsName=%s&varName=%s' % \ (quote_plus(ws_name), quote_plus(var_name)) return make_infopage(message, 'Variable Deleted', refresh_url=refresh_url, wsname=ws_name) def infopage_var_fetched(ws_name, var_name): """Build an HTML info page indicating that the variable ``var_name`` has been successfully fetched (i.e. its first value removed) from the workspace ``ws_name``. Parameters: ws_name - the workspace name var_name - the variable which was fetched """ message = 'Value in variable "%s" was (possibly) removed.' % \ escape(var_name) refresh_url = 'doit?op=showVar&wsName=%s&varName=%s' % \ (quote_plus(ws_name), quote_plus(var_name)) return make_infopage(message, 'Variable Deleted', refresh_url=refresh_url, wsname=ws_name, varname=var_name) class Translator: #pylint: disable-msg=R0903 """Utility class to encapsulate communication with the Babelfish.""" def __init__(self, request, ws_name, var_name, num_values): self.request = request self.ws_name = ws_name self.var_name = var_name self.num_values = num_values self.pop = 0 self.queue = [None] * self.num_values self.title = 'Values in %s' % escape(self.var_name) def run(self, server, var_vals, truncated): """Send the variable values to the Babelfish and stream the response to the HTTP client. """ url = 'doit?op=showVar&wsName=%s&varName=%s' % \ (quote_plus(self.ws_name), self.var_name) menubase = menu_provider_var(self.ws_name, self.var_name) menu = menu_provider_refresh(url, menubase) self.request.write(make_header(self.title, menu)) if truncated: msg = 'Display has been truncated to %d values.' % self.num_values self.request.write(make_div('warning', msg)) self.request.write('
    \n') # we kick off all translations here, rather than ping/pong-ing # them. we do this to avoid problems with the list of values # changing as the process unfolds. if 0 == self.num_values: self.__close_page() else: for idx, val in zip(xrange(self.num_values), var_vals): translate_value(server, val, self.__callback, idx) def __callback(self, text, idx): """Collect translated values from the Babelfish. Parameters: text - translated text idx - index of translated item """ self.queue[idx] = escape(text) self.pop += 1 if self.pop == self.num_values: oddness = 0 for item in self.queue: if item is None: log.msg('Unescaped text: %s' % repr(text)) self.request.write('
  1. \n' % (EVEN_ODD[oddness])) else: self.request.write('
  2. %s
    3. \n' % (EVEN_ODD[oddness], item.replace('\n', '
    '))) oddness = 1 - oddness self.__close_page() def __close_page(self): """Write the page footer, closing the enclosed HTML tags.""" self.request.write('
\n') self.request.write(make_footer(self.title)) self.request.finish() class Monitor: #pylint: disable-msg=R0903 """Communicates with one of the monitors.""" def __init__(self, server, request, mon_name, reply_var_name): self.server = server self.request = request self.mon_name = mon_name self.arg_count = -1 self.error = 0 self.reply_var_name = reply_var_name self.dummy_conn = DummyConnection(self.__send_reply, peer_id='Monitor') def run(self, ws_name): """Run this monitor attached to the workspace ``ws_name``, pushing the result out to the HTTP client. """ mon_name = self.mon_name request = self.request eng = [me for me in MONITOR_ENGINES if me[0] == mon_name] if not eng: log.msg('error: no monitor found for ' + mon_name) self.request.write(errpage_no_monitor(mon_name)) self.request.finish() return # 'args' is a tuple of the request arguments that the monitor # is interested in args = eng[0][3] use_workspace(self.server, self.dummy_conn, mon_name) if _DEBUG: log.msg('Monitor: storing request') # note that this doesn't currently support multiple values for arguments monargs = {} numargs = 0 for arg in args: if request.args.has_key(arg): monargs[arg] = request.args[arg] numargs += len(monargs[arg]) monargs['wsName'] = [ws_name] monargs['replyVarName'] = [self.reply_var_name] numargs += 2 # note: this code assumes only one monitor process per workspace. # store the number of arguments, followed by the arguments # in NAME=VALUE form desc = PYTHON_ENVIRONMENT | DIRECT_STRING self.server.cmd_store(self.dummy_conn, 'store', mon_name, 'request', desc, str(numargs)) for key, val_list in monargs.items(): assert type(val_list) == list for val in val_list: assert type(val) == str self.server.cmd_store(self.dummy_conn, 'store', mon_name, 'request', desc, key + '=' + val) if _DEBUG: log.msg('Monitor: fetching reply') self.server.cmd_get(self.dummy_conn, 'fetch', mon_name, self.reply_var_name) if _DEBUG: log.msg('Monitor: get returned') def __handle_header_count(self, data): """Handle the header count. The header count is the first item sent back from the monitor, and gives the number of HTTP headers which the monitor is going to send. Parameters: data - the text of the header count field """ arg_count = -1 if self.error: log.msg('Monitor: ignoring header count due to previous ' + \ 'error (argCount < 0)') else: try: arg_count = int(data) except ValueError: log.msg('Monitor: got bad value for header count: %s' % \ data) self.error = 1 self.request.setHeader('content-type', 'text/html') response = 'got bad value for header count' self.request.write(errpage_monitor_error(response)) self.request.finish() else: if _DEBUG: log.msg('Monitor: %d headers coming' % arg_count) self.server.cmd_get(self.dummy_conn, 'fetch', self.mon_name, self.reply_var_name) return arg_count def __handle_header_value(self, data): """Handle a header value. A header value is an item of the form key=value sent back from the monitor, which is turned into an HTTP header sent back to the client. Parameters: data - the text of the header field """ if self.error: log.msg('Monitor: ignoring header due to previous ' + \ 'error (argCount = %d)' % (self.arg_count + 1,)) self.server.cmd_get(self.dummy_conn, 'fetch', self.mon_name, self.reply_var_name) else: try: key, val = [x.strip() for x in data.split('=', 1)] except ValueError: log.msg('Monitor: error handling header specification: %s' % \ data) self.error = 1 self.request.setHeader('content-type', 'text/html') response = errpage_monitor_error( 'error handling header specification') self.request.write(response) self.request.finish() else: if _DEBUG: log.msg('Monitor: got a header; %s = %s' % \ (repr(key), repr(val))) self.request.setHeader(key, val) if _DEBUG: log.msg('Monitor: got a header; %d more coming' % \ self.arg_count) self.server.cmd_get(self.dummy_conn, 'fetch', self.mon_name, self.reply_var_name) def __handle_payload(self, status, data): """Handle the payload. The payload is raw data to send back to the HTTP client from the monitor. Parameters: data - the text of the payload """ if _DEBUG: log.msg('Monitor: got the payload') if self.error: log.msg('Monitor: ignoring payload value due to previous ' + \ 'error (argCount is 0)') else: if status: # XXX: this is incorrect if the browser expects an image... log.msg('Monitor: bad status = ' + str(status)) self.request.setHeader('content-type', 'text/html') page = errpage_monitor_error('status = %d' % status) self.request.write(page) else: self.request.write(data) self.request.finish() # it will be an error if __send_reply is called again self.error = 1 self.server.delete_var(self.dummy_conn, 'delete var', self.mon_name, self.reply_var_name) if _DEBUG: log.msg('Monitor: finished browser reply') def __send_reply(self, status, metadata, value): #pylint: disable-msg=W0613, R0913 """Callback for the dummy connection to receive replies from the monitor. """ if value is None: return if _DEBUG: log.msg('Monitor: monitor replied') if self.arg_count < 0: self.arg_count = self.__handle_header_count(value.val()) elif self.arg_count > 0: self.__handle_header_value(value.val()) self.arg_count -= 1 else: self.__handle_payload(status, value.val()) class NwsWebDynamic(resource.Resource): """Twisted Web handler for dynamic content. Displays a view of the internal state of the NWS server. """ isLeaf = True # never call getChild, go to render_GET directly. def __init__(self, nws_server): resource.Resource.__init__(self) self.int_names = nws_server.get_ext_to_int_mapping() self.spaces = nws_server.spaces self.dummy_conn = DummyConnection(peer_id='NwsWebDynamic') self.nws_server = nws_server self.monid = 0 def __get_space(self, ext_name): """Get the workspace whose external name is ``ext_name``. Returns None if the workspace cannot be found. Parameters: ext_name - the workspace external name """ int_name = self.int_names.get(ext_name) if int_name is None: return None return self.spaces.get(int_name) def __confirm_delete_var(self, request): #pylint: disable-msg=R0201 """Handler for the ``confirmDeleteVar`` page.""" var_name = request.args['varName'][0] ws_name = request.args['wsName'][0] fields = { 'wsname': escape(ws_name), 'varname': escape(var_name), } content = CONFIRM_DELETE_VAR_TEMPLATE % fields return make_page('Confirm Variable Deletion', content, menu=make_menu(menu_provider_ws(ws_name))) def __confirm_delete_ws(self, request): #pylint: disable-msg=R0201 """Handler for the ``confirmDeleteWs`` page.""" ws_name = request.args['wsName'][0] fields = { 'wsname': escape(ws_name), } content = CONFIRM_DELETE_WS_TEMPLATE % fields return make_page('Confirm Workspace Deletion', content) def __confirm_fetchtry_var(self, request): #pylint: disable-msg=R0201 """Handler for the ``confirmFetchTryVar`` page.""" ws_name = request.args['wsName'][0] var_name = request.args['varName'][0] fields = { 'wsname': escape(ws_name), 'varname': escape(var_name), } content = CONFIRM_FETCHTRY_VAR_TEMPLATE % fields return make_page('Confirm FetchTry', content, menu=make_menu(menu_provider_var(ws_name, var_name))) def __delete_var(self, request): """Handler for the ``deleteVar`` page.""" ws_name = request.args['wsName'][0] var_name = request.args['varName'][0] # Get space space = self.__get_space(ws_name) if space is None: return errpage_no_workspace(ws_name) # Get binding value = get_binding(space, var_name) if value is None: return errpage_variable_not_found(var_name, ws_name) use_workspace(self.nws_server, self.dummy_conn, ws_name) self.nws_server.delete_var(self.dummy_conn, 'delete var', ws_name, var_name) return infopage_var_deleted(ws_name, var_name) def __delete_ws(self, request): """Handler for the ``deleteWs`` page.""" ws_name = request.args['wsName'][0] # Get space space = self.__get_space(ws_name) if space is None: return errpage_no_workspace(ws_name) self.nws_server.cmd_delete_workspace(self.dummy_conn, 'delete ws', ws_name) return infopage_ws_deleted(ws_name) def __list_wss(self, request): #pylint: disable-msg=W0613 """Handler for the ``listWss`` page.""" ext_names = self.int_names.keys() ext_names.sort() version = escape(nwss.__version__) title = 'NetWorkSpaces %s' % version ws_list_html = make_header(title, menu_provider_default('Refresh')) ws_list_html += WS_LIST_TABLE_HEADER oddness = 0 for ext_name in ext_names: space = self.__get_space(ext_name) bindings = get_bindings(space) monitors = [mentry for mentry in MONITOR_ENGINES if has_all_keys(bindings, mentry[2])] fields = { 'class': EVEN_ODD[oddness], 'wsname': escape(ext_name), 'wsnameQ': quote_plus(ext_name), 'owner': escape(space.owner), 'persistent': str(space.persistent), 'numbindings': len(bindings), } if len(monitors) > 1: ws_list_html += WS_LIST_TABLE_ENTRY_MULTIMON % fields elif len(monitors) == 1: mon_name = monitors[0][0] display_name = monitors[0][1] fields['monnameQ'] = quote_plus(mon_name) fields['monname'] = escape(display_name) ws_list_html += WS_LIST_TABLE_ENTRY_SINGLEMON % fields else: ws_list_html += WS_LIST_TABLE_ENTRY_NOMON % fields oddness = 1 - oddness ws_list_html += '\n' ws_list_html += make_footer(title) return ws_list_html def __show_monitor(self, request): """Handler for the ``showMonitor`` page.""" ws_name = request.args['wsName'][0] mon_name = request.args['monName'][0] # Get space space = self.__get_space(ws_name) if space is None: return errpage_no_workspace(ws_name) vbindings = get_bindings(space) # Get monitor space mon_space = self.__get_space(mon_name) if mon_space is None: return errpage_monitor_not_running(mon_name) # verify that the ws_name workspace qualifies to be handled by mon_name monlist = [mentry[0] for mentry in MONITOR_ENGINES if has_all_keys(vbindings, mentry[2])] if mon_name not in monlist: return errpage_invalid_ws_for_monitor(ws_name) reply_name = 'reply_%d' % self.monid mon = Monitor(self.nws_server, request, mon_name, reply_name) mon.run(ws_name) self.monid += 1 if self.monid > 1000000: self.monid = 0 return webserver.NOT_DONE_YET def __list_monitors(self, request): """Handler for ``listMonitors`` page.""" ws_name = request.args['wsName'][0] # Get space space = self.__get_space(ws_name) if space is None: return errpage_no_workspace(ws_name) vbindings = get_bindings(space) # Build content content = '

\n

    \n' for monitor in MONITOR_ENGINES: if not has_all_keys(vbindings, monitor[2]): continue content += '''\
  • %s
    • ''' % (quote_plus(ws_name), quote_plus(monitor[0]), escape(monitor[1])) content += '
\n' return make_page('Monitors for %s' % escape(ws_name), content) def __list_vars(self, request): """Handler for ``listVars`` page.""" ws_name = request.args['wsName'][0] # Get space space = self.__get_space(ws_name) if space is None: return errpage_no_workspace(ws_name) vbindings = get_bindings(space) var_names = vbindings.keys() var_names.sort() content = VAR_LIST_HEADER fields = { 'wsname': escape(ws_name), 'wsnameQ': quote_plus(ws_name), } oddness = 0 for var_name in var_names: var = vbindings[var_name] fields['class'] = EVEN_ODD[oddness] fields['varname'] = escape(var_name) fields['varnameQ'] = quote_plus(var_name) fields['mode'] = var.mode() fields['numvalues'] = var.num_values fields['numfetchers'] = var.num_fetchers fields['numfinders'] = var.num_finders content += VAR_LIST_ENTRY % fields oddness = 1 - oddness content += VAR_LIST_FOOTER return make_page('Variables in %s' % escape(ws_name), content, menu=menu_provider_ws(ws_name, 'Refresh')) def __show_var(self, request): """Handler for ``showVar`` page.""" ws_name = request.args['wsName'][0] var_name = request.args['varName'][0] # Get space space = self.__get_space(ws_name) if space is None: return errpage_no_workspace(ws_name) vbinding = get_binding(space, var_name) if vbinding is None: return errpage_variable_not_found(var_name, ws_name) var_vals = vbinding.values() # truncate the number of values to something reasonable num_values = vbinding.num_values if num_values > 1000: num_values = 1000 truncated = True else: truncated = False xlat = Translator(request, ws_name, var_name, num_values) xlat.run(self.nws_server, var_vals, truncated) return webserver.NOT_DONE_YET def __fetchtry_var(self, request): """Handler for ``fetchTryVar`` page.""" ws_name = request.args['wsName'][0] var_name = request.args['varName'][0] use_workspace(self.nws_server, self.dummy_conn, ws_name) self.nws_server.cmd_get(self.dummy_conn, 'fetchTry', ws_name, var_name) return infopage_var_fetched(ws_name, var_name) def __show_server_info(self, request): #pylint: disable-msg=W0613,R0201 """Handler for ``showServerInfo`` page.""" fields = { 'nwsversion': escape(nwss.__version__), 'nwsport': nwss.config.nwsServerPort, 'webport': nwss.config.nwsWebPort, 'tmpdir': escape(nwss.config.nwsTmpDir), 'longvaluesize': nwss.config.nwsLongValueSize, } return make_page('Server Info', SERVER_INFO_TEMPLATE % fields) def __list_clients(self, request): #pylint: disable-msg=W0613 """Handler for ``listClients`` page.""" content = CLIENT_LIST_HEADER oddness = 0 for client in self.nws_server.protocols.values(): last_op, last_time = client.last_operation if client.blocking_var is not None: blocking_var = escape(client.blocking_var) else: blocking_var = '' fields = { 'class': EVEN_ODD[oddness], 'peer': escape(client.peer), 'sessionno': client.transport.sessionno, 'numops': client.num_operations, 'numlong': client.num_long_values, 'numws': client.owned_workspaces.count, 'lastop': escape(last_op), 'blocking': str(client.blocking), 'blockingvar': blocking_var, 'lasttime': escape(last_time) } content += CLIENT_LIST_ENTRY % fields oddness = 1 - oddness content += CLIENT_LIST_FOOTER return make_page('Clients', content, menu=menu_provider_default(clientslabel='Refresh')) OP_TABLE = { 'confirmDeleteVar': __confirm_delete_var, 'confirmDeleteWs': __confirm_delete_ws, 'deleteVar': __delete_var, 'deleteWs': __delete_ws, 'confirmFetchTryVar': __confirm_fetchtry_var, 'fetchTryVar': __fetchtry_var, 'listVars': __list_vars, 'listWss': __list_wss, 'showVar': __show_var, 'showMonitor': __show_monitor, 'listMonitors': __list_monitors, 'listClients': __list_clients, 'showServerInfo': __show_server_info, } def render_GET(self, request): #pylint: disable-msg=C0103 """Callback from Twisted Web to field GET requests.""" try: request.setHeader('cache-control', 'no-cache') op_name = request.args.get('op', ['listWss'])[0] return self.OP_TABLE.get(op_name, self.__list_wss)(self, request) except (KeyboardInterrupt, SystemExit): raise except Exception, exc: #pylint: disable-msg=W0703 traceback.print_exc() log.err(exc) return errpage_malformed_request() def render_POST(self, request): #pylint: disable-msg=C0103 """Callback from Twisted Web to field POST requests.""" try: request.setHeader('cache-control', 'no-cache') op_name = request.args.get('op', ['listWss'])[0] return self.OP_TABLE.get(op_name, self.__list_wss)(self, request) except (KeyboardInterrupt, SystemExit): raise except Exception, exc: #pylint: disable-msg=W0703 traceback.print_exc() log.err(exc) return errpage_malformed_request() class NwsWeb(resource.Resource): """Root web server for NWS web service. Contains a static directory and a dynamic directory, the latter of which allows browsing the internal state of the NWS server. """ def __init__(self, nws_server): resource.Resource.__init__(self) self.dynamic = NwsWebDynamic(nws_server) log.msg('clientCode served from directory ' + nwss.config.nwsWebServedDir) if os.path.isdir(nwss.config.nwsWebServedDir): client_code = static.File(nwss.config.nwsWebServedDir) client_code.contentTypes.update({ '.m': 'text/plain', '.M': 'text/plain', '.py': 'text/plain', '.PY': 'text/plain', '.r': 'text/plain', '.R': 'text/plain'}) else: log.msg("clientCode directory doesn't exist") page = errpage_static_server_error(nwss.config.nwsWebServedDir) client_code = static.Data(page, 'text/html') self.putChild('clientCode', client_code) def getChild(self, name, request): #pylint: disable-msg=W0613,C0103 """Callback from Twisted to get the handler for a specific URL. In this case, unless it matched the 'static content' directory, we want to forward it to the dynamic content provider. """ return self.dynamic nwsserver-2.0.0/nwss/pyutils.py0000644000175000017500000000271111214274547016254 0ustar westonweston# # Copyright (c) 2008-2009, REvolution Computing, Inc. # # NetWorkSpaces is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License as published # by the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 # USA # """A few utilities for portability across Python versions.""" __all__ = ['new_list', 'remove_first'] try: #pylint: disable-msg=C0103 from collections import deque #pylint: disable-msg=W8313 new_list = deque #pylint: disable-msg=C0103 if not hasattr(deque, 'remove'): raise ImportError("'deque' class is missing the 'remove' attribute.") remove_first = deque.popleft clear_list = deque.clear except ImportError: #pylint: disable-msg=C0103 deque = None new_list = list remove_first = lambda l: l.pop(0) def clear_list(the_list): """Replacement for deque.clear when we have to use a list instead of a deque.""" del the_list[:] nwsserver-2.0.0/nwss/webtemplates.py0000644000175000017500000003137411214274547017246 0ustar westonweston# # Copyright (c) 2005-2009, REvolution Computing, Inc. # # NetWorkSpaces is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License as published # by the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 # USA # '''Templates used in the Web Interface.''' from urllib import quote_plus EVEN_ODD = ['even', 'odd'] # modified from twisted's default web file serving. _STYLE_BLOCK = '''\ ''' _HEADER_TEMPLATE = ''' %(title)s %(refresh)s %(style)s

%(title)s

%(menu)s ''' _FOOTER_TEMPLATE = ''' ''' _PAGE_TEMPLATE = ''' %(header)s %(content)s %(footer)s ''' _REFRESH = ''' ''' CONFIRM_DELETE_VAR_TEMPLATE = '''

Really delete variable "%(varname)s"?
''' CONFIRM_DELETE_WS_TEMPLATE = '''
Really delete workspace "%(wsname)s"?
''' CONFIRM_FETCHTRY_VAR_TEMPLATE = '''
Really fetchTry (i.e. remove) variable "%(varname)s"?
''' WS_LIST_TABLE_HEADER = ''' ''' WS_LIST_TABLE_ENTRY_NOMON = '''''' WS_LIST_TABLE_ENTRY_SINGLEMON = '''''' WS_LIST_TABLE_ENTRY_MULTIMON = '''''' VAR_LIST_HEADER = '''

Name Monitor Owner Persistent # Variables Delete?
%(wsname)s [none] %(owner)s %(persistent)s %(numbindings)d
%(wsname)s %(monname)s %(owner)s %(persistent)s %(numbindings)d
%(wsname)s list monitors %(owner)s %(persistent)s %(numbindings)d
''' VAR_LIST_ENTRY = ''' ''' VAR_LIST_FOOTER = '''\
Variable # Values # Fetchers # Finders Mode Delete?
%(varname)s %(numvalues)d %(numfetchers)d %(numfinders)d %(mode)s
''' SERVER_INFO_TEMPLATE = '''

Parameter Value
Version %(nwsversion)s
NWS Server Port %(nwsport)d
Web Interface Port %(webport)d
Temp Directory %(tmpdir)s
Long Value Size %(longvaluesize)d
''' CLIENT_LIST_HEADER = '''

''' CLIENT_LIST_ENTRY = '''\ ''' CLIENT_LIST_FOOTER = '''\
Client Session # # Ops # Long Values # Wss Owned Last Op Blocking Ws and var Time of Last Op
%(peer)s %(sessionno)d %(numops)d %(numlong)d %(numws)d %(lastop)s %(blocking)s %(blockingvar)s %(lasttime)s
''' def make_div(content, cls): """Utility to assemble a DIV html element. Parameters: content - html for inside the DIV cls - CSS class for the DIV """ return '

%s
' % (cls, content) def menu_provider_default(nwslabel='NetWorkSpaces', clientslabel='Clients'): """Utility to assemble the default menu. Parameters: nwslabel - label override for ``listWss`` page """ yield 'doit?op=showServerInfo', 'Server Info' yield 'doit?op=listClients', clientslabel, yield 'doit?op=listWss', nwslabel, def menu_provider_ws(wsname, varslabel='Variables'): """Utility to assemble the menu for workspace-specific pages. Parameters: wsname - name of the current workspace nwslabel - label override for ``listVars`` page """ for item in menu_provider_default(): yield item wsname = quote_plus(wsname) yield ('doit?op=listVars&wsName=%s' % wsname), varslabel def menu_provider_var(wsname, varname): """Utility to assemble the menu for variable-specific pages. Parameters: wsname - name of the current workspace varname - name of the current variable """ for item in menu_provider_ws(wsname): yield item wsname = quote_plus(wsname) varname = quote_plus(varname) url = ('doit?op=showVar&wsName=%s&varName=%s' % (wsname, varname)) yield url, 'Values' def menu_provider_refresh(url, base): """Utility to add a 'Refresh' item to the menu bar. Parameters: url - URL for the 'Refresh' item base - base menu provider """ for item in base: yield item yield url, 'Refresh' def make_menu(provider): """Generate an appropriate 'menu' for a page. Parameters: provider - iterable yielding URL/label pairs for menu """ prefix = '
    \n' content = ['
  • %s
    • ' % x for x in provider] suffix = '\n
\n' return prefix + '\n'.join(content) + suffix def _make_refresh(page, delay=3): """Generate an appropriate tag. If page is None, the empty string will be returned. Parameters: page - page to redirect to delay - delay in seconds (default: 3) """ if page is None: return '' return _REFRESH % (delay, page) def make_header(title, menu=None, refresh_url=None): """Build a page header from the given parameters. Parameters: title - title for page menu - content for menu refresh_url - META refresh URL, if any """ if menu is None: menu = menu_provider_default() fields = { 'title': title, 'style': _STYLE_BLOCK, 'menu': make_menu(menu), 'refresh': _make_refresh(refresh_url), } return _HEADER_TEMPLATE % fields def make_footer(title): """Build a page footer from the given parameters. Parameters: title - title for page """ fields = { 'title': title, } return _FOOTER_TEMPLATE % fields def make_page(title, content, refresh_url=None, menu=None): """Build a complete page from the given parameters. Parameters: title - title for page content - content for page (HTML) refresh_url - META refresh URL, if any menu - content for menu """ fields = { 'header': make_header(title, menu, refresh_url), 'footer': make_footer(title), 'title': title, 'content': content, } return _PAGE_TEMPLATE % fields def make_errpage(msg, title='Error', **kwargs): """Create a stock error page with a given message and title. Parameters: msg - error message (HTML) title - title (HTML) kwargs - other arguments, as per ``make_page`` """ error_div = make_div(msg, 'error') return make_page(title, error_div, **kwargs) def make_infopage(msg, title='Info', **kwargs): """Create a stock info page with a given message and title. Parameters: msg - message (HTML) title - title (HTML) kwargs - other arguments, as per ``make_page`` """ info_div = make_div(msg, 'info') return make_page(title, info_div, **kwargs) nwsserver-2.0.0/nwss/mock.py0000644000175000017500000001036411261677154015502 0ustar westonweston# # Copyright (c) 2005-2009, REvolution Computing, Inc. # # NetWorkSpaces is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License as published # by the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 # USA # """Mock objects, used both for testing and for the internal plumbing of the Babelfish and monitors. """ # these dummy classes are used by the objects that want to pretend # they are clients of the nws server. from twisted.python import log from nwss.protoutils import WsTracker, WsNameMap from nwss.base import DIRECT_STRING, ERROR_VALUE from nwss.base import Response, Value import nwss _DEBUG = nwss.config.is_debug_enabled('NWS:mock') __all__ = ['MockTransport', 'MockConnection', 'DummyConnection'] class MockTransport(object): #pylint: disable-msg=R0903 """Mock 'transport' object, mimicking the various transport objects from Twisted. Presently, only used indirectly (i.e. as part of a DummyConnection). """ def __init__(self): self.sessionno = -1 def write(self, data): #pylint: disable-msg=R0201 """Override of Twisted 'write' method.""" if _DEBUG: log.msg("MockTransport.write: data = %s" % repr(data)) class MockConnection(object): #pylint: disable-msg=R0903 """Mock 'connection' object, mimicking the interface of the NwsProtocol object in a fairly minimal way. """ transport_factory = MockTransport def __init__(self, peer_id='[Mock]'): """Initialize a new mock connection: Parameters: peer_id - peer id to report in monitoring """ self.owned_workspaces = WsTracker() self.workspace_names = WsNameMap() self.peer = peer_id self.transport = self.transport_factory() def set_blocking_var(self, var, blocklist): """Set the currently "blocking" variable.""" pass def send_short_response(self, response=None): #pylint: disable-msg=R0201 """Implementation of NwsProtocol 'send_short_response' interface.""" pass def send_long_response(self, response): #pylint: disable-msg=R0201 """Implementation of NwsProtocol 'send_long_response' interface.""" if response is None: response = Response(value=ERROR_VALUE) assert response.value is not None def send_error(self, reason, status=1, long_reply=False): """Send an error message to the other side.""" pass class DummyConnection(MockConnection): #pylint: disable-msg=R0903 """Simple dummy 'connection' object, used throughout the web interface.""" def __init__(self, send_reply=None, peer_id='[Web Interface]'): """Initialize a new dummy connection: Parameters: send_reply - function to receive replies peer_id - peer id to report in monitoring """ MockConnection.__init__(self, peer_id) if send_reply: self.real_send_reply = send_reply def __str__(self): return 'DummyConnection[%s]' % self.peer def send_long_response(self, response): """Implementation of NwsProtocol 'send_long_response' interface.""" if response is None: response = Response(value=ERROR_VALUE) assert response.value is not None if hasattr(self, 'real_send_reply'): status = response.status metadata = response.metadata value = response.value if isinstance(value, str): value = Value(DIRECT_STRING, value) return self.real_send_reply(status, metadata, value) else: return None nwsserver-2.0.0/nwss/base.py0000644000175000017500000001125211214274547015455 0ustar westonweston# # Copyright (c) 2005-2009, REvolution Computing, Inc. # # NetWorkSpaces is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License as published # by the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 # USA # """ Core NetWorkSpaces server - common bits. """ import os, mmap, traceback from twisted.python import log # bit codings for the descriptor. DIRECT_STRING = 1 class ServerException(Exception): """Base class for all exceptions raised by this module.""" def __str__(self): #pylint: disable-msg=W0141 return '%s[%s]' % (self.__class__.__name__, ' '.join(map(str, self.args))) def __repr__(self): #pylint: disable-msg=W0141 return '%s(%s)' % (self.__class__.__name__, ', '.join(map(repr, self.args))) class WorkspaceFailure(ServerException): """Exception thrown by Workspace to signal that a normal (i.e. user-level) error has occurred during the operation. """ def __init__(self, msg, status=1): ServerException.__init__(self, msg) self.status = status class BadModeException(ServerException): """Variable cannot be set to specified mode.""" class NoSuchVariableException(ServerException): """No variable by specified name.""" class Response(object): #pylint: disable-msg=R0903 """Response from the server to the client.""" def __init__(self, metadata=None, value=None): self.status = 0 if metadata is None: metadata = {} self.metadata = metadata self.value = value self.iterstate = None class Value(object): """Value wrapper class handling out-of-band transmission of long data.""" def __init__(self, desc, val): """Initialize a value object. Arguments: desc - type descriptor for object val - either a string or a (filename, length) tuple """ self.__type_descriptor = desc self._val = val self._consumed = False if isinstance(val, str): self._long = False self._length = len(val) else: # if it's not a string, assume it's a tuple: (filename, length) self._long = True self._length = val[1] def consumed(self): """Flag this value as consumed. """ self._consumed = True def access_complete(self): """Notify this value that it has been sent to the client, and should now deallocate its file if it has been consumed. """ if self._consumed: self.close() def close(self): """Deallocate any resources associated with this value.""" if self._long: try: os.remove(self._val[0]) except OSError: log.msg('error removing file %s' % self._val[0]) traceback.print_exc() def get_file(self): """Memory map the file associated with this long value. If this is not a long value, this method will fail. """ assert self._long, 'get_file illegally called on string value' datafile = open(self._val[0], 'rb') memory = mmap.mmap(datafile.fileno(), self._length, access=mmap.ACCESS_READ) datafile.close() return memory def is_large(self): """Is this a large value?""" return self._long def __get_type_descriptor(self): """Get the type descriptor for this value.""" return self.__type_descriptor type_descriptor = property(__get_type_descriptor) def val(self): """Get the raw value for this non-long value. If this is a long value, this method will fail.""" assert not self._long, 'val illegally called on long value' return self._val def set_val(self, data): """Set the raw value for this non-long value. If this is a long value, this method will fail.""" assert not self._long, 'val illegally called on long value' self._val = data def length(self): """Get the length of this value in bytes.""" return self._length ERROR_VALUE = Value(0, '') nwsserver-2.0.0/INSTALL0000644000175000017500000000650211175420457014231 0ustar westonwestonRequirements ------------ Python 2.2 or later. Twisted 2.1 and Twisted-Web 0.5 or later. Twisted is available from: http://www.twistedmatrix.com/ Twisted itself requires: Zope Interfaces 3.0.1 (http://zope.org/Products/ZopeInterface) If you download the TwistedSumo tarball from the Twisted web site, it includes the core distribution, Twisted-Web, and Zope Interfaces, as well as many other Twisted projects that aren't used by NetWorkSpaces. Unix Installation (including Linux and Mac OS/X) ------------------------------------------------ Once you've installed Twisted, you're ready to install NetWorkSpaces. First, untar and build the distribution: % tar xzvf nwsserver-1.X.tar.gz % python setup.py build You can do this as an unprivileged user, even if you plan to install it as root. Note that on SuSE Linux systems (and perhaps other RPM-based systems) you may to install the "python-devel" package in order to install python source distributions. On Debian systems, you need the "python-dev" package. System Installation ------------------- To do a system wide installation, su to root, and execute the following command: # python setup.py install This will add the nws python package to your python installation, and copy the nws.tac file to /etc. nws.tac is a "Twisted Application Configuration" (TAC) file, which is actually python source. It is used by the twistd command to start the NetWorkSpaces server. The distribution includes a Unix-style init script, called "nws" that is in the misc directory. It is not installed automatically, as systems differ on how that needs to be done. Private Installation -------------------- To do a private installation, say into your home directory, you can execute the following command: % python setup.py install --home $HOME This will copy the nws python package to the directory: $HOME/lib/python You may want to put this directory into your PYTHONPATH environment variable so that python will be able to find it when starting the server. The nws.tac file is copied to: $HOME/nws.tac Configuration ------------- There is very little that can be configured in the NetWorkSpaces server, and there may be nothing that needs to be configured. Unless you want to run multiple servers on one machine, you can probably ignore configuration, and skip to the next section now. To run multiple servers on one machine, you'll need to configure the server and web interface ports to be different. This can be done by editing the TAC file, nws.tac, which is python code. An alternative is to set some environment variables. The following environment variables are used: NWS_SERVER_PORT Port that nws server binds to. The default value is 8765. NWS_WEB_PORT Port that web interface binds to. The default value is 8766. NWS_INTERFACE The name of the network interface that the server should bind to. The default is '', which means to bind to all network interfaces. The value 'localhost' will prevent remote machines from connecting to the server. nwsserver-2.0.0/README0000644000175000017500000001115711175420457014062 0ustar westonwestonNetWorkSpaces Server -------------------- NetWorkSpaces (NWS) is a powerful, open-source software package that makes it easy to use clusters from within scripting languages like Python, R, and Matlab. It uses a Space-based approach, similar to JavaSpaces (TM) for example, that makes it easier to write distributed applications. Installation ------------ NetWorkSpaces Server requires that you have: Python version 2.2 or later. Twisted 2.1 and Twisted-Web 0.5 or later. It is distributed as a Python source distribution, using the standard Python distribution utilities. The full installation instructions are in the INSTALL file that is included in the source distribution, but here's a quick summary of the "System Installation": % tar xzf nwsserver-1.6.X.tar.gz % cd nwsserver-1.6.X % python setup.py install You'll probably need to use a privileged account to do this. If you don't want to (or can't) do that, you can use some options to install into a different location. To get help on the install options, execute the command: % python setup.py install --help Starting the NetWorkSpaces server (System Installation) ------------------------------------------------------- To start the NetWorkSpaces server, you can execute it using the twistd command, as follows: % twistd -ny /etc/nws.tac > nws.log 2>&1 & or, if you're using a csh compatible shell: % twistd -ny /etc/nws.tac >& nws.log & which runs as the current user. See the twistd man page for more information. To use different ports (as discussed in the Configuration section of the INSTALL file), just set the appropriate environment variables before executing twistd: % NWS_SERVER_PORT=9765; export NWS_SERVER_PORT % NWS_WEB_PORT=9766; export NWS_WEB_PORT or, if you're using a csh compatible shell: % setenv NWS_SERVER_PORT 9765 % setenv NWS_WEB_PORT 9766 To shutdown the NetWorkSpaces server, you have to kill it by sending a SIGTERM signal at it. The twistd command creates a file that contains the process id of the server, so you can shut it down with the following command: % kill `cat twistd.pid` You could also use the nwsserver command. Just copy it somewhere in your PATH, and execute the following command: % nwsserver start If you start the NetWorkSpaces server using the nwsserver command, you can shut it down using the command: % nwsserver stop The nwsserver command can be set up for use as an init script on some systems, but that is not documented at this time. Starting the NetWorkSpaces server (Private Installation) -------------------------------------------------------- To start the NetWorkSpaces server with a private installation, you can execute it using the twistd command, as follows: % twistd -ny $HOME/etc/nws.tac > nws.log 2>&1 & or, if you're using a csh compatible shell: % twistd -ny $HOME/etc/nws.tac >& nws.log & Shutting down is the same as for a system installation: % kill `cat twistd.pid` You can also use the nwsserver command, which you could copy to a directory in your PATH, such as $HOME/bin, and execute as follows: % nwsserver start and shut it down with: % nwsserver stop Web Interface ------------- The NetWorkSpaces server includes a builtin web interface that allows you to browser the workspaces. To view from your local machine, open the following URL with a web browser: http://localhost:8766/ or use the appropriate host name if coming from a different machine. Babelfish --------- In order to examine the values of a workspace using the server's web interface, you'll usually need to have a babelfish running. The babelfish translates the values into a human readable format. If a value is a string, then the web interface simply displays the contents of the string, without any help from a babelfish. But if the value is any other type of object, it needs the help of a babelfish. For this reason, the babelfish are packaged with the client distributions, and are not included with the NetWorkSpaces server. But since the babelfish relate to the web interface (which is part of the server), it's something that you should be aware of when setting up the server. Using NetWorkSpaces ------------------- The server alone doesn't really do anything: you need to install one of the client packages to actually do something useful with NetWorkSpaces. There are currently client packages available for python, R, and Matlab. See http://www.lindaspaces.com/ for more information. nwsserver-2.0.0/setup.py0000644000175000017500000000627511175420457014721 0ustar westonweston#!/usr/bin/env python # # Copyright (c) 2005-2008, REvolution Computing, Inc. # # NetWorkSpaces is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License as published # by the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 # USA # """ Distutils installer for NetWorkSpaces """ import os, sys if sys.version_info < (2, 2): print >> sys.stderr, "You must use at least Python 2.2 for NetWorkSpaces" sys.exit(1) if os.environ.get('NWS_TAC_DIR'): tacdir = os.environ['NWS_TAC_DIR'] elif hasattr(os, 'getuid') and os.getuid() == 0: tacdir = '/etc' else: tacdir = '' if os.environ.get('NWS_DOC_DIR'): docdir = os.environ['NWS_DOC_DIR'] elif hasattr(os, 'getuid') and os.getuid() == 0: docdir = '/usr/share/doc/nws-server' else: docdir = 'nws-server' top_srcdir = os.environ.get('NWS_TOP_SRCDIR', '.') doc_files = [os.path.join(top_srcdir, x) for x in ['README']] scripts = [] if os.environ.get('NWS_WINDOWS', 'no') == 'yes': scripts += ['misc/NwsService.py'] from distutils import core kw = { 'name': 'nwsserver', 'version': '2.0.0', 'author': 'REvolution Computing, Inc.', 'author_email': 'sbweston@users.sourceforge.net', 'url': 'http://nws-py.sourceforge.net/', 'license': 'GPL version 2 or later', 'description': 'Python NetWorkSpaces Server', 'packages': ['nwss'], 'scripts': scripts, 'data_files': [ (tacdir, ['misc/nws.tac']), (docdir, doc_files), ], 'platforms': ['any'], 'long_description': """\ NetWorkSpaces (NWS) is a system that makes it very easy for different scripts and programs running (potentially) on different machines to communicate and coordinate with one another. The requirements for the NWS server are: Python 2.2 or later on Linux, Mac OS X, and other Unix systems. Python 2.4 or later on Windows. Twisted 2.1 and Twisted-Web 0.5 or later. Twisted is available from: http://www.twistedmatrix.com/ Twisted itself requires: Zope Interfaces 3.0.1 (http://zope.org/Products/ZopeInterface)""", } if (hasattr(core, 'setup_keywords') and 'classifiers' in core.setup_keywords): kw['classifiers'] = [ 'Topic :: System :: Clustering', 'Topic :: System :: Distributed Computing', 'License :: OSI Approved :: GNU General Public License (GPL)', 'Development Status :: 5 - Production/Stable', 'Environment :: Console', 'Intended Audience :: Developers', 'Intended Audience :: Science/Research', 'Intended Audience :: System Administrators', 'Operating System :: OS Independent', 'Programming Language :: Python', 'Natural Language :: English', ] core.setup(**kw) nwsserver-2.0.0/misc/0000755000175000017500000000000011261701137014121 5ustar westonwestonnwsserver-2.0.0/misc/nwsCfg.tac0000644000175000017500000000644611261677130016060 0ustar westonweston# # Copyright (c) 2005-2008, REvolution Computing, Inc. # # NetWorkSpaces is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License as published # by the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 # USA # import os from twisted.application import internet, service from twisted.python import log try: from twisted.web import server except ImportError: server = None log.msg('WARNING: unable to import twisted.web') from nwss.server import NwsService, NwsWeb import nwss nwss.config.load_from_file("nws.cfg") ######## Start of configuration section ######## # # Set interface to 'localhost' to disallow connections from # other machines. If you're using an SMP, it might be a good idea. interface = os.environ.get('NWS_INTERFACE', '') # Port for the NWS server to listen on try: nwss.config.nwsServerPort = int(os.environ['NWS_SERVER_PORT']) except: pass # Temporary directory for the NWS server try: nwss.config.nwsTmpDir = os.environ['NWS_TMP_DIR'] except: pass # Port for the web interface to listen on try: nwss.config.nwsWebPort = int(os.environ['NWS_WEB_PORT']) except: pass # Directory for the web interface to serve files from try: nwss.config.nwsWebServedDir = os.environ['NWS_WEB_SERVED_DIR'] except: pass # Size at which we consider a value to be too big for memory try: nwss.config.nwsLongValueSize = int(os.environ['NWS_LONG_VALUE_SIZE']) except: pass # SSL Certificate to use try: nwss.config.nwsServerSslCert = os.environ['NWS_SERVER_SSL_CERT'] except: pass try: nwss.config.nwsServerSslKey = os.environ['NWS_SERVER_SSL_KEY'] except: if nwss.config.nwsServerSslCert is not None and nwss.config.nwsServerSslCert.count('.') > 0: nwss.config.nwsServerSslKey = nwss.config.nwsServerSslCert[0:nwss.config.nwsServerSslCert.rindex('.')] + ".key" # Plugin directories to use try: nwss.config.nwsPluginDirs = os.environ['NWS_SERVER_PLUGIN_PATH'].split(':') except: pass # ######## End of configuration section ######## # Create the NWS service nwssvc = NwsService() nwssvr = internet.TCPServer(nwss.config.nwsServerPort, nwssvc, interface=interface) # Create the web interface service if the twisted.web module is installed if server: websvr = internet.TCPServer(nwss.config.nwsWebPort, server.Site(NwsWeb(nwssvc)), interface=interface) nwssvc.nwsWebPort = lambda: websvr._port.getHost().port if not os.environ.get('NWS_NO_SETUID') and hasattr(os, 'getuid') and os.getuid() == 0: # we're root, so become user 'daemon' application = service.Application('nwss', uid=1, gid=1) else: application = service.Application('nwss') nwssvr.setServiceParent(application) if server: websvr.setServiceParent(application) nwsserver-2.0.0/misc/nws.tac0000644000175000017500000000647511255715377015453 0ustar westonweston# # Copyright (c) 2005-2008, REvolution Computing, Inc. # # NetWorkSpaces is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License as published # by the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 # USA # import os from twisted.application import internet, service from twisted.python import log try: from twisted.web import server except ImportError: server = None log.msg('WARNING: unable to import twisted.web') from nwss.server import NwsService, NwsWeb import nwss ######## Start of configuration section ######## # # Set interface to 'localhost' to disallow connections from # other machines. If you're using an SMP, it might be a good idea. interface = os.environ.get('NWS_INTERFACE', '') # Port for the NWS server to listen on try: nwss.config.nwsServerPort = int(os.environ['NWS_SERVER_PORT']) except: pass # Temporary directory for the NWS server try: nwss.config.nwsTmpDir = os.environ['NWS_TMP_DIR'] except: pass # Port for the web interface to listen on try: nwss.config.nwsWebPort = int(os.environ['NWS_WEB_PORT']) except: pass # Directory for the web interface to serve files from try: nwss.config.nwsWebServedDir = os.environ['NWS_WEB_SERVED_DIR'] except: pass # Size at which we consider a value to be too big for memory try: nwss.config.nwsLongValueSize = int(os.environ['NWS_LONG_VALUE_SIZE']) except: pass # SSL Certificate to use try: nwss.config.nwsServerSslCert = os.environ['NWS_SERVER_SSL_CERT'] except: pass try: nwss.config.nwsServerSslKey = os.environ['NWS_SERVER_SSL_KEY'] except: if nwss.config.nwsServerSslCert is not None and nwss.config.nwsServerSslCert.count('.') > 0: nwss.config.nwsServerSslKey = nwss.config.nwsServerSslCert[0:nwss.config.nwsServerSslCert.rindex('.')] + ".key" # Plugin directories to use try: nwss.config.nwsPluginDirs = os.environ['NWS_SERVER_PLUGIN_PATH'].split(':') except: nwss.config.nwsPluginDirs = [os.path.join(os.getcwd(), 'plugins')] # ######## End of configuration section ######## # Create the NWS service nwssvc = NwsService() nwssvr = internet.TCPServer(nwss.config.nwsServerPort, nwssvc, interface=interface) # Create the web interface service if the twisted.web module is installed if server: websvr = internet.TCPServer(nwss.config.nwsWebPort, server.Site(NwsWeb(nwssvc)), interface=interface) nwssvc.nwsWebPort = lambda: websvr._port.getHost().port if not os.environ.get('NWS_NO_SETUID') and hasattr(os, 'getuid') and os.getuid() == 0: # we're root, so become user 'daemon' application = service.Application('nwss', uid=1, gid=1) else: application = service.Application('nwss') nwssvr.setServiceParent(application) if server: websvr.setServiceParent(application) nwsserver-2.0.0/misc/NwsService.py0000644000175000017500000001474311214274547016604 0ustar westonweston# # Copyright (c) 2005-2009, REvolution Computing, Inc. # # NetWorkSpaces is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License as published # by the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 # USA # import sys, os, inspect import win32serviceutil, win32service, win32event, win32process, win32api from nwss.util import msc_argv2str # XXX: should be a better way than this: maybe the registry? # try to find the Python installation directory _PYTHONDIR = os.path.dirname(inspect.getfile(win32api)) while not os.path.isfile(os.path.join(_PYTHONDIR, 'python.exe')) and \ os.path.basename(_PYTHONDIR): _PYTHONDIR = os.path.dirname(_PYTHONDIR) _INTERP = _PYTHONDIR + '\\python.exe' # XXX: should this use pythonw.exe? _SCRIPT = _PYTHONDIR + '\\scripts\\twistd.py' _TACFILE = _PYTHONDIR + '\\nws.tac' _LOGFILE = _PYTHONDIR + '\\nws.log' # nws server states _NWS_SERVER_RUNNING = 100 _NWS_SERVER_DIED = 101 _NWS_SERVER_RESTARTED = 102 # timeouts _DIED_TIMEOUT = 10 * 1000 # milliseconds to wait before restarting # a dead nws server _RESTARTED_TIMEOUT = 10 * 1000 # milliseconds to wait before considering a # restarted nws server to be running class NwsService(win32serviceutil.ServiceFramework): _svc_name_ = 'NwsService' _svc_display_name_ = 'NetWorkSpaces Service' _svc_description_ = 'Enables multiple independent applications written in ' \ 'scripting languages to share data and coordinate their computations. ' \ 'Client support currently exists for R, Python, and Matlab.' def __init__(self, args): win32serviceutil.ServiceFramework.__init__(self, args) self.hWaitStop = win32event.CreateEvent(None, 0, 0, None) self.applicationName = _INTERP argv = [self.applicationName, _SCRIPT, '-o', '-l', _LOGFILE, '-y', _TACFILE] self.commandLine = msc_argv2str(argv) def SvcStop(self): self.ReportServiceStatus(win32service.SERVICE_STOP_PENDING) win32event.SetEvent(self.hWaitStop) def SvcDoRun(self): import servicemanager self.sm = servicemanager self.sm.LogMsg(self.sm.EVENTLOG_INFORMATION_TYPE, self.sm.PYS_SERVICE_STARTED, (self._svc_name_, '')) self.procHandle = self._startNws() handles = (self.hWaitStop, self.procHandle) timeout = win32event.INFINITE state = _NWS_SERVER_RUNNING while True: # wait for a stop request, a nws server death, or possibly a timeout s = win32event.WaitForMultipleObjects(handles, 0, timeout) if s == win32event.WAIT_TIMEOUT: if state == _NWS_SERVER_RESTARTED: self._info("nws server restarted successfully") timeout = win32event.INFINITE state = _NWS_SERVER_RUNNING elif state == _NWS_SERVER_DIED: self.procHandle = self._startNws() handles = (self.hWaitStop, self.procHandle) timeout = _RESTARTED_TIMEOUT state = _NWS_SERVER_RESTARTED else: self._error("got an unexpected timeout while in state %d" % state) break elif s == win32event.WAIT_OBJECT_0: # a shutdown was requested, so kill the nws server # and break out of the while loop if self.procHandle: self._info("shutdown requested: terminating nws server") try: win32process.TerminateProcess(self.procHandle, 0) except: e = sys.exc_info()[1] self._info("caught exception terminating nws server: %s" % str(e)) else: self._info("shutdown requested while no nws server running") break elif s == win32event.WAIT_OBJECT_0 + 1: # the nws server exited by itself, which probably means # that the NWS server shutdown. we want to reconnect # when it comes back up, so sleep awhile, and then # start another nws server. this will probably happen # over and over again, so don't do it too frequently. if state == _NWS_SERVER_RUNNING: self._info("nws server died: restarting in a bit") win32api.CloseHandle(self.procHandle) self.procHandle = None handles = (self.hWaitStop,) timeout = _DIED_TIMEOUT state = _NWS_SERVER_DIED else: self._error("illegal status from WaitForMultipleObjects: stopping") break self.sm.LogMsg(self.sm.EVENTLOG_INFORMATION_TYPE, self.sm.PYS_SERVICE_STOPPED, (self._svc_name_, '')) def _info(self, msg): self.sm.LogMsg(self.sm.EVENTLOG_INFORMATION_TYPE, 1, (msg,)) def _error(self, msg): self.sm.LogMsg(self.sm.EVENTLOG_ERROR_TYPE, 1, (msg,)) def _startNws(self): processSecurityAttributes = None threadSecurityAttributes = None fInheritHandles = 0 creationFlags = win32process.CREATE_NO_WINDOW environment = None currentDirectory = None startupInfo = win32process.STARTUPINFO() procHandle, threadHandle, procId, threadId = win32process.CreateProcess( self.applicationName, self.commandLine, processSecurityAttributes, threadSecurityAttributes, fInheritHandles, creationFlags, environment, currentDirectory, startupInfo) win32api.CloseHandle(threadHandle) return procHandle if __name__ == '__main__': win32serviceutil.HandleCommandLine(NwsService) nwsserver-2.0.0/misc/com.revolution-computing.nwsserver.plist0000644000175000017500000000414111175420457024230 0ustar westonweston Label com.revolution-computing.nwsserver ProgramArguments /usr/bin/python /usr/local/bin/twistd --syslog --prefix nwsserver --pidfile nwsserver_8765.pid --nodaemon --python /etc/nws.tac RunAtLoad EnvironmentVariables PYTHONPATH /Library/Python/2.5/site-packages NWS_NO_SETUID true NWS_SERVER_PORT 8765 NWS_WEB_PORT 8766 NWS_INTERFACE NWS_TMP_DIR /private/tmp UserName nobody GroupName nogroup WorkingDirectory /private/tmp nwsserver-2.0.0/misc/nwsserver0000755000175000017500000000575711175420457016132 0ustar westonweston#!/bin/sh # # Copyright (c) 2005-2008, REvolution Computing, Inc. # # NetWorkSpaces is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License as published # by the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 # USA # ### BEGIN INIT INFO # Provides: nwsserver # Required-Start: $local_fs $remote_fs $network $named # Required-Stop: $local_fs $remote_fs # Default-Start: 2 3 4 5 # Default-Stop: 0 1 6 # Short-Description: NWS server, providing access to shared workspaces. # Description: The NWS server enables multiple independent # applications written in scripting languages to share data and # coordinate their computations. Client support currently exists # for R, Python, and Matlab. ### END INIT INFO ### CONFIGURATION SECTION ### NWS_BASE="" NWS_INSTALL="" if [ "`id -u`" = "0" ] then NWS_PID="$NWS_BASE/var/run/nws.pid" NWS_LOG="$NWS_BASE/var/log/nws.log" else NWS_PID=$HOME/nws.pid NWS_LOG=$HOME/nws.log fi NWS_TAC="$NWS_INSTALL/etc/nws.tac" TWISTD=/usr/bin/twistd WORKDIR=/tmp # PYTHONPATH=${PYTHONPATH:+$PYTHONPATH:}$NWS_INSTALL/lib/python # export PYTHONPATH ### END OF CONFIGURATION SECTION ### if [ ! -e $NWS_TAC ] then NWS_TAC=/etc/nws.tac fi if [ ! -e $NWS_TAC ] then NWS_TAC=$HOME/nws.tac fi if [ ! -e $NWS_TAC ] then echo "NWS .tac file $NWS_TAC not found!" >&2 exit 1 fi case $# in 0) echo "usage: $0 [start|stop]" 1>&2 exit 1 ;; esac case "$1" in start) # Let twistd handle existing pid files /bin/rm -f $NWS_LOG echo -n "Starting the NWS server" if $TWISTD -o -d $WORKDIR -l $NWS_LOG --pidfile $NWS_PID -y $NWS_TAC then echo "." else echo " (failed)." exit 1 fi ;; stop) if [ ! -e $NWS_PID ] then echo "$NWS_PID file doesn't exist: is the server really running?" 1>&2 exit 1 fi echo -n "Stopping the NWS server" kill `cat $NWS_PID` 2> /dev/null # XXX I think we might need to do this due to priviledge shedding issues /bin/rm -f $NWS_PID echo "." ;; force-reload|restart) echo -n "Restarting the NWS server" kill `cat $NWS_PID` 2> /dev/null sleep 3 /bin/rm -f $NWS_PID /bin/rm -f $NWS_LOG echo -n "Starting the NWS server" if $TWISTD -o -d $WORKDIR -l $NWS_LOG --pidfile $NWS_PID -y $NWS_TAC then echo "." else echo " (failed)." exit 1 fi ;; *) echo "usage: $0 [start|stop]" 1>&2 exit 1 ;; esac exit 0