pushy-0.5.3/0000755000175000017500000000000012204421005013327 5ustar jamespagejamespagepushy-0.5.3/setup.py0000664000175000017500000000563112173703716015071 0ustar jamespagejamespage# Copyright (c) 2008, 2011 Andrew Wilkins # # Permission is hereby granted, free of charge, to any person # obtaining a copy of this software and associated documentation # files (the "Software"), to deal in the Software without # restriction, including without limitation the rights to use, # copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the # Software is furnished to do so, subject to the following # conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES # OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT # HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. from distutils.core import setup long_description = """ Pushy is a Python library for allowing one to connect to a remote Python interpreter, and transparently access objects in that interpreter as if they were objects within the local interpreter. Pushy has the novel ability to execute a remote Python interpreter and start a request servicing loop therein, without requiring any custom Python libraries (including Pushy) to be initially present. To accomplish this, Pushy requires an SSH daemon to be present on the remote machine. Pushy was initially developed to simplify automated testing to the point that testing software on remote machines becomes little different to testing on the local machine. """.strip() description = """ A library for transparently accessing objects in a remote Python interpreter """.strip() classifiers = [ "Intended Audience :: Developers", "Operating System :: Microsoft :: Windows", "Operating System :: POSIX", "Programming Language :: Python", "Programming Language :: Java", "License :: OSI Approved :: MIT License", "Development Status :: 4 - Beta", "Topic :: Software Development :: Libraries", "Topic :: Software Development :: Object Brokering", "Topic :: Software Development :: Testing", "Topic :: System :: Distributed Computing", ] dependencies = [ "paramiko", ] setup( name="pushy", version="0.5.3", description = description, long_description = long_description, classifiers = classifiers, requires = dependencies, author="Andrew Wilkins", author_email="axwalk@gmail.com", url="http://awilkins.id.au/pushy/", packages=["pushy", "pushy.protocol", "pushy.transport", "pushy.transport.smb", "pushy.util"] ) pushy-0.5.3/PKG-INFO0000664000175000017500000000304512173704370014446 0ustar jamespagejamespageMetadata-Version: 1.1 Name: pushy Version: 0.5.3 Summary: A library for transparently accessing objects in a remote Python interpreter Home-page: http://awilkins.id.au/pushy/ Author: Andrew Wilkins Author-email: axwalk@gmail.com License: UNKNOWN Description: Pushy is a Python library for allowing one to connect to a remote Python interpreter, and transparently access objects in that interpreter as if they were objects within the local interpreter. Pushy has the novel ability to execute a remote Python interpreter and start a request servicing loop therein, without requiring any custom Python libraries (including Pushy) to be initially present. To accomplish this, Pushy requires an SSH daemon to be present on the remote machine. Pushy was initially developed to simplify automated testing to the point that testing software on remote machines becomes little different to testing on the local machine. Platform: UNKNOWN Classifier: Intended Audience :: Developers Classifier: Operating System :: Microsoft :: Windows Classifier: Operating System :: POSIX Classifier: Programming Language :: Python Classifier: Programming Language :: Java Classifier: License :: OSI Approved :: MIT License Classifier: Development Status :: 4 - Beta Classifier: Topic :: Software Development :: Libraries Classifier: Topic :: Software Development :: Object Brokering Classifier: Topic :: Software Development :: Testing Classifier: Topic :: System :: Distributed Computing Requires: paramiko pushy-0.5.3/pushy/0000755000175000017500000000000012204421005014477 5ustar jamespagejamespagepushy-0.5.3/pushy/protocol/0000755000175000017500000000000012204421005016340 5ustar jamespagejamespagepushy-0.5.3/pushy/protocol/message.py0000664000175000017500000001244012172577702020365 0ustar jamespagejamespage# Copyright (c) 2008, 2011 Andrew Wilkins # # Permission is hereby granted, free of charge, to any person # obtaining a copy of this software and associated documentation # files (the "Software"), to deal in the Software without # restriction, including without limitation the rights to use, # copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the # Software is furnished to do so, subject to the following # conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES # OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT # HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. import os, struct, thread import marshal import pushy.util class MessageType(object): "A class for describing the type of a message." def __init__(self, code, name): self.code = code self.name = name def __repr__(self): return "MessageType(%d, '%s')" % (self.code, self.name) def __int__(self): return self.code def __str__(self): return self.name def __hash__(self): return self.code def __eq__(self, other): if type(other) is MessageType: return other.code == self.code elif type(other) is int: return other == self.code elif type(other) is str: return other == self.name return False def read(file, length): # Read message payload. data = "" if length: while len(data) < length: try: partial = file.read(length - len(data)) except Exception, e: raise IOError, e if partial == "": raise IOError, "End of file" data += partial return data class Message: PACKING_FORMAT = ">BqqI" PACKING_SIZE = struct.calcsize(PACKING_FORMAT) def __init__(self, type, payload, target=0, source=None): self.type = type self.payload = payload self.target = target if source is None: source = thread.get_ident() self.source = source def __eq__(self, other): if other.__class__ is not Message: return False return self.type == other.type and \ self.source == other.source and \ self.target == other.target and \ self.payload == other.payload def __repr__(self): return "Message(%r, %d->%d, %r [%d bytes])" % \ (self.type, self.source, self.target, self.payload, len(self.payload)) def pack(self): return struct.pack(self.PACKING_FORMAT, int(self.type), self.source, self.target, len(self.payload)) + self.payload @staticmethod def unpack(file): header = read(file, Message.PACKING_SIZE) (type, source, target, length) = \ struct.unpack(Message.PACKING_FORMAT, header) type = message_types[type] payload = read(file, length) return Message(type, payload, target, source) ############################################################################### # Create enumeration of message types ############################################################################### message_names = ( "evaluate", "response", "exception", "getattr", "setattr", "getstr", "getrepr", "delete", "as_tuple", # All object ops go at the end "op__call__", "op__lt__", "op__le__", "op__eq__", "op__ne__", "op__gt__", "op__ge__", "op__cmp__", "op__rcmp__", "op__hash__", "op__nonzero__", "op__unicode__", "op__len__", "op__getitem__", "op__setitem__", "op__delitem__", "op__iter__", "op__contains__", #"op__slots__", "op__get__", "op__set__", "op__delete__", "op__getslice__", "op__setslice__", "op__delslice__", "op__add__", "op__sub__", "op__mul__", "op__floordiv__", "op__mod__", "op__divmod__", "op__pow__", "op__lshift__", "op__rshift__", "op__and__", "op__xor__", "op__or__", "op__div__", "op__truediv__", "op__radd__", "op__rsub__", "op__rdiv__", "op__rtruediv__", "op__rfloordiv__", "op__rmod__", "op__rdivmod__", "op__rpow__", "op__rlshift__", "op__rrshift__", "op__rand__", "op__rxor__", "op__ror__", "op__iadd__", "op__isub__", "op__imul__", "op__idiv__", "op__itruediv__", "op__ifloordiv__", "op__imod__", "op__ipow__", "op__ilshift__", "op__irshift__", "op__iand__", "op__ixor__", "op__ior__", "op__neg__", "op__pos__", "op__abs__", "op__invert__", "op__complex__", "op__int__", "op__long__", "op__float__", "op__oct__", "op__hex__", "op__index__", "op__coerce__", "op__enter__", "op__exit__", ) message_types = [] for i,t in enumerate(message_names): m = MessageType(i, t) message_types.append(m) setattr(MessageType, t, m) message_types = tuple(message_types) pushy-0.5.3/pushy/protocol/connection.py0000664000175000017500000001204312172577702021077 0ustar jamespagejamespage# Copyright (c) 2009 Andrew Wilkins # # Permission is hereby granted, free of charge, to any person # obtaining a copy of this software and associated documentation # files (the "Software"), to deal in the Software without # restriction, including without limitation the rights to use, # copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the # Software is furnished to do so, subject to the following # conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES # OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT # HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. from pushy.protocol.baseconnection import BaseConnection import logging, marshal, os, struct, threading from pushy.protocol.message import Message, MessageType, message_types import pushy.util import platform class Connection(BaseConnection): def __init__(self, istream, ostream, initiator=True): BaseConnection.__init__(self, istream, ostream, initiator) # Add message handlers self.message_handlers.update({ MessageType.evaluate: self.__handle_evaluate, MessageType.getattr: self.__handle_getattr, MessageType.setattr: self.__handle_setattr, MessageType.getstr: self.__handle_getstr, MessageType.getrepr: self.__handle_getrepr, MessageType.as_tuple: self.__handle_as_tuple, MessageType.op__call__: self.__handle_call, }) for message_type in message_types: if message_type.name == "op__call__": continue if message_type.name.startswith("op__"): self.message_handlers[message_type] = self.__handle_operator def eval(self, expression, globals=None, locals=None): args = (expression, globals, locals) return self.send_request(MessageType.evaluate, args) def operator(self, type_, object, args, kwargs): if args is not None: args = tuple(args) if not args: args = None if kwargs is not None: kwargs = tuple(kwargs.items()) if not kwargs: kwargs = None return self.send_request(type_, (object, args, kwargs)) def getattr(self, object, name): return self.send_request(MessageType.getattr, (object, name)) def setattr(self, object, name, value): return self.send_request(MessageType.setattr, (object, name, value)) def getstr(self, object): return self.send_request(MessageType.getstr, object) def getrepr(self, object): return self.send_request(MessageType.getrepr, object) def __handle_getattr(self, type, args): (object, name) = args return getattr(object, name) def __handle_setattr(self, type, args): (object, name, value) = args return setattr(object, name, value) def __handle_getstr(self, type, object): return str(object) def __handle_getrepr(self, type, object): return repr(object) def __handle_evaluate(self, type_, args): (expression, globals, locals) = args return eval(expression, globals, locals) def __handle_call(self, type_, args_): (object, args, kwargs) = args_ # Copy the *args and **kwargs. In particular, the **kwargs dict # must be a real dict, because Python will do a PyDict_CheckExact # somewhere along the line. if args is None: args = [] else: args = list(args) if kwargs is None: kwargs = {} else: kwargs = dict(kwargs) result = object(*args, **kwargs) return result def __handle_operator(self, type, args_): object = args_[0] args = None kwargs = None if len(args_) > 1: args = args_[1] if len(args_) > 2: kwargs = args_[2] # Copy the *args and **kwargs. In particular, the **kwargs dict # must be a real dict, because Python will do a PyDict_CheckExact # somewhere along the line. if args is None: args = [] else: args = list(args) if kwargs is None: kwargs = {} else: kwargs = dict(kwargs) # TODO handle slot pointer methods specially? name = type.name[2:] method = getattr(object, name) return method(*args, **kwargs) def __handle_as_tuple(self, type_, args): (type_, args) = args return tuple(self.message_handlers[type_](type_, args)) pushy-0.5.3/pushy/protocol/proxy.py0000664000175000017500000002322312172577702020123 0ustar jamespagejamespage# Copyright (c) 2008, 2011 Andrew Wilkins # # Permission is hereby granted, free of charge, to any person # obtaining a copy of this software and associated documentation # files (the "Software"), to deal in the Software without # restriction, including without limitation the rights to use, # copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the # Software is furnished to do so, subject to the following # conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES # OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT # HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. from pushy.protocol.message import message_types, MessageType import pushy.util import exceptions, types class ProxyType(object): def __init__(self, code, name): self.code = code self.name = name def __repr__(self): return "ProxyType(%d, '%s')" % (self.code, self.name) def __str__(self): return self.name def __int__(self): return self.code def __hash__(self): return self.code def __eq__(self, other): if type(other) is ProxyType: return other.code == self.code if type(other) is int: return other == self.code elif type(other) is str: return other == self.name return False @staticmethod def get(obj): # First check if it's a class class_ = type(obj) if hasattr(class_, "pushy_proxy_type"): return class_.pushy_proxy_type if isinstance(obj, Exception): return ProxyType.exception if isinstance(obj, dict): return ProxyType.dictionary if isinstance(obj, list): return ProxyType.list if isinstance(obj, set): return ProxyType.list if isinstance(obj, types.ModuleType): return ProxyType.module if isinstance(obj, types.ClassType): return ProxyType.oldstyleclass return ProxyType.object @staticmethod def getargs(proxy_type, obj): if type(proxy_type) is int: proxy_type = proxy_types[proxy_type] if proxy_type is ProxyType.exception: module_ = obj.__class__.__module__ if module_ == "exceptions": return obj.__class__.__name__ return None @staticmethod def getoperators(obj): class_ = type(obj) if hasattr(class_, "pushy_operator_mask"): return class_.pushy_operator_mask else: mask = sum(( (1 << t.code) for t in message_types \ if t.name.startswith("op__") and hasattr(obj, t.name[2:]) )) return mask proxy_names = ( "oldstyleclass", "object", "exception", "dictionary", "list", "set", "module" ) proxy_types = [] for i,t in enumerate(proxy_names): p = ProxyType(i, t) proxy_types.append(p) setattr(ProxyType, t, p) ############################################################################### def create_instance(args, conn, on_proxy_init): class ClassObjectProxy: def __init__(self): on_proxy_init(self) def __call__(self, *args, **kwargs): message_type = pushy.protocol.message.MessageType.op__call__ return conn.operator(message_type, self, args, kwargs) def __getattr__(self, name): if name == "__call__": raise AttributeError, "__call__" return conn.getattr(self, name) return ClassObjectProxy def create_object(args, conn, on_proxy_init): class ObjectProxy(object): def __init__(self): on_proxy_init(self) object.__init__(self) def __getattribute__(self, name): return conn.getattr(self, name) return ObjectProxy def create_exception(args, conn, on_proxy_init): BaseException = Exception if args is not None: BaseException = getattr(exceptions, args) class ExceptionProxy(BaseException): def __init__(self): on_proxy_init(self) BaseException.__init__(self) def __getattribute__(self, name): return conn.getattr(self, name) return ExceptionProxy def create_dictionary(args, conn, on_proxy_init): overridden_methods = frozenset(("items", "keys", "update", "values")) class DictionaryProxy(dict): def __init__(self): on_proxy_init(self) def keys(self): return list(conn.as_tuple(conn.getattr(self, "keys"))) def items(self): return list(conn.as_tuple(conn.getattr(self, "items"))) def update(self, rhs): if type(rhs) is dict: conn.getattr(self, "update")(tuple(rhs.items())) elif isinstance(rhs, dict): conn.getattr(self, "update")(rhs) else: conn.getattr(self, "update")(tuple(map(tuple, rhs))) def values(self): return list(conn.as_tuple(conn.getattr(self, "values"))) def __eq__(self, rhs): if self is rhs: return True return self.items() == rhs.items() def __getattribute__(self, name): if name in overridden_methods: return object.__getattribute__(self, name) return conn.getattr(self, name) return DictionaryProxy def create_list(args, conn, on_proxy_init): class ListProxy(list): def __init__(self): on_proxy_init(self) list.__init__(self) def __eq__(self, rhs): if self is rhs: return True as_tuple = conn.eval("tuple")(self) rhs_tuple = tuple(rhs) return as_tuple == rhs_tuple def __getattribute__(self, name): return conn.getattr(self, name) return ListProxy def create_set(args, conn, on_proxy_init): class SetProxy(set): def __init__(self): on_proxy_init(self) set.__init__(self, args) def __getattribute__(self, name): return conn.getattr(self, name) return SetProxy def create_module(args, conn, on_proxy_init): class ModuleProxy(types.ModuleType): def __init__(self): on_proxy_init(self) types.ModuleType.__init__(self, "") self.__importer = None def __getattribute__(self, name): try: # __dict__ on a module is expected to return a dict, not a # descendant thereof. Its items will be (in CPython) obtained # by ferretting around in the object's internals, rather than # going through the usual Python interface. result = conn.getattr(self, name) if isinstance(result, dict) and name == "__dict__": return dict(result.items()) return result except AttributeError: if self.__importer: return self.__importer("%s.%s" % (self.__name__, name)) else: raise return ModuleProxy class_factories = { ProxyType.oldstyleclass: create_instance, ProxyType.object: create_object, ProxyType.exception: create_exception, ProxyType.dictionary: create_dictionary, ProxyType.list: create_list, ProxyType.set: create_set, ProxyType.module: create_module } def Proxy(opmask, proxy_type, args, conn, on_proxy_init): """ Create a proxy object, which delegates attribute access and method invocation to an object in a remote Python interpreter. """ # Determine the class to use for the proxy type. ProxyClass = class_factories[proxy_type](args, conn, on_proxy_init) # Store the operator mask and proxy type in the class. This will be used by # ProxyType.getoperators() and ProxyType.get() # # XXX Important! Without this, the performance of tunnelled connections is # unusably slow due to the chatter induced by many calls to # hasattr/getattr. ProxyClass.pushy_operator_mask = opmask ProxyClass.pushy_proxy_type = proxy_type # Creates a lambda with the operator type and connection bound. def bound_operator(type_): return lambda self, *args, **kwargs: \ (conn.operator(type_, self, args, kwargs)) # Operators should be defined if and only if they're not implemented in the # local ProxyClass. def should_setattr(t): if not opmask & (1 << t.code): return False a = getattr(ProxyClass, t.name[2:], None) return a is None or not \ (type(a) is types.MethodType and a.im_class is ProxyClass) # Create proxy operators. types_ = (t for t in message_types if should_setattr(t)) map(lambda t: setattr(ProxyClass, t.name[2:], bound_operator(t)), types_) # Add other standard methods. setattr(ProxyClass, "__str__", lambda self: conn.getstr(self)) setattr(ProxyClass, "__repr__", lambda self: conn.getrepr(self)) setattr(ProxyClass, "__setattr__", lambda *args: conn.setattr(*args)) # Make sure we support the iteration protocol properly #if hasattr(ProxyClass, "__iter__"): setattr(ProxyClass, "next", lambda self: conn.getattr(self, "next")()) return ProxyClass() pushy-0.5.3/pushy/protocol/baseconnection.py0000664000175000017500000006476712172577702021756 0ustar jamespagejamespage# Copyright (c) 2008, 2011 Andrew Wilkins # # Permission is hereby granted, free of charge, to any person # obtaining a copy of this software and associated documentation # files (the "Software"), to deal in the Software without # restriction, including without limitation the rights to use, # copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the # Software is furnished to do so, subject to the following # conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES # OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT # HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. import logging import marshal import os import struct import sys import thread import threading import time import weakref from pushy.protocol.message import Message, MessageType, message_types from pushy.protocol.proxy import Proxy, ProxyType, proxy_types import pushy.util # This collection should contain only immutable types. Builtin, mutable types # such as list, set and dict need to be handled specially. marshallable_types = [ unicode, slice, frozenset, float, basestring, long, str, int, complex, bool, type(None) ] # The 'buffer' type doesn't exist in Jython. try: marshallable_types.append(buffer) except NameError: pass # Message types that may received in response to a request. response_types = ( MessageType.response, MessageType.exception ) marshallable_types = tuple(marshallable_types) # Marshalling constants. MARSHAL_TUPLE = 0 MARSHAL_ORIGIN = 1 MARSHAL_PROXY = 2 class LoggingFile: def __init__(self, stream, log): self.stream = stream self.log = log def close(self): self.stream.close() def write(self, s): self.log.write(s) self.stream.write(s) def flush(self): self.log.flush() self.stream.flush() def read(self, n): data = self.stream.read(n) self.log.write(data) self.log.flush() return data class MessageStream: def __init__(self, file_): self.__lock = threading.Lock() self.__file = file_ def close(self): self.__lock.acquire() try: self.__file.close() finally: self.__lock.release() def send_message(self, m): bytes_ = m.pack() self.__lock.acquire() try: self.__file.write(bytes_) self.__file.flush() finally: self.__lock.release() def receive_message(self): self.__lock.acquire() try: return Message.unpack(self.__file) finally: self.__lock.release() class ResponseHandler: def __init__(self, condition): self.condition = condition self.message = None self.thread = thread.get_ident() connection_count_lock = threading.Lock() connection_count = 0 def get_connection_id(): global connection_count connection_count_lock.acquire() try: connection_id = connection_count connection_count += 1 return connection_id finally: connection_count_lock.release() class BaseConnection(object): def __init__(self, istream, ostream, initiator=True): self.__open = True self.__istream = MessageStream(istream) self.__ostream = MessageStream(ostream) self.__initiator = initiator self.__marshal_lock = threading.Lock() self.__delete_lock = threading.RLock() self.__connid = get_connection_id() self.__last_delete = time.time() self.gc_enabled = True self.gc_interval = 5.0 # Every 5 seconds # Define message handlers (MessageType -> method) self.message_handlers = { MessageType.response: self.__handle_response, MessageType.exception: self.__handle_exception } # Attributes required to track responses. self.__thread_local = threading.local() self.__response_handlers = {} # Attributes required to track number of threads processing requests. # The following has to be true for the message receiving thread to be # allowed to attempt to receive a message: # - There are no threads currently processing a request, and # there are no requests pending. # OR # - There are threads currently processing requests, but they # are all waiting on responses. self.__receiving = False # Is someone calling self.__recv? self.__processing = 0 # How many requests are being processed. self.__waiting = 0 # How many responses are pending. self.__responses = 0 self.__requests = [] self.__processing_condition = threading.Condition(threading.Lock()) # Uncomment the following for debugging. if False: self.__istream = \ LoggingFile( istream, open("%d-%d.in"% (os.getpid(), self.__connid), "wb")) self.__ostream = \ LoggingFile( ostream, open("%d-%d.out" % (os.getpid(), self.__connid),"wb")) # (Client) Contains mapping of id(obj) -> proxy self.__proxies = {} # (Client) Contains mapping of id(proxy) -> (id(obj), version) self.__proxy_ids = {} # (Client) Contains a mapping of id(obj) -> version self.__pending_deletes = {} # (Server) Contains mapping of id(obj) -> (obj, proxy-info) self.__proxied_objects = {} def __del__(self): if self.__open: self.close() STATE_FORMAT = """ Connection State ------------------------ ID: %r Open: %r Receiving: %r Processing Count: %r Waiting Count: %r Response Count: %r Request Count: %r Thread Request Count: %r Peer Thread: %r Proxy Count: %r Proxied Object Count: %r """.strip() def __log_state(self): state_args = (self.__connid, self.__open, self.__receiving, self.__processing, self.__waiting, self.__responses, len(self.__requests), self.__thread_request_count, self.__peer_thread, len(self.__proxies), len(self.__proxied_objects)) pushy.util.logger.debug("\n"+self.STATE_FORMAT, *state_args) # Property for determining the number of requests the current thread is # processing. def __get_thread_request_count(self): return getattr(self.__thread_local, "request_count", 0) def __set_thread_request_count(self, value): self.__thread_local.request_count = value __thread_request_count = \ property(__get_thread_request_count, __set_thread_request_count) # Property for getting the current thread's peer thread. def __get_peer_thread(self): return getattr(self.__thread_local, "peer_thread", 0) def __set_peer_thread(self, value): self.__thread_local.peer_thread = value __peer_thread = property(__get_peer_thread, __set_peer_thread) def close(self): try: if not self.__open: return # Flag the connection as closed, and wake up all request # handlers. We'll then wait until there are no more # response handlers waiting. self.__open = False self.__processing_condition.acquire() try: # Wake up request/response handlers. self.__processing_condition.notifyAll() finally: self.__processing_condition.release() self.__ostream.close() pushy.util.logger.debug("Closed ostream") self.__istream.close() pushy.util.logger.debug("Closed istream") except: import traceback traceback.print_exc() pushy.util.logger.debug(traceback.format_exc()) def serve_forever(self): "Serve asynchronous requests from the peer forever." try: while self.__open: try: m = self.__waitForRequest() if m is not None and self.__open: self.__handle(m) except IOError: return except ValueError as err: # File could already be closed if err.message == 'I/O operation on closed file': return # re-raise if its something non-IO raise finally: pushy.util.logger.debug("Leaving serve_forever") def send_request(self, message_type, args): "Send a request message and wait for a response." # Convert the message type to a "as_tuple", if the user called # self.as_tuple. try: if self.__thread_local.as_tuple_count: if self.__thread_local.as_tuple_count == 1: del self.__thread_local.as_tuple_count else: self.__thread_local.as_tuple_count -= 1 args = (int(message_type), args) message_type = MessageType.as_tuple except AttributeError: pass # If a request is being processed, then increase the 'waiting' count, # so other threads may attempt to receive messages. self.__processing_condition.acquire() try: if not self.__open: raise Exception, "Connection is closed" handler = self.__response_handlers.get(thread.get_ident(), None) if handler is None: handler = ResponseHandler(self.__processing_condition) self.__response_handlers[handler.thread] = handler if self.__thread_request_count > 0: self.__waiting += 1 if self.__processing == self.__waiting: self.__processing_condition.notify() finally: self.__processing_condition.release() # Send the message. self.__send_message(message_type, args) # Wait for the response handler to be signalled. try: m = self.__waitForResponse(handler) while self.__open and (m is None or m.type not in response_types): if m is not None: self.__handle(m) m = self.__waitForResponse(handler) finally: if self.__thread_request_count == 0: del self.__response_handlers[handler.thread] return self.__handle(m) def __send_response(self, result): # Allow the message receiving thread to proceed. We must do this # *before* sending the message, in case the other side is # attempting to send a message at the same time. self.__processing_condition.acquire() try: self.__processing -= 1 if self.__processing == 0: self.__processing_condition.notifyAll() finally: self.__processing_condition.release() # Now send the message. self.__send_message(MessageType.response, result) def __waitForRequest(self): pushy.util.logger.debug("Enter waitForRequest") # Wait for a request message. If a response message is received first, # then set the relevant response handler and wait until we're allowed # to read a message before proceeding. self.__processing_condition.acquire() try: # Wait until we're allowed to read from the input stream, or # another thread has enqueued a request for us. while (self.__open and (len(self.__requests) == 0)) and \ (self.__receiving or \ self.__responses > 0 or \ (self.__processing > 0 and \ (self.__processing > self.__waiting))): self.__log_state() self.__processing_condition.notify() self.__processing_condition.wait() self.__log_state() # Check if the connection is still open. if not self.__open: return None # Check if another thread received a request message. if len(self.__requests) > 0: request = self.__requests.pop() self.__processing += 1 self.__processing_condition.notify() return request # Release the processing condition, and wait for a message. self.__receiving = True self.__processing_condition.release() notifyAll = True try: m = self.__recv() if m.target != 0: self.__responses += 1 self.__response_handlers[m.target].message = m else: # We got a request, so return it. Wake up one other thread # waiting to receive a message. if self.__open: self.__processing += 1 notifyAll = False return m finally: self.__processing_condition.acquire() self.__receiving = False if notifyAll: self.__processing_condition.notifyAll() else: self.__processing_condition.notify() finally: self.__processing_condition.release() pushy.util.logger.debug("Leave waitForRequest") def __waitForResponse(self, handler): pushy.util.logger.debug("Enter waitForResponse") self.__processing_condition.acquire() try: # Wait until we're allowed to read from the input stream, or # another thread has enqueued a request for us. while (self.__open and handler.message is None) and \ (self.__receiving or \ (self.__processing > 0 and \ (self.__processing > self.__waiting))): self.__log_state() self.__processing_condition.notify() self.__processing_condition.wait() self.__log_state() # Wait until we've got a response message. if handler.message is None and self.__open: self.__receiving = True self.__processing_condition.release() try: m = self.__recv() if m.target == 0: self.__requests.insert(0, m) else: self.__response_handlers[m.target].message = m if m.target != handler.thread: self.__responses += 1 finally: self.__processing_condition.acquire() self.__receiving = False elif self.__open: self.__responses -= 1 if handler.message is not None: if handler.message.type not in response_types: # Increment 'processing' count. self.__processing += 1 elif self.__thread_request_count > 0: # If we were waiting on a response, let the request # handler thread know that we're once again processing our # request. self.__waiting -= 1 elif not self.__open: raise Exception, "Connection is closed" return handler.message finally: handler.message = None self.__processing_condition.notifyAll() self.__processing_condition.release() pushy.util.logger.debug("Leave waitForResponse") def __marshal(self, obj): # XXX perhaps we can check refcount to optimise (if 1, immutable) if type(obj) in marshallable_types: return obj # If it's a tuple, try to marshal each item individually. if type(obj) is tuple: return (MARSHAL_TUPLE, tuple(map(self.__marshal, obj))) i = id(obj) if i in self.__proxied_objects: # The object has previously been proxied. self.__marshal_lock.acquire() try: if i in self.__proxied_objects: obj, result, version = self.__proxied_objects[i] self.__proxied_objects[i] = (obj, result, version+1) return (MARSHAL_PROXY, result, version+1) finally: self.__marshal_lock.release() if i in self.__proxy_ids: # Object originates at the peer. return (MARSHAL_ORIGIN, self.__proxy_ids[i][0]) else: # Create new entry in proxy objects map: # id -> (obj, opmask, proxy_type[, args]) # # opmask is a bitmask defining whether or not the object # defines various methods (__add__, __iter__, etc.) opmask = ProxyType.getoperators(obj) proxy_type = ProxyType.get(obj) args = ProxyType.getargs(proxy_type, obj) pushy.util.logger.debug( "Marshalling object: %r, %r", i, proxy_type) version = 0 if args is not None: marshalled_args = self.__marshal(args) result = (i, opmask, int(proxy_type), marshalled_args) else: result = (i, opmask, int(proxy_type)) self.__proxied_objects[i] = (obj, result, version) return (MARSHAL_PROXY, result, version) def __unmarshal(self, obj): if type(obj) is tuple: if obj[0] is MARSHAL_TUPLE: return tuple(map(self.__unmarshal, obj[1])) elif obj[0] is MARSHAL_ORIGIN: return self.__proxied_objects[obj[1]][0] elif obj[0] is MARSHAL_PROXY: description, version = obj[1], obj[2] oid = description[0] ref = self.__proxies.get(oid, None) if ref is not None: obj = ref() if obj is not None: # Update the local version self.__proxy_ids[id(obj)] = (oid, version) return obj opmask = description[1] proxy_type = proxy_types[description[2]] args = None if len(description) > 3: args = self.__unmarshal(description[3]) pushy.util.logger.debug( "Unmarshalling object: %r, %r, %r", oid, proxy_type, opmask) # New object: (id, opmask, object_type, args) register_proxy = \ lambda proxy: self.__register_proxy(proxy, oid, version) return Proxy(opmask, proxy_type, args, self, register_proxy) else: raise ValueError, "Invalid type: %r" % obj[0] else: # Simple type. return obj def __register_proxy(self, proxy, remote_id, version): id_proxy = id(proxy) pushy.util.logger.debug( "Registering a proxy: %r -> id=%r, version=%r", id_proxy, remote_id, version) if self.gc_enabled: ref = weakref.ref(proxy, lambda ref: self.delete(id_proxy)) else: ref = lambda: proxy self.__proxies[remote_id] = ref self.__proxy_ids[id_proxy] = (remote_id, version) def __send_message(self, message_type, args): # See if there are any objects to delete. If there are, send a delete # message first. self.__send_pending_deletes() # Send the original message. thread_id = self.__peer_thread marshalled = self.__marshal(args) payload = marshal.dumps(marshalled, 1) m = Message(message_type, payload, thread_id) pushy.util.logger.debug("Sending %r -> %r", m, thread_id) self.__ostream.send_message(m) def __send_pending_deletes(self): """ Checks if there are any pending deletions, and, if the garbage collecton timer has expired, sends a deletion message and resets the timer. Note that this method does not check whether GC is enabled, since there may be deletions enqueued since before GC was disabled. The initial check of "not self.__pending_deletes" should be sufficient to keep the overhead down. """ if not self.__pending_deletes: return time_now = time.time() if time_now - self.__last_delete > self.gc_interval: pending = self.__pending_deletes self.__pending_deletes = {} self.__delete_lock.acquire() try: if pending: self.__last_delete = time.time() try: pending_items = tuple(pending.items()) pushy.util.logger.debug("Deleting %r", pending_items) payload = marshal.dumps(pending_items, 1) m = Message(MessageType.delete, payload, 0, 0) pushy.util.logger.debug("Sending %r", m) self.__ostream.send_message(m) finally: pending.clear() finally: self.__delete_lock.release() def __recv(self): pushy.util.logger.debug("Waiting for message") m = self.__istream.receive_message() while m.type == MessageType.delete: pushy.util.logger.debug("Received %r", m) deleted_ids = marshal.loads(m.payload) self.__handle_delete(deleted_ids) m = self.__istream.receive_message() pushy.util.logger.debug("Received %r", m) return m def __handle(self, m): pushy.util.logger.debug("[%r] Handling message: %r", self.__connid, m) # Track the number of requests being processed in this thread. May be # greater than one, if there is to-and-fro. We need to track this so # we know when to set the 'peer_thread'. is_request = m.type not in response_types if is_request: self.__thread_request_count += 1 if self.__thread_request_count == 1: self.__peer_thread = m.source try: try: args = self.__unmarshal(marshal.loads(m.payload)) result = self.message_handlers[m.type](m.type, args) if m.type not in response_types: self.__send_response(result) return result except SystemExit, e: self.__send_response(e.code) raise e except: e = sys.exc_info()[1] # An exception raised while handling an exception message # should be sent up to the caller. if m.type is MessageType.exception: raise e # Allow the message receiving thread to proceed. self.__processing_condition.acquire() try: self.__processing -= 1 if self.__processing == 0: self.__processing_condition.notifyAll() finally: self.__processing_condition.release() # Send the above three objects to the caller import traceback pushy.util.logger.debug(traceback.format_exc()) self.__send_message(MessageType.exception, e) finally: if is_request: self.__thread_request_count -= 1 if self.__thread_request_count == 0: self.__peer_thread = 0 def __handle_delete(self, deleted): pushy.util.logger.debug("Handling delete: %r", deleted) try: self.__marshal_lock.acquire() try: for (id_, remote_version) in deleted: if id_ in self.__proxied_objects: obj, result, version = self.__proxied_objects[id_] if remote_version == version: del self.__proxied_objects[id_] finally: self.__marshal_lock.release() except: import traceback pushy.util.logger.debug(traceback.format_exc()) raise def __handle_response(self, message_type, result): return result def __handle_exception(self, message_type, e): raise e def delete(self, id_proxy): """ This is the weakref callback for proxied objects. This will enqueue the original object's ID and most recently received version for deletion at the originator. The __send_message method is reponsible for picking up these pending deletions, and sending them prior to any new messages. """ try: # If the connection is not closed, send a message to the peer to # delete its copy. id_orig, version = self.__proxy_ids[id_proxy] del self.__proxies[id_orig] del self.__proxy_ids[id_proxy] self.__delete_lock.acquire() try: self.__pending_deletes[id_orig] = version finally: self.__delete_lock.release() except: # [lp:784619] Pushy was causing an access violation at interpreter # exit on Windows 7 + Python 2.7.1. This was caused by an exception # raised during weak references being collected. Furthermore, since # it occurs during interpreter finalisation, various facilities # (e.g. logging) are unavailable. The only logical thing to do is # to swallow the exception and immediately return. return def as_tuple(self, fn): """ Calls the given function, ensuring that the next request sent to the peer will be returned as a tuple. """ try: self.__thread_local.as_tuple_count += 1 except AttributeError: self.__thread_local.as_tuple_count = 1 res = fn() assert type(res) is tuple return res pushy-0.5.3/pushy/protocol/__init__.py0000664000175000017500000000266412172577702020507 0ustar jamespagejamespage# Copyright (c) 2008 Andrew Wilkins # # Permission is hereby granted, free of charge, to any person # obtaining a copy of this software and associated documentation # files (the "Software"), to deal in the Software without # restriction, including without limitation the rights to use, # copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the # Software is furnished to do so, subject to the following # conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES # OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT # HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. """ This package defines the transport-independent protocol for communicating between interpreters. This is the core code for implementing proxies to objects in remote Python interpreters. """ import message import proxy from connection import Connection __all__ = ["message", "proxy", "Connection"] pushy-0.5.3/pushy/server.py0000664000175000017500000001072412172577702016411 0ustar jamespagejamespage# Copyright (c) 2009 Andrew Wilkins # # Permission is hereby granted, free of charge, to any person # obtaining a copy of this software and associated documentation # files (the "Software"), to deal in the Software without # restriction, including without limitation the rights to use, # copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the # Software is furnished to do so, subject to the following # conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES # OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT # HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. """ This module provides functions for starting a Pushy server within an existing Python process. """ import asyncore, os, socket, sys, threading __all__ = ["DEFAULT_PORT", "serve_forever", "run"] DEFAULT_PORT = 10101 def serve_forever(stdin, stdout): """ Start a Pushy server-side connection for a single client, and processes requests forever. @type stdin: file @param stdin: The reading file for the client. @type stdout: file @param stdout: The writing file for the client. """ import pushy.protocol, pushy.util c = pushy.protocol.Connection(stdin, stdout, False) try: c.serve_forever() finally: pushy.util.logger.debug("Closing connection") c.close() # Define a function for making a file descriptor's mode binary. try: import msvcrt def try_set_binary(fd): msvcrt.setmode(fd, os.O_BINARY) except ImportError: try_set_binary = lambda fd: None def serve_stdio_forever(stdin=sys.stdin, stdout=sys.stdout): """ Serve forever on the stdio files. This will replace the stdin/stdout/stderr file handles with pipes, and then creates threads to poll these pipes in the background and redirect them to sys.stdin/sys.stdout/sys.stderr. """ STDIN_FILENO = 0 STDOUT_FILENO = 1 STDERR_FILENO = 2 # Redirect stdout/stderr. We will redirect stdout/stderr to # /dev/null to start with, but this behaviour can be overridden # by assigning a different file to os.stdout/os.stderr. import pushy.util (so_r, so_w) = os.pipe() (se_r, se_w) = os.pipe() os.dup2(so_w, STDOUT_FILENO) os.dup2(se_w, STDERR_FILENO) for f in (so_r, so_w, se_r, se_w): try_set_binary(f) os.close(so_w) os.close(se_w) sys.stdout = open(os.devnull, "w") sys.stderr = sys.stdout so_redir = pushy.util.StdoutRedirector(so_r) se_redir = pushy.util.StderrRedirector(se_r) so_redir.start() se_redir.start() # Start the request servicing loop. try: serve_forever(stdin, stdout) stdout.close() stdin.close() os.close(STDOUT_FILENO) os.close(STDERR_FILENO) so_redir.join() se_redir.join() finally: stdout.close() stdin.close() class pushy_server(asyncore.dispatcher): def __init__(self, port): asyncore.dispatcher.__init__(self) self.create_socket(socket.AF_INET, socket.SOCK_STREAM) self.set_reuse_addr() self.bind(("", port)) self.listen(3) def handle_accept(self): (sock,addr) = self.accept() sock.setblocking(1) sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) stdin = sock.makefile("rb") stdout = sock.makefile("wb") threading.Thread(target=serve_forever, args=(stdin,stdout)).start() def handle_close(self): self.close() def run(port = DEFAULT_PORT): """ Start a socket server, which creates Pushy connections as client connections come in. @param port: The port number to listen on, or "stdio" to use standard I/O. """ if port == "stdio": serve_forever(sys.stdin, sys.stdout) else: server = pushy_server(int(port)) while True: asyncore.loop() if __name__ == "__main__": import pushy.util, logging, sys pushy.util.logger.addHandler(logging.StreamHandler()) run(*sys.argv[1:]) pushy-0.5.3/pushy/transport/0000755000175000017500000000000012204421005016533 5ustar jamespagejamespagepushy-0.5.3/pushy/transport/local.py0000664000175000017500000000412012172577702020222 0ustar jamespagejamespage# Copyright (c) 2008, 2009 Andrew Wilkins # # Permission is hereby granted, free of charge, to any person # obtaining a copy of this software and associated documentation # files (the "Software"), to deal in the Software without # restriction, including without limitation the rights to use, # copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the # Software is furnished to do so, subject to the following # conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES # OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT # HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. import os, shutil, subprocess, sys import pushy.transport class Popen(pushy.transport.BaseTransport): def __init__(self, command, address, **kwargs): pushy.transport.BaseTransport.__init__(self, address) if not sys.platform.startswith("java"): command[0] = sys.executable self.__proc = subprocess.Popen(command, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, bufsize=65535) self.stdout = self.__proc.stdout self.stderr = self.__proc.stderr self.stdin = self.__proc.stdin def getfile(self, src, dest): shutil.copyfile(src, dest) def putfile(self, src, dest): shutil.copyfile(src, dest) def __del__(self): self.close() def close(self): self.stdin.close() self.__proc.wait() pushy-0.5.3/pushy/transport/ssh.py0000664000175000017500000003023212173703444017723 0ustar jamespagejamespage# Copyright (c) 2008, 2011 Andrew Wilkins # # Permission is hereby granted, free of charge, to any person # obtaining a copy of this software and associated documentation # files (the "Software"), to deal in the Software without # restriction, including without limitation the rights to use, # copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the # Software is furnished to do so, subject to the following # conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES # OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT # HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. """ Popen-like interface for executing commands on a remote host via SSH. """ import os, sys import pushy.transport import pushy.util.askpass __all__ = ["ParamikoPopen", "NativePopen", "Popen"] is_windows = (sys.platform == "win32") try: import paramiko except ImportError: paramiko = None if paramiko: class WrappedChannelFile(object): def __init__(self, file_, how): self.__file = file_ self.__how = how def close(self): try: self.__file.channel.shutdown(self.__how) except: pass return self.__file.close() def __getattr__(self, name): return getattr(self.__file, name) class ParamikoPopen(pushy.transport.BaseTransport): """ An SSH transport for Pushy, which uses Paramiko. """ def __init__(self, command, address, missing_host_key_policy="reject", **kwargs): """ @param missing_host_key_policy: A string describing which policy to use ('autoadd', 'warning', or 'reject'), or an object of type paramiko.MissingHostKeyPolicy. @param kwargs: Any parameter that can be passed to U{paramiko.SSHClient.connect}. - port - username - password - pkey - key_filename - timeout - allow_agent - look_for_keys """ pushy.transport.BaseTransport.__init__(self, address) self.__client = paramiko.SSHClient() self.__client.load_system_host_keys() # Set the missing host key policy. if type(missing_host_key_policy) is str: missing_host_key_policy = missing_host_key_policy.lower() if missing_host_key_policy == "autoadd": self.__client.set_missing_host_key_policy( paramiko.AutoAddPolicy()) elif missing_host_key_policy == "warning": self.__client.set_missing_host_key_policy( paramiko.WarningPolicy()) else: if missing_host_key_policy != "reject": import warnings warnings.warn("Unknown missing host key policy: " +\ missing_host_key_policy) self.__client.set_missing_host_key_policy( paramiko.RejectPolicy()) else: # Assume it is a user-defined policy object. self.__client.set_missing_host_key_policy( missing_host_key_policy) # Create the connection. connect_args = {"hostname": address} for name in ("port", "username", "password", "pkey", "key_filename", "timeout", "allow_agent", "look_for_keys"): if name in kwargs: connect_args[name] = kwargs[name] self.__client.connect(**connect_args) # Join arguments into a string args = command for i in range(len(args)): if " " in args[i]: args[i] = "'%s'" % args[i] command = " ".join(args) stdin, stdout, stderr = self.__client.exec_command(command) self.stdin = WrappedChannelFile(stdin, 1) self.stdout = WrappedChannelFile(stdout, 0) self.stderr = stderr self.fs = self.__client.open_sftp() def __del__(self): self.close() def close(self): if hasattr(self, "stdin"): self.stdin.close() self.stdout.close() self.stderr.close() self.__client.close() ############################################################################### # Native SSH. Truly secure only when a password is not specified. native_ssh = os.environ.get("PUSHY_NATIVE_SSH", None) if native_ssh is not None: if not os.path.exists(native_ssh): import warnings warnings.warn( "Ignoring non-existant PUSHY_NATIVE_SSH (%r)" % native_ssh) native_ssh = None if native_ssh is None: paths = [] if "PATH" in os.environ: paths = os.environ["PATH"].split(os.pathsep) ssh_program = "ssh" if is_windows: ssh_program = "plink.exe" for path in paths: path = os.path.join(path, ssh_program) if os.path.exists(path): native_ssh = path break native_scp = os.environ.get("PUSHY_NATIVE_SCP", None) if native_scp is not None: if not os.path.exists(native_scp): import warnings warnings.warn( "Ignoring non-existant PUSHY_NATIVE_SCP (%r)" % native_scp) native_scp = None if native_scp is None: paths = [] if "PATH" in os.environ: paths = os.environ["PATH"].split(os.pathsep) scp_program = "scp" if is_windows: scp_program = "pscp.exe" for path in paths: path = os.path.join(path, scp_program) if os.path.exists(path): native_scp = path break if native_ssh is not None: import subprocess class NativePopen(pushy.transport.BaseTransport): """ An SSH transport for Pushy, which uses the native ssh or plink program. """ def __init__(self, command, address, username=None, password=None, port=None, **kwargs): """ @param address: The hostname/address to connect to. @param username: The username to connect with. """ pushy.transport.BaseTransport.__init__(self, address) if username is not None: address = username + "@" + address command = ['"%s"' % word for word in command] args = [native_ssh] # Plink allows specification of password on the command-line. Not # secure, as other processes may see others' command-line # arguments. "use_native=False" should be specified if no password # specified, and security is of great importance. if password is not None and ssh_program == "plink.exe": args.extend(["-pw", password]) # If a port other than 22 is specified, add it to the arguments. # Plink expects "-P", while ssh expects "-p". self.__port = 22 if port is not None: port = int(port) if port != 22: self.__port = port if ssh_program == "plink.exe": args.extend(["-P", str(port)]) else: args.extend(["-p", str(port)]) # Add the address and command to the arguments. args.extend([address, " ".join(command)]) if is_windows or password is None: self.__proc = subprocess.Popen(args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) else: self.__proc = pushy.util.askpass.Popen(args, password, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) self.stdout = self.__proc.stdout self.stdin = self.__proc.stdin self.stderr = self.__proc.stderr # If we have native SCP, store the address and password for later # use. if native_scp: self.__address = address self.__password = password self.putfile = self._putfile self.getfile = self._getfile def _putfile(self, src, dest): "Copy local file 'src' to remote file 'dest'" return self.scp(src, self.__address + ":" + dest) def _getfile(self, src, dest): "Copy remote file 'src' to local file 'dest'" return self.scp(self.__address + ":" + src, dest) def scp(self, src, dest): args = [native_scp, "-q"] if self.__password is not None and scp_program == "pscp.exe": args.extend(["-pw", self.__password]) if self.__port != 22: # Both pscp and scp expect "-P" (hooray for consistency!) args.extend(["-P", str(self.__port)]) args.extend((src, dest)) if is_windows or self.__password is None: proc = subprocess.Popen(args, stdin=subprocess.PIPE) else: proc = pushy.util.askpass.Popen(args, self.__password, stdin=subprocess.PIPE) proc.stdin.close() return proc.wait() def __del__(self): self.close() def close(self): self.stdin.close() self.stdout.close() self.stderr.close() try: import os, signal os.kill(self.__proc.pid, signal.SIGTERM) except: pass self.__proc.wait() ############################################################################### # Define Popen. if native_ssh is not None: def Popen(command, use_native=None, password=None, *args, **kwargs): """ Selects between the native and Paramiko SSH transports, opting to use the native program where possible. @type use_native: bool @param use_native: If set to False, then the native implementation will never be used, regardless of whether a password is specified. @type password: string @param password: The password to use for authentication. If a password is not specified, then native ssh/plink program is used, except if L{use_native} is False. @type port: int @param port: The port of the SSH daemon to connect to. If not specified, the default port of 22 will be used. """ # Determine default value for "use_native", based on other parameters. if use_native is None: # On Windows, PuTTy is actually slower than Paramiko! use_native = \ (not paramiko) or ((not is_windows) and (password is None)) if use_native: kwargs["password"] = password return NativePopen(command, *args, **kwargs) assert paramiko is not None return ParamikoPopen(command, password=password, *args, **kwargs) elif paramiko: Popen = ParamikoPopen else: import warnings warnings.warn("No paramiko or native ssh transport") pushy-0.5.3/pushy/transport/smb/0000755000175000017500000000000012204421005017314 5ustar jamespagejamespagepushy-0.5.3/pushy/transport/smb/native.py0000664000175000017500000002704112172577702021206 0ustar jamespagejamespage# Copyright (c) 2009 Andrew Wilkins # # Permission is hereby granted, free of charge, to any person # obtaining a copy of this software and associated documentation # files (the "Software"), to deal in the Software without # restriction, including without limitation the rights to use, # copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the # Software is furnished to do so, subject to the following # conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES # OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT # HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. __all__ = ["NativePopen"] import os, struct, sys, subprocess, StringIO, threading import pushy.transport from ctypes import * from ctypes.wintypes import * import msvcrt # Win32 API Constants LOGON32_LOGON_INTERACTIVE = 2 LOGON32_PROVIDER_DEFAULT = 0 NameSamCompatible = 2 GENERIC_READ = 0x80000000 GENERIC_WRITE = 0x40000000 OPEN_EXISTING = 3 INVALID_HANDLE_VALUE = 2**32-1 FILE_FLAG_NO_BUFFERING = 0x20000000 FILE_FLAG_WRITE_THROUGH = 0x80000000 FILE_FLAG_OVERLAPPED = 0x40000000 ERROR_IO_PENDING = 997 SC_MANAGER_CONNECT = 0x0001 SC_MANAGER_CREATE_SERVICE = 0x0002 SC_MANAGER_ENUMERATE_SERVICE = 0x0004 SERVICE_WIN32 = 0x00000030 SERVICE_STATE_ALL = 0x00000003 ERROR_MORE_DATA = 234 # Structures class overlapped_internal_struct(Structure): _fields_ = [("Offset", DWORD), ("OffsetHigh", DWORD)] class overlapped_internal_union(Union): _fields_ = [("struct", overlapped_internal_struct), ("Pointer", c_void_p)] class OVERLAPPED(Structure): _fields_ = [("Internal", c_void_p), ("InternalHigh", c_void_p), ("union", overlapped_internal_union), ("hEvent", HANDLE)] # Win32 API Functions GetLastError = windll.kernel32.GetLastError GetLastError.restype = DWORD GetLastError.argtypes = [] CreateFile = windll.kernel32.CreateFileA CreateFile.restype = HANDLE CreateFile.argtypes = [c_char_p, DWORD, DWORD, c_void_p, DWORD, DWORD, HANDLE] WriteFile = windll.kernel32.WriteFile WriteFile.restype = BOOL WriteFile.argtypes = [ HANDLE, c_void_p, DWORD, POINTER(DWORD), POINTER(OVERLAPPED)] ReadFile = windll.kernel32.ReadFile ReadFile.restype = BOOL ReadFile.argtypes = [ HANDLE, c_void_p, DWORD, POINTER(DWORD), POINTER(OVERLAPPED)] CreateEvent = windll.kernel32.CreateEventA CreateEvent.restype = HANDLE CreateEvent.argtypes = [c_void_p, BOOL, BOOL, c_char_p] GetOverlappedResult = windll.kernel32.GetOverlappedResult GetOverlappedResult.restype = BOOL GetOverlappedResult.argtypes = [ HANDLE, POINTER(OVERLAPPED), POINTER(DWORD), BOOL] CloseHandle = windll.kernel32.CloseHandle CloseHandle.restype = BOOL CloseHandle.argtypes = [HANDLE] LogonUser = windll.advapi32.LogonUserA LogonUser.restype = BOOL LogonUser.argtypes = [c_char_p, c_char_p, c_char_p, DWORD, DWORD, POINTER(HANDLE)] ImpersonateLoggedOnUser = windll.advapi32.ImpersonateLoggedOnUser ImpersonateLoggedOnUser.restype = BOOL ImpersonateLoggedOnUser.argtypes = [HANDLE] RevertToSelf = windll.advapi32.RevertToSelf RevertToSelf.restype = BOOL RevertToSelf.argtypes = [] GetUserNameExA = windll.secur32.GetUserNameExA GetUserNameExA.restype = BOOL GetUserNameExA.argtypes = [DWORD, c_char_p, POINTER(DWORD)] FlushFileBuffers = windll.kernel32.FlushFileBuffers FlushFileBuffers.restype = BOOL FlushFileBuffers.argtypes = [HANDLE] OpenSCManager = windll.advapi32.OpenSCManagerA OpenSCManager.restype = HANDLE OpenSCManager.argtypes = [c_char_p, c_char_p, DWORD] CloseServiceHandle = windll.advapi32.CloseServiceHandle CloseServiceHandle.restype = BOOL CloseServiceHandle.argtypes = [HANDLE] EnumServicesStatusEx = windll.advapi32.EnumServicesStatusExA EnumServicesStatusEx.restype = BOOL EnumServicesStatusEx.argtypes = [ HANDLE, DWORD, DWORD, DWORD, c_char_p, DWORD, POINTER(DWORD), POINTER(DWORD), POINTER(DWORD), c_char_p] class Handle: "Class for closing underlying Win32 handle on deletion." def __init__(self, handle=HANDLE()): self.handle = handle def __del__(self): self.close() def close(self): if self.handle: value = CloseHandle(self.handle) if not value: raise Exception, "Failed to close handle" self.handle = None class Win32File(Handle): """ Not sure why, but a simple open(pipe_path, "w+b") can't be shared for both stdin and stdout. Calling ReadFile/WriteFile directly, we can do this. We must use overlapped I/O, since a read and write may occur on the pipe at the same time. Without using overlapped I/O, one caller would be blocked by the other. """ def __init__(self, handle): Handle.__init__(self, HANDLE(handle)) self.__fd = msvcrt.open_osfhandle(self.handle.value, 0) def close(self): Handle.close(self) def flush(self): FlushFileBuffers(self.handle) def fileno(self): return self.__fd def write(self, data): buffer = create_string_buffer(data) total_written = 0 while total_written < len(data): written = DWORD(0) overlapped = OVERLAPPED() overlapped.hEvent = CreateEvent(None, True, False, None) try: # Cross-network pipes are limited to 65535 bytes at a time. n = len(data) - total_written if n > 65535: n = 65535 if not WriteFile(self.handle, addressof(buffer)+total_written, n, byref(written), byref(overlapped)): error = GetLastError() if error != ERROR_IO_PENDING: raise IOError, error if not GetOverlappedResult(self.handle, byref(overlapped), byref(written), True): raise IOError, GetLastError() finally: CloseHandle(overlapped.hEvent) total_written += written.value def read(self, size): buffer = create_string_buffer(size) total_nread = 0 while total_nread < size: overlapped = OVERLAPPED() overlapped.hEvent = CreateEvent(None, True, False, None) try: nread = DWORD(0) if not ReadFile(self.handle, addressof(buffer)+total_nread, size-total_nread, byref(nread), byref(overlapped)): error = GetLastError() if error != ERROR_IO_PENDING: raise IOError, error if not GetOverlappedResult(self.handle, byref(overlapped), byref(nread), True): raise IOError, GetLastError() total_nread += nread.value finally: CloseHandle(overlapped.hEvent) return buffer.raw[:total_nread] def readlines(self): data = "" p = self.read(8192) while len(p): data += p p = self.read(8192) return data.splitlines() def authenticate(username, password, domain): if domain is None: slash = username.find("\\") if slash != -1: domain = username[:slash] username = username[slash+1:] handle = Handle() if LogonUser(username, domain, password, LOGON32_LOGON_INTERACTIVE, LOGON32_PROVIDER_DEFAULT, byref(handle.handle)): return handle return None def logon(token): """ Impersonates a user, given a token returned from 'authenticate'. If the function succeeds, it will reutrn TRUE, else FALSE. """ if token: return ImpersonateLoggedOnUser(token.handle) return False def install_service(hostname): access = SC_MANAGER_CONNECT | SC_MANAGER_ENUMERATE_SERVICE handle = OpenSCManager(hostname, None, access) if handle == 0: raise IOError, GetLastError() try: bufsize = 2**16-1 buf = create_string_buffer(bufsize) bytesneeded = DWORD(0) servicecount = DWORD(0) resumehandle = DWORD(0) while not EnumServicesStatusEx(handle, 0, SERVICE_WIN32, SERVICE_STATE_ALL, buf, bufsize, byref(bytesneeded), byref(servicecount), byref(resumehandle), None): if GetLastError() != ERROR_MORE_DATA: raise IOError, GetLastError() print "Enum succeeded", servicecount, resumehandle, error print servicecount finally: CloseServiceHandle(handle) def get_current_username_domain(): username = create_string_buffer(1024) namelen = DWORD(1024) if GetUserNameExA(NameSamCompatible, username, byref(namelen)): username = tuple(username.value.split("\\")) if len(username) == 1: return (None, username[0]) return username return None class NativePopen: """ SMB transport implementation which uses the Windows native API for connecting to a remote named pipe. """ def __init__(self, hostname, username, password, domain): """ @param hostname: The hostname of the target machine. @param username: The username to authenticate with. @param password: The password to authenticate with. @param domain: The domain to authenticate with. """ # Determine current username/domain. If they're the same as the # one specified, don't bother authenticating. current_username = get_current_username_domain() if domain == "": import platform already_logged = \ (current_username == (platform.node(), username)) else: already_logged = (current_username == (domain, username)) # Log on, if we're not already logged on. if already_logged or \ logon(authenticate(username, password, domain)): try: # Install the pushy service. install_service(hostname) # Connect to the pushy daemon's named pipe. flags = FILE_FLAG_NO_BUFFERING | FILE_FLAG_WRITE_THROUGH |\ FILE_FLAG_OVERLAPPED handle = \ CreateFile(r"\\%s\pipe\pushy" % hostname, GENERIC_READ | GENERIC_WRITE, 0, 0, OPEN_EXISTING, flags, 0) if handle == INVALID_HANDLE_VALUE: raise Exception, "Failed to open named pipe" self.stdin = self.stdout = Win32File(handle) finally: if not already_logged: RevertToSelf() else: raise Exception, "Authentication failed" pushy-0.5.3/pushy/transport/smb/impacket_transport.py0000664000175000017500000000722612172577702023634 0ustar jamespagejamespage# Copyright (c) 2009 Andrew Wilkins # # Permission is hereby granted, free of charge, to any person # obtaining a copy of this software and associated documentation # files (the "Software"), to deal in the Software without # restriction, including without limitation the rights to use, # copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the # Software is furnished to do so, subject to the following # conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES # OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT # HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. __all__ = ["ImpacketPopen"] import impacket.smb class PushySMB(impacket.smb.SMB): def __init__(self, *args, **kwargs): impacket.smb.SMB.__init__(self, *args, **kwargs) class SMBFile: def __init__(self, server, tid, fid): self.__lock = threading.Lock() self.__server = server self.__tid = tid self.__fid = fid def __del__(self): self.close() def flush(self): pass def close(self): if self.__server is not None: self.__lock.acquire() try: if self.__server is not None: self.__server.close(self.__tid, self.__fid) self.__server = None finally: self.__lock.release() def write(self, data): self.__lock.acquire() try: self.__server.write_raw(self.__tid, self.__fid, data) finally: self.__lock.release() def read(self, size): self.__lock.acquire() try: data = self.__server.read(self.__tid, self.__fid, 0, size) if data is None: data = "" return data finally: self.__lock.release() def readlines(self): data = "" p = self.read(8192) while len(p): data += p p = self.read(8192) return data.splitlines() class ImpacketPopen: """ SMB transport implementation which uses Impacket for connecting to a remote named pipe. """ def __init__(self, hostname, username, password, domain): """ @param hostname: The hostname of the target machine. @param username: The username to authenticate with. @param password: The password to authenticate with. @param domain: The domain to authenticate with. """ # Create SMB connection, authenticate user. # TODO make port configurable server = PushySMB("*SMBSERVER", hostname) server.login(username, password, domain) # Connect to IPC$ share. tid_ipc = server.tree_connect_andx(r"\\*SMBSERVER\IPC$") # Open Pushy service's named pipes. fid = server.open(tid_ipc, r"\pipe\pushy", impacket.smb.SMB_O_OPEN, impacket.smb.SMB_ACCESS_READWRITE)[0] # Create file-like objects for stdin/stdout/stderr. self.stdin = SMBFile(server, tid_ipc, fid) self.stdout = self.stdin self.server = server def close(self): self.stdin.close() self.stdout.close() pushy-0.5.3/pushy/transport/smb/__init__.py0000664000175000017500000000525312172577702021460 0ustar jamespagejamespage# Copyright (c) 2009 Andrew Wilkins # # Permission is hereby granted, free of charge, to any person # obtaining a copy of this software and associated documentation # files (the "Software"), to deal in the Software without # restriction, including without limitation the rights to use, # copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the # Software is furnished to do so, subject to the following # conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES # OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT # HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. __all__ = ["Popen"] import struct, sys, pushy.transport, StringIO # On Windows, prefer the native interface. BasePopen = None if sys.platform == "win32": try: import native BasePopen = native.NativePopen except ImportError: pass # If we're not on Windows, or the native interface is unavailable, use the # Impacket library instead. if BasePopen is None: import impacket_transport BasePopen = impacket_transport.ImpacketPopen # Common class which inherits from either the native win32 or Impacket class. class Popen(pushy.transport.BaseTransport, BasePopen): def __init__(self, command, address, username=None, password=None, domain="", **kwargs): """ @param username: The username to authenticate with. @param password: The password to authenticate with. @param domain: The domain to authenticate with. """ # If no domain is specified, and the domain is specified in the # username, split it out. if not domain and username and "\\" in username: domain, username = username.split("\\") pushy.transport.BaseTransport.__init__(self, address) BasePopen.__init__(self, address, username, password, domain) self.stderr = StringIO.StringIO() # Write program arguments. packed_args = struct.pack(">B", len(command)) for arg in command: packed_args += struct.pack(">I", len(arg)) + arg self.stdin.write(packed_args) self.stdin.flush() pushy-0.5.3/pushy/transport/daemon.py0000664000175000017500000000451612172577702020404 0ustar jamespagejamespage# Copyright (c) 2009, 2011 Andrew Wilkins # # Permission is hereby granted, free of charge, to any person # obtaining a copy of this software and associated documentation # files (the "Software"), to deal in the Software without # restriction, including without limitation the rights to use, # copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the # Software is furnished to do so, subject to the following # conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES # OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT # HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. import os, socket, StringIO import pushy.transport import pushy.server DEFAULT_PORT = pushy.server.DEFAULT_PORT class WrappedSocketFile(object): def __init__(self, file_, socket_, how): self.__file = file_ self.__socket = socket_ self.__how = how def close(self): try: self.__socket.shutdown(self.__how) except: pass return self.__file.close() def __getattr__(self, name): return getattr(self.__file, name) class Popen(pushy.transport.BaseTransport): def __init__(self, command, address, port=DEFAULT_PORT, **kwargs): pushy.transport.BaseTransport.__init__(self, address, daemon=True) self.__socket = socket.socket() self.__socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) self.__socket.connect((address, port)) self.stdin = WrappedSocketFile(self.__socket.makefile("wb"), self.__socket, socket.SHUT_WR) self.stdout = WrappedSocketFile(self.__socket.makefile("rb"), self.__socket, socket.SHUT_RD) self.stderr = StringIO.StringIO() self.stdin._close = True def close(self): self.__socket.close() pushy-0.5.3/pushy/transport/__init__.py0000664000175000017500000000337712172577702020704 0ustar jamespagejamespage# Copyright (c) 2008 Andrew Wilkins # # Permission is hereby granted, free of charge, to any person # obtaining a copy of this software and associated documentation # files (the "Software"), to deal in the Software without # restriction, including without limitation the rights to use, # copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the # Software is furnished to do so, subject to the following # conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES # OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT # HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. class BaseTransport: "Base class for all transports." def __init__(self, address, daemon=False): """ @param address: The transport-specific address. @type daemon: bool @param daemon: True if the transport is a long-lived transport, such as the L{daemon} transport. If this is set to True, then the Pushy client will not send the Pushy package to the remote process when a connection is initiated. """ self.address = address self.daemon = daemon pushy-0.5.3/pushy/util/0000755000175000017500000000000012204421005015454 5ustar jamespagejamespagepushy-0.5.3/pushy/util/askpass.py0000664000175000017500000001110612172577702017520 0ustar jamespagejamespage# Copyright (c) 2009 Andrew Wilkins # # Permission is hereby granted, free of charge, to any person # obtaining a copy of this software and associated documentation # files (the "Software"), to deal in the Software without # restriction, including without limitation the rights to use, # copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the # Software is furnished to do so, subject to the following # conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES # OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT # HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. import sys, os, subprocess, tempfile from subprocess import PIPE, STDOUT def forknotty(delegate=None): """ Fork the process, detach the child from the terminal, and then wait for it to complete in the parent process. By using this as the "preexec_fn" parameter for subprocess.Popen, one can run a sub-process without a terminal. If a parameter is passed to this, then it is expected to be another callable object which will be invoked. """ # Fork. pid = os.fork() if pid > 0: for fd in range(subprocess.MAXFD): try: os.close(fd) except OSError: pass (pid, return_code) = os.waitpid(pid, 0) os._exit(return_code) # Detach from parent process. os.setsid() os.umask(0) # Fork again - some UNIXes need this. pid = os.fork() if pid > 0: for fd in range(subprocess.MAXFD): try: os.close(fd) except OSError: pass (pid, return_code) = os.waitpid(pid, 0) os._exit(return_code) # Call the delegate, if any. if delegate: delegate() class _Popen(subprocess.Popen): """ Wrapper for subprocess.Popen, which removes the temporary SSH_ASKPASS program. """ def __init__(self, askpass, *args, **kwargs): subprocess.Popen.__init__(self, *args, **kwargs) self.__askpass = askpass def __del__(self): if os.path.exists(self.__askpass): os.remove(self.__askpass) subprocess.Popen.__del__(self) def wait(self): try: return subprocess.Popen.wait(self) finally: if os.path.exists(self.__askpass): os.remove(self.__askpass) def Popen(command, password, **kwargs): """ Execute the specified command in a subprocess, with SSH_ASKPASS set in such a way as to allow passing a password non-interactively to programs that need it (e.g. ssh, scp). """ # Set up the "preexec" function, which is forknotty and whatever the user # passes in. preexec_fn = forknotty if "preexec_fn" in kwargs: fn = kwargs["preexec_fn"] preexec_fn = lambda: forknotty(fn) kwargs["preexec_fn"] = preexec_fn # Create a temporary, executable-by-owner file. (fd, path) = tempfile.mkstemp(text=True) os.chmod(path, 0700) try: tf = os.fdopen(fd, "w") print >> tf, "#!" + sys.executable print >> tf, "import os; print os.environ['SSH_PASSWORD']" tf.close() # Configure environment variables. environ = kwargs.get("env", os.environ) if environ is None: environ = os.environ environ = environ.copy() environ["SSH_PASSWORD"] = password environ["DISPLAY"] = "none:0.0" environ["SSH_ASKPASS"] = path kwargs["env"] = environ # Finally, call subprocess.Popen. return _Popen(path, command, **kwargs) except: import traceback traceback.print_exc() if os.path.exists(path): os.remove(path) if __name__ == "__main__": program = """\ import os, sys, pprint pprint.pprint(os.environ.items()) print sys.stdin.read() """ new_env = os.environ.copy() new_env["ARARARARAR"] = "1231231231312" proc = Popen(["python", "-c", program], "blahblah", stdin=subprocess.PIPE, stdout=subprocess.PIPE, env=new_env) print proc.communicate("abcxyz")[0] proc.wait() pushy-0.5.3/pushy/util/_zipwalk.py0000664000175000017500000000445212172577702017701 0ustar jamespagejamespage# Copyright (c) 2008 Andrew Wilkins # # Permission is hereby granted, free of charge, to any person # obtaining a copy of this software and associated documentation # files (the "Software"), to deal in the Software without # restriction, including without limitation the rights to use, # copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the # Software is furnished to do so, subject to the following # conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES # OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT # HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. import os def zipwalk(zf, subdir=None): """ An os.walk-like function for iterating through the hierarchy of a zip archive. """ root = [{}, []] for name in zf.namelist(): is_dir = name.endswith("/") if is_dir: name = name[:-1] parts = name.split("/") parent = root for part in parts[:-1]: if part not in parent[0]: parent[0][part] = [{}, []] parent = parent[0][part] if is_dir: parent[0][parts[-1]] = [{}, []] else: parent[1].append(parts[-1]) def _walk(hierarchy, dir): yield (dir, hierarchy[0].keys(), hierarchy[1]) for (subdir, sub_hierarchy) in hierarchy[0].items(): if dir is not None: subdir = os.path.join(dir, subdir) for result in _walk(sub_hierarchy, subdir): yield result subdir_hierarchy = root if subdir is not None: subdir = os.path.normpath(subdir) for part in subdir.split(os.sep): subdir_hierarchy = subdir_hierarchy[0][part] return _walk(subdir_hierarchy, subdir) pushy-0.5.3/pushy/util/redirector.py0000664000175000017500000000436412172577702020225 0ustar jamespagejamespage# Copyright (c) 2008 Andrew Wilkins # # Permission is hereby granted, free of charge, to any person # obtaining a copy of this software and associated documentation # files (the "Software"), to deal in the Software without # restriction, including without limitation the rights to use, # copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the # Software is furnished to do so, subject to the following # conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES # OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT # HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. import os, sys, threading import pushy.util class OutputRedirector(threading.Thread): """ A class for reading data from a file and passing it to the "write" method of an object. """ def __init__(self, fileno, bufsize = 8192): threading.Thread.__init__(self) self.setDaemon(True) self.__fileno = fileno self.__bufsize = bufsize def getfile(self): raise "Not implemented" def run(self): try: pushy.util.logger.debug("Entered output redirector") try: data = os.read(self.__fileno, self.__bufsize) while len(data) > 0: self.getfile().write(data) data = os.read(self.__fileno, self.__bufsize) finally: self.getfile().flush() except: pushy.util.logger.debug("Leaving output redirector") pass class StdoutRedirector(OutputRedirector): def getfile(self): return sys.stdout class StderrRedirector(OutputRedirector): def getfile(self): return sys.stderr pushy-0.5.3/pushy/util/_logging.py0000664000175000017500000000373412172577702017650 0ustar jamespagejamespage# Copyright (c) 2009 Andrew Wilkins # # Permission is hereby granted, free of charge, to any person # obtaining a copy of this software and associated documentation # files (the "Software"), to deal in the Software without # restriction, including without limitation the rights to use, # copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the # Software is furnished to do so, subject to the following # conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES # OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT # HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. import logging, os, sys logger = logging.getLogger("pushy") logger.disabled = True logger.setLevel(logging.WARNING) if not logger.disabled: class ShutdownSafeFileHandler(logging.FileHandler): """ A FileHandler class which protects calls to "close" from concurrent logging operations. """ def __init__(self, filename): logging.FileHandler.__init__(self, filename) def close(self): self.acquire() try: logging.FileHandler.close(self) finally: self.release() handler = ShutdownSafeFileHandler("pushy.%d.log" % os.getpid()) handler.setFormatter( logging.Formatter( "[%(process)d:(%(threadName)s:%(thread)d)] %(message)s")) logger.addHandler(handler) logger.setLevel(logging.DEBUG) logger.debug("sys.argv: %r", sys.argv) pushy-0.5.3/pushy/util/clone_function.py0000664000175000017500000000341112172577702021060 0ustar jamespagejamespage# Copyright (c) 2011 Andrew Wilkins # # Permission is hereby granted, free of charge, to any person # obtaining a copy of this software and associated documentation # files (the "Software"), to deal in the Software without # restriction, including without limitation the rights to use, # copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the # Software is furnished to do so, subject to the following # conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES # OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT # HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. def clone_function(func, globals=None): """ Clone a function object by creating a new FunctionType with a cloned code object. """ from types import CodeType, FunctionType co = func.func_code code = CodeType( co.co_argcount, co.co_nlocals, co.co_stacksize, co.co_flags, co.co_code, co.co_consts, co.co_names, co.co_varnames, co.co_filename, co.co_name, co.co_firstlineno, co.co_lnotab ) import __builtin__ if globals is None: globals = __builtin__.globals() return FunctionType(code, globals, func.func_name, func.func_defaults) pushy-0.5.3/pushy/util/__init__.py0000664000175000017500000000244512172577702017620 0ustar jamespagejamespage# Copyright (c) 2008, 2009 Andrew Wilkins # # Permission is hereby granted, free of charge, to any person # obtaining a copy of this software and associated documentation # files (the "Software"), to deal in the Software without # restriction, including without limitation the rights to use, # copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the # Software is furnished to do so, subject to the following # conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES # OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT # HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. from _zipwalk import zipwalk from redirector import StdoutRedirector, StderrRedirector from _logging import logger __all__ = ["zipwalk", "StdoutRedirector", "StderrRedirector", "logger"] pushy-0.5.3/pushy/__init__.py0000664000175000017500000000567012173703706016642 0ustar jamespagejamespage# Copyright (c) 2008, 2011 Andrew Wilkins # # Permission is hereby granted, free of charge, to any person # obtaining a copy of this software and associated documentation # files (the "Software"), to deal in the Software without # restriction, including without limitation the rights to use, # copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the # Software is furnished to do so, subject to the following # conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES # OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT # HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. """ I{Pushy} is a Python package for simple, Pythonic remote procedure calls. Pushy allows one to connect two Python interpreters together, and provide access to their modules, objects and functions to each other. What sets Pushy apart from other such frameworks is its ability to operate without any custom services running to achieve this. Pushy comes packaged with several different transports: - L{SSH} (Secure Shell), via U{Paramiko}, or the standard ssh/plink programs. - L{SMB} (Windows Named Pipes), via Impacket, or (on Windows) native Windows named-pipe operations. - L{Local}, for connecting to a new Python interpreter on the local machine. - L{Daemon}, for connecting to a "nailed-up" socket-based service. By using the SSH transport (originally the only transport available), one can create and connect to a new Python interpreter on a remote machine, with only Python installed and an SSH daemon running. Pushy does not need to be installed on the remote machine; the necessary code is I{pushed} to it - hence the name. Moreover, Pushy gains the the encryption and authentication provided by SSH for free. There's no need to worry about your RPC service being compromised, if you run it with reduced privileges. A Pushy connection can be initiated by invoking L{pushy.connect}. This function will return a connection object, which provides access to the remote interpreter's modules, whose classes, objects and functions can be transparently accessed from the local interpreter. @version: 0.5.3 @author: Andrew Wilkins @contact: axwalk@gmail.com @license: MIT """ transports = {"ssh": None, "local": None, "daemon": None, "smb": None} from client import connect from server import serve_forever pushy-0.5.3/pushy/client.py0000664000175000017500000004674512172577702016375 0ustar jamespagejamespage# Copyright (c) 2008, 2011 Andrew Wilkins # # Permission is hereby granted, free of charge, to any person # obtaining a copy of this software and associated documentation # files (the "Software"), to deal in the Software without # restriction, including without limitation the rights to use, # copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the # Software is furnished to do so, subject to the following # conditions: # # The above copyright notice and this permission notice shall be # included in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES # OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT # HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR # OTHER DEALINGS IN THE SOFTWARE. """ This module provides the code to used to start a remote Pushy server running, and initiate a connection. """ import __builtin__ import imp import inspect import marshal import os import struct import sys import threading # Import zipimport, for use in PushyPackageLoader. try: import zipimport except ImportError: zipimport = None # Import hashlib if it exists, with md5 as a backup. try: import hashlib except ImportError: import md5 as hashlib __all__ = ["PushyPackageLoader", "InMemoryLoader", "InMemoryLoader", "AutoImporter", "PushyClient", "connect"] class PushyPackageLoader: """ This class loads packages and modules into the format expected by PushyServer.InMemoryImporter. """ def load(self, *args): self.__packages = {} self.__modules = {} for arg in args: self.__load(arg) return self.__packages, self.__modules def __load(self, package_or_module): if hasattr(package_or_module, "__path__"): self.__loadPackage(package_or_module) else: self.__loadModule(package_or_module) def __loadPackage(self, package): if len(package.__path__) == 0: return path = package.__path__[0] if os.sep != "/": path = path.replace(os.sep, "/") # Check to see if the package was loaded from a zip file. is_zip = False if zipimport is not None: if hasattr(package, "__loader__"): is_zip = isinstance(package.__loader__, zipimport.zipimporter) if is_zip: zippath = package.__loader__.archive subdir = path[len(zippath)+1:] # Skip '/' import zipfile, pushy.util zf = zipfile.ZipFile(zippath) walk_fn = lambda: pushy.util.zipwalk(zf, subdir) read_fn = lambda path: zf.read(path) else: prefix_len = len(path) - len(package.__name__) #prefix_len = path.rfind("/") + 1 walk_fn = lambda: os.walk(path) read_fn = lambda path: open(path).read() for root, dirs, files in walk_fn(): if os.sep != "/": root = root.replace(os.sep, "/") if not is_zip: modulename = root[prefix_len:].replace("/", ".") else: modulename = root.replace("/", ".") if "__init__.py" in files: modules = {} for f in [f for f in files if f.endswith(".py")]: source = read_fn(root + "/" + f) source = source.replace("\r", "") modules[f[:-3]] = marshal.dumps(source, 1) parent = self.__packages parts = modulename.split(".") for part in parts[:-1]: if part not in parent: parent[part] = [root[:prefix_len] + part, {}, {}] # Parent must have an __init__.py, else it wouldn't be # a package. source = read_fn(root[:prefix_len] + part + \ "/__init__.py") source = source.replace("\r", "") #parent[part][2]["__init__"] = marshal.dumps(source, 0) parent = parent[part][1] parent[parts[-1]] = [root, {}, modules] def __loadModule(self, module): source = inspect.getsource(module) self.__modules[module.__name__] = marshal.dumps(source, 1) class InMemoryImporter: """ Custom "in-memory" importer, which imports preconfigured packages. This is used to load modules without requiring the Python source to exist on disk. Thus, one can load modules on a remote machine without copying the source to the machine. See U{PEP 302} for information on custom importers. """ def __init__(self, packages, modules): # name => [packagedir, subpackages, modules] self.__packages = packages self.__modules = modules def find_module(self, fullname, path=None): parts = fullname.split(".") parent = [None, self.__packages, self.__modules] for part in parts[:-1]: if parent[1].has_key(part): parent = parent[1][part] else: return if parent[1].has_key(parts[-1]): package = parent[1][parts[-1]] filename = package[0] + "/__init__.pyc" code = marshal.loads(package[2]["__init__"]) return InMemoryLoader(fullname, filename, code, path=[]) elif parent[2].has_key(parts[-1]): if parent[0]: filename = "%s/%s.pyc" % (parent[0], parts[-1]) else: filename = parts[-1] + ".pyc" code = marshal.loads(parent[2][parts[-1]]) return InMemoryLoader(fullname, filename, code) class InMemoryLoader: """ Custom "in-memory" package/module loader (used by InMemoryImporter). """ def __init__(self, fullname, filename, code, path=None): self.__fullname = fullname self.__filename = filename self.__code = code self.__path = path def load_module(self, fullname): module = imp.new_module(fullname) module.__file__ = self.__filename if self.__path is not None: module.__path__ = self.__path module.__loader__ = self sys.modules[fullname] = module exec self.__code in module.__dict__ return module def try_set_binary(fd): try: import msvcrt msvcrt.setmode(fd, os.O_BINARY) except ImportError: pass def pushy_server(stdin, stdout): import sys # Reconstitute the package hierarchy delivered from the client (packages, modules) = marshal.load(stdin) # Add the package to the in-memory package importer importer = InMemoryImporter(packages, modules) sys.meta_path.insert(0, importer) import pushy.server pushy.server.serve_stdio_forever(stdin, stdout) ############################################################################### # Auto-import object. # Define a remote function which we'll use to import modules, to avoid lots of # exception passing due to misuse of "getattr" on the AutoImporter. quiet_import = """ def quiet_import(name): try: module = __import__(name) # Traverse down to the target module. if "." in name: for part in name.split(".")[1:]: module = getattr(module, part) return module except ImportError: return None """.strip() class AutoImporter(object): "RPyc-inspired automatic module importer." def __init__(self, client): self.__client = client locals = {} remote_compile = self.__client.eval("compile") code = remote_compile(quiet_import, "", "exec") self.__client.eval(code, locals=locals) self.__import = locals["quiet_import"] def __getattr__(self, name): try: value = self.remote_import(name) # Modify the module proxy to automatically import modules in # the same manner as this auto-importer on a failed getattr. value._ModuleProxy__importer = lambda name: getattr(self, name) return value except: pass return object.__getattribute__(self, name) def remote_import(self, name): return self.__import(name) ############################################################################### # Read the source for the server into a string. If we're the server, we'll # have defined __builtin__.pushy_source (by the "realServerLoaderSource"). if not hasattr(__builtin__, "pushy_source"): if "__loader__" in locals(): serverSource = __loader__.get_source(__name__) serverSource = marshal.dumps(serverSource, 1) else: serverSource = open(inspect.getsourcefile(AutoImporter)).read() serverSource = marshal.dumps(serverSource, 1) else: serverSource = __builtin__.pushy_source md5ServerSource = hashlib.md5(serverSource).digest() # This is the program we run on the command line. It'll read in a # predetermined number of bytes, and execute them as a program. So once we # start the process up, we immediately write the "real" server source to it. realServerLoaderSource = """ import __builtin__, os, marshal, sys try: import hashlib except ImportError: import md5 as hashlib # Back up old stdin/stdout. stdout = os.fdopen(os.dup(sys.stdout.fileno()), "wb", 0) stdin = os.fdopen(os.dup(sys.stdin.fileno()), "rb", 0) try: import msvcrt msvcrt.setmode(stdout.fileno(), os.O_BINARY) msvcrt.setmode(stdin.fileno(), os.O_BINARY) except ImportError: pass sys.stdout.close() sys.stdin.close() serverSourceLength = %d serverSource = stdin.read(serverSourceLength) while len(serverSource) < serverSourceLength: serverSource += stdin.read(serverSourceLength - len(serverSource)) try: assert hashlib.md5(serverSource).digest() == %r __builtin__.pushy_source = serverSource serverCode = marshal.loads(serverSource) exec serverCode pushy_server(stdin, stdout) except: import traceback # Uncomment for debugging # traceback.print_exc(file=open("stderr.txt", "w")) raise """.strip() % (len(serverSource), md5ServerSource) # Avoid putting any quote characters in the program at all costs. serverLoaderSource = \ "exec reduce(lambda a,b: a+b, map(chr, (%s)))" \ % str((",".join(map(str, map(ord, realServerLoaderSource))))) ############################################################################### def get_transport(target): import pushy colon = target.find(":") if colon == -1: raise Exception, "Missing colon from transport address" transport_name = target[:colon] if transport_name not in pushy.transports: raise Exception, "Transport '%s' does not exist in pushy.transport" \ % transport_name transport = pushy.transports[transport_name] if transport is None: transport = __import__("pushy.transport.%s" % transport_name, globals(), locals(), ["pushy.transport"]) pushy.transports[transport_name] = transport return (transport, target[colon+1:]) ############################################################################### logid = 0 class ClientInitException(Exception): pass class PushyClient(object): "Client-side Pushy connection initiator." pushy_packages = None packages_lock = threading.Lock() def __init__(self, target, python="python", **kwargs): (transport, address) = get_transport(target) # Start the server command = [python, "-u", "-c", serverLoaderSource] kwargs["address"] = address self.server = transport.Popen(command, **kwargs) try: if not self.server.daemon: # Write the "real" server source to the remote process self.server.stdin.write(serverSource) self.server.stdin.flush() # Send the packages over to the server packages = self.__load_packages() self.server.stdin.write(marshal.dumps(packages, 1)) self.server.stdin.flush() # Finally... start the connection. Magic! import pushy.protocol remote = pushy.protocol.Connection(self.server.stdout, self.server.stdin) # Start a thread for processing asynchronous requests from the peer self.serve_thread = threading.Thread(target=remote.serve_forever) self.serve_thread.setDaemon(True) self.serve_thread.start() # putfile/getfile self.putfile = getattr(self.server, "putfile", self.__putfile) self.getfile = getattr(self.server, "getfile", self.__getfile) self.remote = remote """The L{connection} for the remote interpreter""" # Create an auto-importer. self.modules = AutoImporter(self) "An instance of L{AutoImporter} for the remote interpreter." rsys = self.modules.sys rsys.stdout = sys.stdout rsys.stderr = sys.stderr # Specify the filesystem interface if hasattr(self.server, "fs"): self.fs = self.server.fs else: self.fs = self.modules.os except: lines = self.server.stderr.readlines() msg = "\n" + "".join([" [remote] " + line for line in lines]) self.server = None self.remote = None self.serve_thread = None raise ClientInitException, msg, sys.exc_info()[2] # With-statement/context-manager support def __enter__(self): return self def __exit__(self, exc_type, exc_val, exc_tb): self.close() return False def __get_gc_enabled(self): return self.remote.gc_enabled def __set_gc_enabled(self, enabled): self.remote.gc_enabled = enabled gc_enabled = property(__get_gc_enabled, __set_gc_enabled) def __get_gc_interval(self): return self.remote.gc_interval def __set_gc_interval(self, interval): self.remote.gc_interval = interval gc_interval = property(__get_gc_interval, __set_gc_interval) def __putfile(self, local, remote): "Transport-independent fallback for putfile." f_read = open(local, "rb") f_write = self.modules.__builtin__.open(remote, "wb") try: d = f_read.read(8192) while len(d) > 0: f_write.write(d) d = f_read.read(8192) finally: f_write.close() f_read.close() def __getfile(self, remote, local): "Transport-independent fallback for getfile." f_read = self.modules.__builtin__.open(remote, "rb") try: f_write = open(local, "wb") try: d = f_read.read(8192) while len(d) > 0: f_write.write(d) d = f_read.read(8192) finally: f_write.close() finally: f_read.close() def __del__(self): try: self.close() except: pass def remote_import(self, name): "Import a remote Python module." return self.modules.remote_import(name) def eval(self, code, globals=None, locals=None): """ Evaluate an expression or code object in the remote Python interpreter. """ return self.remote.eval(code, globals, locals) def compile(self, source, mode="exec"): """ Compiles Python source into a code object, or a local function into a remotely defined function that executes wholly in the remote Python interpreter. """ if inspect.isfunction(source): func = source try: remote_compile = self.eval("compile") # Get and unindent the source. source = inspect.getsourcelines(func)[0] unindent_len = len(source[0]) - len(source[0].lstrip()) source = "".join([l[unindent_len:] for l in source]) code = remote_compile(source, inspect.getfile(func), "exec") locals = {} self.eval(code, locals=locals) # We can't use func_name, because that doesn't apply to # lambdas. Lambdas seem to have their assigned name built-in, # but I'm not sure how to extract it. return locals.values()[0] except IOError: from pushy.util.clone_function import clone_function return self.compile(clone_function)(func) else: return self.eval("compile")(source, "", mode) def execute(self, source, globals=None, locals=None): """ Shortcut for self.eval(self.compile(source), globals, locals). """ self.eval(self.compile(source), globals, locals) def close(self): "Close the connection." if self.remote is not None: self.remote.close() if self.serve_thread is not None: self.serve_thread.join() if self.server is not None: self.server.close() def __load_packages(self): if self.pushy_packages is None: self.packages_lock.acquire() try: import pushy loader = PushyPackageLoader() self.pushy_packages = loader.load(pushy) finally: self.packages_lock.release() return self.pushy_packages def enable_logging(self, client=True, server=False): global logid logid += 1 if client: import pushy.util, logging filename = "pushy-client.%d.log" % logid if os.path.exists(filename): os.remove(filename) handler = logging.FileHandler(filename) handler.setFormatter(logging.Formatter( "[%(process)d:(%(threadName)s:%(thread)d)] %(message)s")) pushy.util.logger.addHandler(handler) pushy.util.logger.setLevel(logging.DEBUG) pushy.util.logger.disabled = False if server: filename = "pushy-server.%d.log" % logid ros = self.modules.os if ros.path.exists(filename): ros.remove(filename) handler = self.modules.logging.FileHandler(filename) handler.setFormatter(self.modules.logging.Formatter( "[%(process)d:(%(threadName)s:%(thread)d)] %(message)s")) self.modules.pushy.util.logger.addHandler(handler) self.modules.pushy.util.logger.setLevel(self.modules.logging.DEBUG) self.modules.pushy.util.logger.disabled = False def connect(target, **kwargs): """ Creates a Pushy connection. e.g. C{pushy.connect("ssh:somewhere.com", username="me", password="...")} @type target: string @param target: A string specifying the target to connect to, in the format I{transport}:I{address}. @param kwargs: Any arguments supported by the transport specified by I{target}. @rtype: L{PushyClient} """ return PushyClient(target, **kwargs)