pax_global_header00006660000000000000000000000064140615040670014515gustar00rootroot0000000000000052 comment=31a7918a3eb2d9b1057a63396ec7ea9d14d2be27 ppft-ppft-1.6.6.4/000077500000000000000000000000001406150406700136115ustar00rootroot00000000000000ppft-ppft-1.6.6.4/.codecov.yml000066400000000000000000000012401406150406700160310ustar00rootroot00000000000000comment: false coverage: status: project: default: # Commits pushed to master should not make the overall # project coverage decrease by more than 1%: target: auto threshold: 1% patch: default: # Be tolerant on slight code coverage diff on PRs to limit # noisy red coverage status on github PRs. # Note The coverage stats are still uploaded # to codecov so that PR reviewers can see uncovered lines # in the github diff if they install the codecov browser # extension: # https://github.com/codecov/browser-extension target: auto threshold: 1% ppft-ppft-1.6.6.4/.codecov.yml.local000066400000000000000000000003751406150406700171320ustar00rootroot00000000000000fixes: - "*/python2.7/site-packages/ppft*/pp/::$PWD/python2/" - "*/python2.7/site-packages/ppft*/ppft/server/::$PWD/server2/" - "*/python3.6/site-packages/ppft*/pp/::$PWD/python3/" - "*/python3.6/site-packages/ppft*/ppft/server/::$PWD/server3/" ppft-ppft-1.6.6.4/.codecov.yml.travis000066400000000000000000000003511406150406700173420ustar00rootroot00000000000000fixes: - "*/python2.7/site-packages/ppft*/pp/::python2/" - "*/python2.7/site-packages/ppft*/ppft/server/::server2/" - "*/python3.6/site-packages/ppft*/pp/::python3/" - "*/python3.6/site-packages/ppft*/ppft/server/::server3/" ppft-ppft-1.6.6.4/.coveragerc000066400000000000000000000007651406150406700157420ustar00rootroot00000000000000[run] # source = */ppft/* include = */ppft/* */ppft/server/* omit = */examples/* */tests/* branch = true # timid = true # parallel = true # concurrency = multiprocessing # thread # data_file = $TRAVIS_BUILD_DIR/.coverage # data_file = ./.coverage # debug = trace [report] include = */ppft/* */ppft/server/* exclude_lines = pragma: no cover raise NotImplementedError if __name__ == .__main__.: # show_missing = true ignore_errors = true # pragma: no branch # noqa ppft-ppft-1.6.6.4/.gitignore000066400000000000000000000000401406150406700155730ustar00rootroot00000000000000.tox/ .cache/ *.egg-info/ *.pyc ppft-ppft-1.6.6.4/.travis.yml000066400000000000000000000022231406150406700157210ustar00rootroot00000000000000dist: focal language: python matrix: include: - python: '2.7' dist: bionic env: - python: '3.6' env: - COVERAGE="true" - python: '3.7' env: - python: '3.8' env: - python: '3.9' env: - python: '3.10-dev' env: - python: 'nightly' env: - python: 'pypy' dist: xenial env: - python: 'pypy3' env: allow_failures: - python: '2.7' - python: 'pypy' - python: '3.10-dev' - python: 'nightly' fast_finish: true cache: pip: true before_install: - set -e # fail on any error - if [[ $COVERAGE == "true" ]]; then pip install coverage; fi install: - python setup.py build && python setup.py install script: - for test in examples/*.py; do if [[ $test != *"__"* ]]; then echo $test ; if [[ $COVERAGE == "true" ]]; then coverage run -a $test > /dev/null; else python $test > /dev/null; fi ; fi ; done after_success: - if [[ $COVERAGE == "true" ]]; then bash <(curl -s https://codecov.io/bash); else echo ''; fi ppft-ppft-1.6.6.4/AUTHORS000066400000000000000000000000571406150406700146630ustar00rootroot00000000000000Vitalii Vanovschi - support@parallelpython.com ppft-ppft-1.6.6.4/CHANGELOG000066400000000000000000000073601406150406700150310ustar00rootroot00000000000000pp-1.6.6: 1) Fixed bug in config parsing. 2) Removed harmless error message that sometimes displayed on Windows. pp-1.6.5: 1) Fixed bug in search path order. pp-1.6.4: 1) Start ppworker using -m 2) Fixed windows compatibility issue. pp-1.6.3: 1) Added -P pid_file command line argument to ppserver.py 2) Modified print_stats() to output the number of active tasks. 3) Added SIGUSR1 handler to ppserver.py to print stats when signal is received. pp-1.6.2: 1) Made socket timeout configurable via constructor argument socket_timeout and command line argument of ppserver.py -k. 2) Fixed sresult referenced before assignment bug 3) Removed shell from subprocess.Popen call. 4) Fixed race condition in autodiscovery. pp-1.6.1: 1) Fixed struct.unpack("!Q", size_packed) bug which started to happen with Python 2.7 on certain platforms. 2) Fixed bug with auto-discovery not working after ppserver is restarted. 3) Added full support of python import statements. For instance "from numpy.linalg import det as determinant" is now supported. For compatibility old module name imports will continue to work. 4) Exposed more detailed network error messages in ppserver.py. pp-1.6.0: 1) Changed logging mechanism. Now logger is obtained as logging.getLogger('pp'). 2) Modified ppworker to use exec instead of eval. 3) Modified exception handling on destruction. Now if server was destroyed, uncompleted jobs throw DestroyedServerError exception on call. 4) Fixed issue with submitting a method of an instance of a class inherited from another. 5) Added timeouts to all socket operations. 6) Changed default proto type to 2. 7) Moved from thread module to threading. Made all pp threads daemons. 8 ) Refactored ppserver.py to improve testability 9) Fixed bug with ppsecret in user module Changes w.r.t RC1: 10) Fixed issue with argument which is an instance of an imported class Changes w.r.t RC2: 11) Fixed DEBUG logging in ppserver. 12) Added a flag (-f) to ppserver to set a custom log format. Changed default log format. 13) Made printing of the expected exceptions optional and improved the way they are handled. 14) Removed default logging handler from pp module (to improve logging flexibility). Changes w.r.t RC3: 15) Created a common module ppcommon.py and moved common functions there. 16) Fixed issue with pipes not being closed. Changes w.r.t. RC4: 17) Fixed issues with ppserver exiting on first connection. 18) Fixed deadlock when using ppworker restart option. 19) Enables support for submodule importing. pp-1.5.7: 1) Added ppworker restart after task completion functionality 2) Added pickle protocol option 3) Merged patch for Python 2.6 compatibility (contributed by mrtss) 4) Merged patch for config file support (contributed by stevemilner) 5) Documentation has been moved to doc folder pp-1.5.6 1) Fixed problem with autodiscovery service on Winsows XP and Vista 2) Merged new code quality improvement patches (contributed by stevemilner) pp-1.5.5 1) Fixed bug which caused segmentation fault when calling destroy() method. 2) Merged performance and quality improvement patches (contributed by stevemilner) pp-1.5.4 1) Fixed bug with unindented comments 2) easy_intall functionality repaired 3) Code quality improved (many small changes) pp-1.5.3 1) Added support for methods of new-style classes. 2) Added ability to read secret key from pp_secret variable of .pythonrc.py 3) ppdoc.html and ppserver.1 are included in the distribution 4) examples bundled with the distribution CHANGELOG started * - nicknames of the contributors refer to the PP forum profile login names. ppft-ppft-1.6.6.4/COPYING000066400000000000000000000030541406150406700146460ustar00rootroot00000000000000Parallel Python Software: http://www.parallelpython.com Copyright (c) 2005-2012, Vitalii Vanovschi All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of the author nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ppft-ppft-1.6.6.4/LICENSE000066400000000000000000000035601406150406700146220ustar00rootroot00000000000000Copyright (c) 2004-2016 California Institute of Technology. Copyright (c) 2016-2021 The Uncertainty Quantification Foundation. All rights reserved. This software forks the python package "pp". Licence and copyright information for pp can be found in "COPYING". This software is available subject to the conditions and terms laid out below. By downloading and using this software you are agreeing to the following conditions. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:: - Redistribution of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - Redistribution in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentations and/or other materials provided with the distribution. - Neither the names of the copyright holders nor the names of any of the contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ppft-ppft-1.6.6.4/MANIFEST.in000066400000000000000000000003351406150406700153500ustar00rootroot00000000000000include COPYING include CHANGELOG include LICENSE include README* include tox.ini recursive-include doc * recursive-include examples * recursive-include pp * recursive-include ppft * include .* prune .git prune .coverage ppft-ppft-1.6.6.4/MODS.diff000066400000000000000000000061051406150406700152070ustar00rootroot00000000000000Only in ppft: MODS.diff Only in ppft: MODS2.diff Only in ppft: MODS3.diff Only in ppft: MODSe.diff Only in pp-1.6.4: README Only in ppft: README.md Only in ppft: README_MODS Common subdirectories: pp-1.6.4/doc and ppft/doc Common subdirectories: pp-1.6.4/examples and ppft/examples Only in pp-1.6.4: pp.py Only in pp-1.6.4: ppauto.py Only in pp-1.6.4: ppcommon.py Only in pp-1.6.4: ppserver.py Only in pp-1.6.4: pptransport.py Only in pp-1.6.4: ppworker.py Only in ppft: python2 Only in ppft: python3 diff pp-1.6.4/MANIFEST.in ppft/MANIFEST.in 6,7c6 < include README < include python-restlib.spec --- > include README.md 15a15,16 > recursive-include python2 * > recursive-include python3 * diff pp-1.6.4/setup.py ppft/setup.py 9,10c9,12 < import os.path < from distutils.core import setup --- > import sys > import os, os.path > pyversion = sys.version_info[0] > pkgdir = 'python%s' % pyversion 12c14,30 < from pp import version as VERSION --- > try: > from setuptools import setup > has_setuptools = True > except ImportError: > from distutils.core import setup > has_setuptools = False > > stable_version = '1.6.4.5' > target_version = '1.6.4.7' > is_release = False > VERSION = stable_version if is_release else target_version + '.dev0' > # os.chdir(pkgdir) > # sys.path.insert(0, '.') > # from ppcommon import __version__ as VERSION > # sys.path.pop(0) > # if os.path.exists('ppcommon.pyc'): os.remove('ppcommon.pyc') > # os.chdir('..') 26,40c44,60 < setup( < name="pp", < url="http://www.parallelpython.com", < version=VERSION, < download_url="http://www.parallelpython.com/downloads/pp/pp-%s.zip" % ( < VERSION), < author="Vitalii Vanovschi", < author_email="support@parallelpython.com", < py_modules=["pp", "ppauto", "ppcommon", "pptransport", "ppworker"], < scripts=["ppserver.py"], < description="Parallel and distributed programming for Python", < platforms=["Windows", "Linux", "Unix"], < long_description=LONG_DESCRIPTION, < license="BSD-like", < classifiers=[ --- > kwds = { > "name" : "ppft", > "url" : "http://www.parallelpython.com", > "version" : VERSION, > "download_url" : "http://dev.danse.us/packages/", > "author" : "Vitalii Vanovschi", > "author_email" : "support@parallelpython.com", > "maintainer" : "Mike McKerns", > "maintainer_email" : "mmckerns@caltech.edu", > "package_dir" : {'': pkgdir}, > "py_modules" : ["pp", "ppauto", "ppcommon", "pptransport", "ppworker"], > "scripts" : ["%s/ppserver.py" % pkgdir], > "description" : "Parallel and distributed programming for Python", > "platforms" : ["Windows", "Linux", "Unix"], > "long_description" : LONG_DESCRIPTION, > "license" : "BSD-like", > "classifiers" : [ 50c70,82 < ) --- > } > > if has_setuptools: > kwds.update({ > "zip_safe" : False, > }) > if has_setuptools and pyversion > 2: > kwds.update({ > "install_requires" : ['six>=1.7.3'], > }) > > setup(**kwds) > ppft-ppft-1.6.6.4/MODS2.diff000066400000000000000000000102241406150406700152660ustar00rootroot00000000000000Only in pp-1.6.4: AUTHORS Only in pp-1.6.4: CHANGELOG Only in pp-1.6.4: COPYING Only in pp-1.6.4: MANIFEST.in Only in pp-1.6.4: PKG-INFO Only in pp-1.6.4: README Only in pp-1.6.4: doc Only in pp-1.6.4: examples diff pp-1.6.4/pp.py ppft/python2/pp.py 42c42,58 < import cPickle as pickle --- > try: > import dill as pickle > from dill.source import importable > from dill.source import getname > from dill.source import _wrap > except ImportError: > import cPickle as pickle > def importable(func): # the original code > #get lines of the source and adjust indent > sourcelines = inspect.getsourcelines(func)[0] > #remove indentation from the first line > sourcelines[0] = sourcelines[0].lstrip() > return "".join(sourcelines) > def getname(obj): # just get __name__ > return obj.__name__ > def _wrap(f): # do nothing > return f 48c64 < version = "1.6.4" --- > __version__ = version = "1.6.4.7" 352c368 < port = Server.default_port --- > port = ppcommon.randomport() 467c483 < (tid, func.func_name)) --- > (tid, getname(func))) 637a654 > # should probably just 'try' above, if fail rely on dill.dumps 639c656 < (funcs[0].func_name, sources, modules), --- > (getname(funcs[0]), sources, modules), 703,707c720 < #get lines of the source and adjust indent < sourcelines = inspect.getsourcelines(func)[0] < #remove indentation from the first line < sourcelines[0] = sourcelines[0].lstrip() < self.__sourcesHM[hashf] = "".join(sourcelines) --- > self.__sourcesHM[hashf] = importable(func) diff pp-1.6.4/ppauto.py ppft/python2/ppauto.py 40c40 < version = "1.6.4" --- > __version__ = version = "1.6.4.7" diff pp-1.6.4/ppcommon.py ppft/python2/ppcommon.py 36c36 < version = "1.6.4" --- > __version__ = version = "1.6.4.7" 62a63,115 > > class portnumber(object): > '''port selector > > Usage: > >>> pick = portnumber(min=1024,max=65535) > >>> print( pick() ) > ''' > > def __init__(self, min=0, max=64*1024): > '''select a port number from a given range. > > The first call will return a random number from the available range, > and each subsequent call will return the next number in the range. > > Inputs: > min -- minimum port number [default = 0] > max -- maximum port number [default = 65536] > ''' > self.min = min > self.max = max > self.first = -1 > self.current = -1 > return > > def __call__(self): > import random > > if self.current < 0: #first call > self.current = random.randint(self.min, self.max) > self.first = self.current > return self.current > else: > self.current += 1 > > if self.current > self.max: > self.current = self.min > if self.current == self.first: > raise RuntimeError( 'Range exhausted' ) > return self.current > return > > > def randomport(min=1024, max=65536): > '''select a random port number > > Inputs: > min -- minimum port number [default = 1024] > max -- maximum port number [default = 65536] > ''' > return portnumber(min, max)() > > diff pp-1.6.4/ppserver.py ppft/python2/ppserver.py 32a33 > from __future__ import with_statement 53c54 < version = "1.6.4" --- > __version__ = version = "1.6.4.7" 84c85 < self.port = self.default_port --- > self.port = ppcommon.randomport() 139c140,141 < except socket.error as e: --- > except socket.error: > e = sys.exc_info()[1] diff pp-1.6.4/pptransport.py ppft/python2/pptransport.py 40c40 < version = "1.6.4" --- > __version__ = version = "1.6.4.7" diff pp-1.6.4/ppworker.py ppft/python2/ppworker.py 35c35,38 < import cPickle as pickle --- > try: > import dill as pickle > except ImportError: > import cPickle as pickle 39c42 < version = "1.6.4" --- > __version__ = version = "1.6.4.7" Only in pp-1.6.4: setup.py ppft-ppft-1.6.6.4/MODS3.diff000066400000000000000000000630521406150406700152760ustar00rootroot00000000000000Only in pp-1.6.4: AUTHORS Only in pp-1.6.4: CHANGELOG Only in pp-1.6.4: COPYING Only in pp-1.6.4: MANIFEST.in Only in pp-1.6.4: PKG-INFO Only in pp-1.6.4: README Only in pp-1.6.4: doc Only in pp-1.6.4: examples diff pp-1.6.4/pp.py ppft/python3/pp.py 41,42c41,63 < import user < import cPickle as pickle --- > try: > import user > except ImportError: > user = types # using as a stub > try: > import dill as pickle > from dill.source import importable > from dill.source import getname > from dill.source import _wrap > except ImportError: > try: import cPickle as pickle > except ImportError: import pickle > def importable(func): # the original code > #get lines of the source and adjust indent > sourcelines = inspect.getsourcelines(func)[0] > #remove indentation from the first line > sourcelines[0] = sourcelines[0].lstrip() > return "".join(sourcelines) > def getname(obj): # just get __name__ > return obj.__name__ > def _wrap(f): # do nothing > return f > import six 45c66 < import ppcommon --- > import ppcommon as ppc 48c69 < version = "1.6.4" --- > __version__ = version = "1.6.4.7" 68a90,91 > > 118c141 < self.result, sout = pickle.loads(self.sresult) --- > self.result, sout = pickle.loads(ppc.b_(self.sresult)) 121c144 < print sout, --- > six.print_(sout, end=' ') 149a173,174 > #open('/tmp/pp.debug', 'a+').write('Starting _Worker\n') > #open('/tmp/pp.debug', 'a+').write('receiving... \n') 150a176,177 > #open('/tmp/pp.debug', 'a+').write('received: %s\n' % self.pid) > #open('/tmp/pp.debug', 'a+').write('sending: %s\n' % self.pickle_proto) 151a179 > #open('/tmp/pp.debug', 'a+').write('...sent \n') 197a226 > #open('/tmp/pp.debug', 'a+').write('connect: %s\n' % repr(message)) 201a231 > #open('/tmp/pp.debug', 'a+').write('connected: %s\n' % self.port) 205a236 > #open('/tmp/pp.debug', 'a+').write('authenticated \n') 207a239 > #open('/tmp/pp.debug', 'a+').write('sent: %s\n' % repr(message)) 352c384 < port = Server.default_port --- > port = ppc.randomport() 364c396 < if not isinstance(secret, types.StringType): --- > if not isinstance(secret, str): 369c401 < if not isinstance(secret, types.StringType): --- > if not isinstance(secret, str): 418c450 < if not isinstance(module, types.StringType): --- > if not isinstance(module, str): 422a455 > other_type = types.FunctionType if six.PY3 else types.ClassType 430c463 < or isinstance(object1, types.ClassType): --- > or isinstance(object1, other_type): 440,441c473,476 < if isinstance(func, types.MethodType) and func.im_self is not None: < args = (func.im_self, ) + args --- > if isinstance(func, types.MethodType): > func_self = func.__self__ if six.PY3 else func.im_self > if func_self is not None: > args = (func_self, ) + args 443c478 < # if there is an instance of a user deined class in the arguments add --- > # if there is an instance of a user defined class in the arguments add 447,448c482,486 < if isinstance(arg, types.InstanceType) \ < or str(type(arg))[:6] == " if (six.PY2 and isinstance(arg, types.InstanceType)) \ > or str(type(arg))[:6] == " # in PY3, all instances are if getattr(inspect.getmodule(arg), '__name__', None) \ > in ['builtins', '__builtin__', None]: pass 450,451c488,489 < if ppcommon.is_not_imported(arg, modules): < depfuncs += tuple(ppcommon.get_class_hierarchy(arg.__class__)) --- > elif ppc.is_not_imported(arg, modules): > depfuncs += tuple(ppc.get_class_hierarchy(arg.__class__)) 467c505 < (tid, func.func_name)) --- > (tid, getname(func))) 534c572 < print "Job execution statistics:" --- > print("Job execution statistics:") 536c574 < statistics = self.get_stats().items() --- > statistics = list(self.get_stats().items()) 540,541c578,579 < print " job count | % of all jobs | job time sum | " \ < "time per job | job server" --- > print(" job count | % of all jobs | job time sum | " \ > "time per job | job server") 544c582 < print " %6i | %6.2f | %8.4f | %11.6f | %s" \ --- > print(" %6i | %6.2f | %8.4f | %11.6f | %s" \ 546,548c584,587 < stat.time/stat.njobs, ppserver, ) < print "Time elapsed since server creation", walltime < print self.__active_tasks, "active tasks,", self.get_ncpus(), "cores" --- > stat.time/stat.njobs, ppserver, )) > print("Time elapsed since server creation %s" % walltime) > print("%s active tasks, %s cores" % (self.__active_tasks, \ > self.get_ncpus())) 551,553c590,592 < print "WARNING: statistics provided above is not accurate" \ < " due to job rescheduling" < print --- > print("WARNING: statistics provided above is not accurate" \ > " due to job rescheduling") > print("") 579a619 > #open('/tmp/pp.debug', 'a+').write('_RWorker(STAT)\n') 581a622 > #open('/tmp/pp.debug', 'a+').write('_RWorker: %s\n' % ncpus) 584a626 > #open('/tmp/pp.debug', 'a+').write('_RWorker(EXEC)\n') 590a633 > #open('/tmp/pp.debug', 'a+').write('_RWorker(EXEC)_\n') 595a639 > #open('/tmp/pp.debug', 'a+').write('_RWorker(sched)\n') 606c650 < ppcommon.start_thread("connect1", self.connect1, ppserver) --- > ppc.start_thread("connect1", self.connect1, ppserver) 610c654 < ppcommon.start_thread("discover.run", self.discover.run, ppserver) --- > ppc.start_thread("discover.run", self.discover.run, ppserver) 637a682 > # should probably just 'try' above, if fail rely on dill.dumps 639c684 < (funcs[0].func_name, sources, modules), --- > (getname(funcs[0]), sources, modules), 672c717 < ppcommon.start_thread("run_local", self._run_local, task+(worker, )) --- > ppc.start_thread("run_local", self._run_local, task+(worker, )) 681c726 < ppcommon.start_thread("run_remote", self._run_remote, task+(rworker, )) --- > ppc.start_thread("run_remote", self._run_remote, task+(rworker, )) 690c735 < ppcommon.start_thread("run_remote", self._run_remote, task+(rworker, )) --- > ppc.start_thread("run_remote", self._run_remote, task+(rworker, )) 703,707c748 < #get lines of the source and adjust indent < sourcelines = inspect.getsourcelines(func)[0] < #remove indentation from the first line < sourcelines[0] = sourcelines[0].lstrip() < self.__sourcesHM[hashf] = "".join(sourcelines) --- > self.__sourcesHM[hashf] = importable(func) 719a761 > #open('/tmp/pp.debug', 'a+').write('_local: %s\n' % repr(sfunc)) 720a763 > #open('/tmp/pp.debug', 'a+').write('_local: %s\n' % repr(sargs)) 721a765 > #open('/tmp/pp.debug', 'a+').write('_local: \n') 722a767 > #open('/tmp/pp.debug', 'a+').write('_local: %s\n' % repr(sresult)) 723a769 > #open('/tmp/pp.debug', 'a+').write('_local: _\n') 748a795 > #open('/tmp/pp.debug', 'a+').write('_remote: %s\n' % repr(sfunc)) 749a797 > #open('/tmp/pp.debug', 'a+').write('_remote: %s\n' % repr(sargs)) 750a799 > #open('/tmp/pp.debug', 'a+').write('_remote: %s\n') 751a801 > #open('/tmp/pp.debug', 'a+').write('_remote: %s\n' % repr(sresult)) 753a804 > #open('/tmp/pp.debug', 'a+').write('_remote: _%s\n') diff pp-1.6.4/ppauto.py ppft/python3/ppauto.py 37c37 < import ppcommon --- > import ppcommon as ppc 40c40 < version = "1.6.4" --- > __version__ = version = "1.6.4.7" 72c72 < self.bsocket.sendto("C", self.broadcast_addr) --- > self.bsocket.sendto(ppc.b_("C"), self.broadcast_addr) 79c79 < self.bsocket.sendto("S", self.broadcast_addr) --- > self.bsocket.sendto(ppc.b_("S"), self.broadcast_addr) 92c92 < ppcommon.start_thread("broadcast", self.broadcast) --- > ppc.start_thread("broadcast", self.broadcast) 98a99 > message = ppc.str_(message) 106c107 < ppcommon.start_thread("ppauto_connect1", self.base.connect1, --- > ppc.start_thread("ppauto_connect1", self.base.connect1, 110c111 < self.bsocket.sendto("S", self.broadcast_addr) --- > self.bsocket.sendto(ppc.b_("S"), self.broadcast_addr) diff pp-1.6.4/ppcommon.py ppft/python3/ppcommon.py 33a34,62 > try: # workaround for install process > import six > except ImportError: > import types > import sys > six = types.ModuleType('six') > six.PY3 = sys.version_info[0] == 3 > six.b = lambda x:x > if six.PY3: > long = int > import io > file = io.IOBase > def str_(byte): # convert to unicode > if not hasattr(byte, 'decode'): return byte > try: > return byte.decode('ascii') > except UnicodeDecodeError: # non-ascii needs special handling > return repr([i for i in byte])+'{B}' > def b_(string): > if not hasattr(string, 'encode'): return string > if not string.endswith(']{B}'): return six.b(string) > return bytes(eval(string[:-3])) # special handling for non-ascii > else: > long = long > file = file > def str_(string): # is already str > return string > def b_(string): > return six.b(string) 36c65 < version = "1.6.4" --- > __version__ = version = "1.6.4.7" 62a92,144 > > class portnumber(object): > '''port selector > > Usage: > >>> pick = portnumber(min=1024,max=65535) > >>> print( pick() ) > ''' > > def __init__(self, min=0, max=64*1024): > '''select a port number from a given range. > > The first call will return a random number from the available range, > and each subsequent call will return the next number in the range. > > Inputs: > min -- minimum port number [default = 0] > max -- maximum port number [default = 65536] > ''' > self.min = min > self.max = max > self.first = -1 > self.current = -1 > return > > def __call__(self): > import random > > if self.current < 0: #first call > self.current = random.randint(self.min, self.max) > self.first = self.current > return self.current > else: > self.current += 1 > > if self.current > self.max: > self.current = self.min > if self.current == self.first: > raise RuntimeError( 'Range exhausted' ) > return self.current > return > > > def randomport(min=1024, max=65536): > '''select a random port number > > Inputs: > min -- minimum port number [default = 1024] > max -- maximum port number [default = 65536] > ''' > return portnumber(min, max)() > > diff pp-1.6.4/ppserver.py ppft/python3/ppserver.py 32a33 > from __future__ import with_statement 45a47 > import six 49c51 < import ppcommon --- > import ppcommon as ppc 53c55 < version = "1.6.4" --- > __version__ = version = "1.6.4.7" 77c79 < print >>pfile, os.getpid() --- > six.print_(os.getpid(), file=pfile) 84c86 < self.port = self.default_port --- > self.port = ppc.randomport() 95c97 < ppcommon.start_thread("timeout_check", self.check_timeout) --- > ppc.start_thread("timeout_check", self.check_timeout) 127c129,130 < except socket.error, e: --- > except socket.error: > e = sys.exc_info()[1] 139c142,143 < except socket.error as e: --- > except socket.error: > e = sys.exc_info()[1] 147c151 < ppcommon.start_thread("client_socket", self.crun, (csocket, )) --- > ppc.start_thread("client_socket", self.crun, (csocket, )) 163c167 < for i in xrange(16)]) --- > for i in range(16)]) 165,166c169,170 < answer = sha_new(srandom+self.secret).hexdigest() < clientanswer = mysocket.receive() --- > answer = sha_new(ppc.b_(srandom+self.secret)).hexdigest() > clientanswer = ppc.str_(mysocket.receive()) 176c180 < ctype = mysocket.receive() --- > ctype = ppc.str_(mysocket.receive()) 182a187 > #open('/tmp/pp.debug', 'a+').write('STAT: \n') 183a189 > #open('/tmp/pp.debug', 'a+').write('STAT: get_ncpus\n') 185a192 > #open('/tmp/pp.debug', 'a+').write('STAT: recvd\n') 186a194 > #open('/tmp/pp.debug', 'a+').write('STAT: _\n') 188a197 > #open('/tmp/pp.debug', 'a+').write('EXEC: \n') 189a199 > #open('/tmp/pp.debug', 'a+').write('EXEC: '+repr((sfunc,))+'\n') 190a201 > #open('/tmp/pp.debug', 'a+').write('EXEC: '+repr((sargs,))+'\n') 192a204 > #open('/tmp/pp.debug', 'a+').write('EXEC: '+repr((sresult,))+'\n') 193a206 > #open('/tmp/pp.debug', 'a+').write('EXEC: _\n') 206c219 < ppcommon.start_thread("server_broadcast", discover.run, --- > ppc.start_thread("server_broadcast", discover.run, 217,219c230,234 < except ImportError, ie: < print >> sys.stderr, ("ERROR: You must have config obj installed to use" < "configuration files. You can still use command line switches.") --- > except ImportError: > ie = sys.exc_info()[1] > #sysstderr = getattr(sys.stderr, 'buffer', sys.stderr) > six.print_(("ERROR: You must have config obj installed to use" > "configuration files. You can still use command line switches."), file=sys.stderr) 223c238 < print >> sys.stderr, "ERROR: Can not access %s." % arg --- > six.print_("ERROR: Can not access %s." % arg, file=sys.stderr) 296,297c311,312 < print "Parallel Python Network Server (pp-" + version + ")" < print "Usage: ppserver.py [-hdar] [-f format] [-n proto]"\ --- > print("Parallel Python Network Server (pp-" + version + ")") > print("Usage: ppserver.py [-hdar] [-f format] [-n proto]"\ 300,328c315,343 < " [-k seconds] [-P pid_file]" < print < print "Options: " < print "-h : this help message" < print "-d : set log level to debug" < print "-f format : log format" < print "-a : enable auto-discovery service" < print "-r : restart worker process after each"\ < " task completion" < print "-n proto : protocol number for pickle module" < print "-c path : path to config file" < print "-i interface : interface to listen" < print "-b broadcast : broadcast address for auto-discovery service" < print "-p port : port to listen" < print "-w nworkers : number of workers to start" < print "-s secret : secret for authentication" < print "-t seconds : timeout to exit if no connections with "\ < "clients exist" < print "-k seconds : socket timeout in seconds" < print "-P pid_file : file to write PID to" < print < print "To print server stats send SIGUSR1 to its main process (unix only). " < print < print "Due to the security concerns always use a non-trivial secret key." < print "Secret key set by -s switch will override secret key assigned by" < print "pp_secret variable in .pythonrc.py" < print < print "Please visit http://www.parallelpython.com for extended up-to-date" < print "documentation, examples and support forums" --- > " [-k seconds] [-P pid_file]") > print("") > print("Options: ") > print("-h : this help message") > print("-d : set log level to debug") > print("-f format : log format") > print("-a : enable auto-discovery service") > print("-r : restart worker process after each"\ > " task completion") > print("-n proto : protocol number for pickle module") > print("-c path : path to config file") > print("-i interface : interface to listen") > print("-b broadcast : broadcast address for auto-discovery service") > print("-p port : port to listen") > print("-w nworkers : number of workers to start") > print("-s secret : secret for authentication") > print("-t seconds : timeout to exit if no connections with "\ > "clients exist") > print("-k seconds : socket timeout in seconds") > print("-P pid_file : file to write PID to") > print("") > print("To print server stats send SIGUSR1 to its main process (unix only). ") > print("") > print("Due to the security concerns always use a non-trivial secret key.") > print("Secret key set by -s switch will override secret key assigned by") > print("pp_secret variable in .pythonrc.py") > print("") > print("Please visit http://www.parallelpython.com for extended up-to-date") > print("documentation, examples and support forums") diff pp-1.6.4/pptransport.py ppft/python3/pptransport.py 37c37 < import types --- > import six 40c40 < version = "1.6.4" --- > __version__ = version = "1.6.4.7" 43c43 < # compartibility with Python 2.6 --- > # compatibility with Python 2.6 53a54 > import ppcommon as ppc 66c67 < remote_version = self.receive() --- > remote_version = ppc.str_(self.receive()) 72c73,74 < srandom = self.receive() --- > srandom = ppc.b_(self.receive()) > secret = ppc.b_(secret) 75,76c77,78 < response = self.receive() < if response == "OK": --- > response = ppc.b_(self.receive()) > if response == ppc.b_("OK"): 94c96 < return md5_new(msg).hexdigest() --- > return md5_new(ppc.b_(msg)).hexdigest() 96a99,103 > #if hasattr(self, 'w'): > # open('/tmp/pp.debug', 'a+').write(repr(('cs', self.w, msg))+'\n') > #else: > # open('/tmp/pp.debug', 'a+').write(repr(('cs', self.socket, msg))+'\n') > msg = ppc.b_(msg) 99c106 < self.send("H" + hash1) --- > self.send(ppc.b_("H" + hash1)) 101c108 < self.send("N" + msg) --- > self.send(ppc.b_("N") + msg) 106,107c113,119 < if msg[0] == 'H': < hash1 = msg[1:] --- > #if hasattr(self, 'r'): > # open('/tmp/pp.debug', 'a+').write(repr(('cr', self.r, msg))+'\n') > #else: > # open('/tmp/pp.debug', 'a+').write(repr(('cr', self.socket, msg))+'\n') > msg = ppc.b_(msg) > if msg[:1] == ppc.b_('H'): > hash1 = ppc.str_(msg[1:]) 111c123,124 < self.rcache[hash1] = map(preprocess, (msg, ))[0] --- > if preprocess is None: preprocess = lambda x:x > self.rcache[hash1] = tuple(map(preprocess, (msg, )))[0] 117a131 > #open('/tmp/pp.debug', 'a+').write(repr((r,w))+'\n') 120c134 < if isinstance(r, types.FileType) and isinstance(w, types.FileType): --- > if isinstance(r, ppc.file) and isinstance(w, ppc.file): 125a140,152 > if six.PY3 and hasattr(self.w, 'buffer'): > self.wb = self.w.buffer > self.has_wb = True > else: > self.wb = self.w > self.has_wb = False > if six.PY3 and hasattr(self.r, 'buffer'): > self.rb = self.r.buffer > self.has_rb = True > else: > self.rb = self.r > self.has_rb = False > 128,130c155,165 < self.w.write(struct.pack("!Q", len(msg))) < self.w.flush() < self.w.write(msg) --- > #l = len(ppc.b_(msg)) if (self.has_wb or self.w.mode == 'wb') else len(ppc.str_(msg)) > #open('/tmp/pp.debug', 'a+').write(repr(('s', l, self.w, msg))+'\n') > if self.has_wb or self.w.mode == 'wb': > msg = ppc.b_(msg) > self.wb.write(struct.pack("!Q", len(msg))) > self.w.flush() > else: #HACK: following may be > 8 bytes, needed for len(msg) >= 256 > msg = ppc.str_(msg) > self.wb.write(ppc.str_(struct.pack("!Q", len(msg)))) > self.w.flush() > self.wb.write(msg) 134c169,170 < e_size = struct.calcsize("!Q") --- > e_size = struct.calcsize("!Q") # 8 > c_size = struct.calcsize("!c") # 1 136c172,173 < data = "" --- > stub = ppc.b_("") if (self.has_rb or self.r.mode == 'rb') else "" > data = stub 138,139c175,178 < msg = self.r.read(e_size-r_size) < if msg == "": --- > msg = self.rb.read(e_size-r_size) > #l = len(msg) > #open('/tmp/pp.debug', 'a+').write(repr(('_r', l, self.r, msg))+'\n') > if msg == stub: 140a180,182 > if stub == "" and msg.startswith('['): #HACK to get str_ length > while not msg.endswith('{B}'): > msg += self.rb.read(c_size) 143c185 < e_size = struct.unpack("!Q", data)[0] --- > e_size = struct.unpack("!Q", ppc.b_(data))[0] # get size of msg 146c188 < data = "" --- > data = stub 148,149c190,193 < msg = self.r.read(e_size-r_size) < if msg == "": --- > msg = self.rb.read(e_size-r_size) > #l = len(msg) > #open('/tmp/pp.debug', 'a+').write(repr(('r_', l, self.r, msg))+'\n') > if msg == stub: 152a197 > data = ppc.b_(data) 154c199,200 < return map(preprocess, (data, ))[0] --- > if preprocess is None: preprocess = lambda x:x > return tuple(map(preprocess, (data, )))[0] 171a218,220 > #l = len(ppc.b_(data)) > #open('/tmp/pp.debug', 'a+').write(repr(('ss', l, self.socket, data))+'\n') > data = ppc.b_(data) 174c223 < s_size = 0L --- > s_size = ppc.long(0) 182c231 < s_size = 0L --- > s_size = ppc.long(0) 192c241,242 < data = "" --- > stub = ppc.b_("") > data = stub 195c245,247 < if msg == "": --- > #l = len(msg) > #open('/tmp/pp.debug', 'a+').write(repr(('_sr', l, self.socket, msg))+'\n') > if msg == stub: 199c251 < e_size = struct.unpack("!Q", data)[0] --- > e_size = struct.unpack("!Q", ppc.b_(data))[0] # get size of msg 202c254 < data = "" --- > data = stub 205c257,259 < if msg == "": --- > #l = len(msg) > #open('/tmp/pp.debug', 'a+').write(repr(('sr_', l, self.socket, msg))+'\n') > if msg == stub: 208a263 > data = ppc.b_(data) diff pp-1.6.4/ppworker.py ppft/python3/ppworker.py 34,35c34,46 < import StringIO < import cPickle as pickle --- > try: > import io > ioStringIO = io.StringIO > #ioStringIO = io.BytesIO > except ImportError: > import StringIO as io > ioStringIO = io.StringIO > try: > import dill as pickle > except ImportError: > try: import cPickle as pickle > except ImportError: import pickle > import six 36a48 > import ppcommon as ppc 39c51 < version = "1.6.4" --- > __version__ = version = "1.6.4.7" 43c55 < fname, fsources, imports = pickle.loads(msg) --- > fname, fsources, imports = pickle.loads(ppc.b_(msg)) 49c61 < exec module --- > six.exec_(module) 52c64 < print "An error has occured during the module import" --- > print("An error has occured during the module import") 61c73 < self.sout = StringIO.StringIO() --- > self.sout = ioStringIO() 65a78,79 > #open('/tmp/pp.debug', 'a+').write('Starting _WorkerProcess\n') > #open('/tmp/pp.debug', 'a+').write('send... \n') 66a81,82 > #open('/tmp/pp.debug', 'a+').write('send: %s\n' % str(os.getpid())) > #open('/tmp/pp.debug', 'a+').write('receive... \n') 67a84 > #open('/tmp/pp.debug', 'a+').write('receive: %s\n' % self.pickle_proto) 79c96 < exec __fobj --- > six.exec_(__fobj) 82,83c99,100 < print "An error has occured during the " + \ < "function import" --- > print("An error has occured during the " + \ > "function import") 86c103 < __args = pickle.loads(__sargs) --- > __args = pickle.loads(ppc.b_(__sargs)) 88c105 < __f = locals()[__fname] --- > __f = locals()[ppc.str_(__fname)] 92c109 < print "An error has occured during the function execution" --- > print("An error has occured during the function execution") 102c119 < print "A fatal error has occured during the function execution" --- > print("A fatal error has occured during the function execution") Only in pp-1.6.4: setup.py ppft-ppft-1.6.6.4/MODSe.diff000066400000000000000000000144231406150406700153560ustar00rootroot00000000000000diff pp-1.6.4/examples/auto_diff.py ppft/examples/auto_diff.py 15d14 < 54a54 > __truediv__ = __div__ # PYTHON 3 57a58 > __rtruediv__ = __rdiv__ # PYTHON 3 88c89 < return sum([float(i%2 and 1 or -1)*x**i/i for i in xrange(1, self.n)]) --- > return sum([float(i%2 and 1 or -1)*x**i/i for i in range(1, self.n)]) 91c92 < print """Usage: python auto_diff.py [ncpus] --- > print("""Usage: python auto_diff.py [ncpus] 94c95 < """ --- > """) 109c110 < print "Starting pp with", job_server.get_ncpus(), "workers" --- > print("Starting pp with %s workers" % job_server.get_ncpus()) 124c125 < print "t_log(%lf) = %lf, t_log'(%lf) = %lf" % (x, val.x, x, val.dx) --- > print("t_log(%lf) = %lf, t_log'(%lf) = %lf" % (x, val.x, x, val.dx)) diff pp-1.6.4/examples/callback.py ppft/examples/callback.py 12c12,15 < import thread --- > try: > import _thread as thread > except ImportError: > import thread 38c41 < for x in xrange(start, end): --- > for x in range(start, end): 46c49 < print """Usage: python callback.py [ncpus] --- > print("""Usage: python callback.py [ncpus] 49c52 < """ --- > """) 56c59 < step = (end - start) / parts + 1 --- > step = int((end - start) / parts + 1) 71c74 < print "Starting pp with", job_server.get_ncpus(), "workers" --- > print("Starting pp with %s workers" % job_server.get_ncpus()) 80c83 < for index in xrange(parts): --- > for index in range(parts): 93c96 < print "Partial sum is", sum.value, "| diff =", math.log(2) - sum.value --- > print("Partial sum is %s | diff = %s" % (sum.value, math.log(2) - sum.value)) diff pp-1.6.4/examples/dynamic_ncpus.py ppft/examples/dynamic_ncpus.py 19c19 < for x in xrange(start, end): --- > for x in range(start, end): 26,27c26,27 < print """Usage: python dynamic_ncpus.py""" < print --- > print("""Usage: python dynamic_ncpus.py""") > print("") 34c34 < step = (end - start) / parts + 1 --- > step = int((end - start) / parts + 1) 45,46c45,46 < print "Starting ", job_server.get_ncpus(), " workers" < for index in xrange(parts): --- > print("Starting pp with %s workers" % job_server.get_ncpus()) > for index in range(parts): 60c60 < print "Partial sum is", part_sum1, "| diff =", math.log(2) - part_sum1 --- > print("Partial sum is %s | diff = %s" % (part_sum1, math.log(2) - part_sum1)) 62,63c62,63 < print "Time elapsed: ", time.time() - start_time, "s" < print --- > print("Time elapsed: %s s" % (time.time() - start_time)) > print("") diff pp-1.6.4/examples/quicksort.py ppft/examples/quicksort.py 10c10,15 < --- > try: > callable(min) > except NameError: > def callable(x): > import collections > return isinstance(x, collections.Callable) 22c27 < print """Usage: python quicksort.py [ncpus] --- > print("""Usage: python quicksort.py [ncpus] 25c30 < """ --- > """) 40c45 < print "Starting pp with", job_server.get_ncpus(), "workers" --- > print("Starting pp with %s workers" % job_server.get_ncpus()) 44c49 < for i in xrange(n): --- > for i in range(n): 65c70 < print "first 30 numbers in increasing order:", output[:30] --- > print("first 30 numbers in increasing order: %s" % output[:30]) diff pp-1.6.4/examples/reverse_md5.py ppft/examples/reverse_md5.py 10c10 < import md5 --- > import hashlib 11a12,17 > if sys.version_info[0] >= 3: > def b_(x): > return x.encode('UTF-8') > else: > def b_(x): > return x 15c21 < """Calculates md5 of the integerss between 'start' and 'end' and --- > """Calculates md5 of the integers between 'start' and 'end' and 17,18c23,24 < for x in xrange(start, end): < if md5.new(str(x)).hexdigest() == hash: --- > for x in range(start, end): > if hashlib.md5(b_(str(x))).hexdigest() == hash: 22c28 < print """Usage: python reverse_md5.py [ncpus] --- > print("""Usage: python reverse_md5.py [ncpus] 25c31 < """ --- > """) 40c46 < print "Starting pp with", job_server.get_ncpus(), "workers" --- > print("Starting pp with %s workers" % job_server.get_ncpus()) 43,44c49,50 < hash = md5.new("1829182").hexdigest() < print "hash =", hash --- > hash = hashlib.md5(b_("1829182")).hexdigest() > print("hash = %s" % hash) 54c60 < step = (end - start) / parts + 1 --- > step = int((end - start) / parts + 1) 57c63 < for index in xrange(parts): --- > for index in range(parts): 65,66c71,72 < # ("md5",) - tuple with module names which must be imported before < # md5test execution --- > # ("hashlib", "_hashlib") - tuple with module names which must be > # imported before md5test execution 70c76 < (), ("md5", ))) --- > (), ("hashlib", "_hashlib"))) 80c86 < print "Reverse md5 for", hash, "is", result --- > print("Reverse md5 for %s is %s" % (hash, result)) 82c88 < print "Reverse md5 for", hash, "has not been found" --- > print("Reverse md5 for %s has not been found" % hash) diff pp-1.6.4/examples/sum_primes.py ppft/examples/sum_primes.py 32c32 < return sum([x for x in xrange(2, n) if isprime(x)]) --- > return sum([x for x in range(2, n) if isprime(x)]) 35c35 < print """Usage: python sum_primes.py [ncpus] --- > print("""Usage: python sum_primes.py [ncpus] 37c37 < if omitted it will be set to the number of processors in the system""" --- > if omitted it will be set to the number of processors in the system""") 51c51 < print "Starting pp with", job_server.get_ncpus(), "workers" --- > print("Starting pp with %s workers" % job_server.get_ncpus()) 68c68 < print "Sum of primes below 100 is", result --- > print("Sum of primes below 100 is %s" % result) 77c77 < print "Sum of primes below", input, "is", job() --- > print("Sum of primes below %s is %s" % (input, job())) diff pp-1.6.4/examples/sum_primes_functor.py ppft/examples/sum_primes_functor.py 31c31 < return sum([x for x in xrange(2,n) if isprime(x)]) --- > return sum([x for x in range(2,n) if isprime(x)]) 34c34 < print """Usage: python sum_primes.py [ncpus] --- > print("""Usage: python sum_primes.py [ncpus] 37c37 < """ --- > """) 52c52 < print "Starting pp with", job_server.get_ncpus(), "workers" --- > print("Starting pp with %s workers" % job_server.get_ncpus()) 74c74 < print "Sum of primes below 100 is", result --- > print("Sum of primes below 100 is %s" % result) 81c81 < print "Sum of primes below", input, "is", job() --- > print("Sum of primes below %s is %s" % (input, job())) ppft-ppft-1.6.6.4/README.md000066400000000000000000000067251406150406700151020ustar00rootroot00000000000000ppft ==== distributed and parallel python About Ppft ---------- ``ppft`` is a fork of Parallel Python, and is developed as part of ``pathos``: https://github.com/uqfoundation/pathos Parallel Python module (``pp``) provides an easy and efficient way to create parallel-enabled applications for SMP computers and clusters. ``pp`` module features cross-platform portability and dynamic load balancing. Thus application written with ``pp`` will parallelize efficiently even on heterogeneous and multi-platform clusters (including clusters running other application with variable CPU loads). Visit http://www.parallelpython.com for further information. ``ppft`` is part of ``pathos``, a python framework for heterogeneous computing. ``ppft`` is in active development, so any user feedback, bug reports, comments, or suggestions are highly appreciated. A list of issues is located at https://github.com/uqfoundation/ppft/issues, with a legacy list maintained at https://uqfoundation.github.io/project/pathos/query. NOTE: ``ppft`` installs as ``pp``. If ``pp`` is installed, it should be uninstalled before ``ppft`` is installed -- otherwise, ``import pp`` may not find the ``ppft`` fork. Major Changes ------------- * ``pip`` and ``setuptools`` support * support for python 3 * enhanced serialization, using ``dill.source`` Current Release --------------- This version is a fork of ``pp-1.6.4``. The latest released version of ``ppft`` is available from:: https://pypi.org/project/ppft ``pp`` and ``ppft`` are distributed under a BSD-like license. Development Version [![Travis Build Status](https://img.shields.io/travis/uqfoundation/ppft.svg?label=build&logo=travis&branch=master)](https://travis-ci.org/uqfoundation/ppft) [![codecov](https://codecov.io/gh/uqfoundation/ppft/branch/master/graph/badge.svg)](https://codecov.io/gh/uqfoundation/ppft) [![Downloads](https://pepy.tech/badge/ppft)](https://pepy.tech/project/ppft) ------------------- You can get the latest development version with all the shiny new features at:: https://github.com/uqfoundation If you have a new contribution, please submit a pull request. More Information ---------------- Probably the best way to get started is to look at the set of example scripts in ``ppft.examples``. You can run the test suite with ``python -m ppft.tests``. ``ppft`` will create and execute jobs on local workers (automatically created using ``python -u -m ppft``). Additionally, remote servers can be created with ``ppserver`` (or ``python -m ppft.server``), and then jobs can be distributed to remote workers. See ``--help`` for more details on how to configure a server. Please feel free to submit a ticket on github, or ask a question on stackoverflow (**@Mike McKerns**). If you would like to share how you use ``ppft`` in your work, please send an email (to **mmckerns at uqfoundation dot org**). Citation -------- If you use ``ppft`` to do research that leads to publication, we ask that you acknowledge use of ``ppft`` by citing the following in your publication:: M.M. McKerns, L. Strand, T. Sullivan, A. Fang, M.A.G. Aivazis, "Building a framework for predictive science", Proceedings of the 10th Python in Science Conference, 2011; http://arxiv.org/pdf/1202.1056 Michael McKerns and Michael Aivazis, "pathos: a framework for heterogeneous computing", 2010- ; https://uqfoundation.github.io/project/pathos Please see https://uqfoundation.github.io/project/pathos or http://arxiv.org/pdf/1202.1056 for further information. ppft-ppft-1.6.6.4/README_MODS000066400000000000000000000062021406150406700153130ustar00rootroot00000000000000### VERSION 1.6.6.3 ### # see MODS*.diff for full patch (against 1.6.6) # made these functional changes - ...TBD... # made these "build" changes - edited pp*py to have version = '1.6.6.3' - ...TBD... ### VERSION 1.6.6.2 ### # see MODS*.diff for full patch (against 1.6.6) # made these functional changes - ...TBD... # made these "build" changes - edited pp*py to have version = '1.6.6.2' - ...TBD... ### VERSION 1.6.6.1 ### # see MODS*.diff for full patch (against 1.6.6) # made these functional changes - updated 1.6.4 to 1.6.6 - ...TBD... # made these "build" changes - edited pp*py to have version = '1.6.6.1' - ...TBD... ### VERSION 1.6.4.9 ### # see MODS*.diff for full patch (against 1.6.4) # made these functional changes - ...TBD... # made these "build" changes - edited pp*py to have version = '1.6.4.9' - ...TBD... ### VERSION 1.6.4.8 ### # see MODS*.diff for full patch (against 1.6.4) # made these functional changes - ...TBD... # made these "build" changes - edited pp*py to have version = '1.6.4.8' - ...TBD... ### VERSION 1.6.4.7 ### # see MODS*.diff for full patch (against 1.6.4) # made these functional changes - ...TBD... # made these "build" changes - edited pp*py to have version = '1.6.4.7' - ...TBD... ### VERSION 1.6.4.6 ### # see MODS*.diff for full patch (against 1.6.4) # made these functional changes - ...TBD... # made these "build" changes - edited pp*py to have version = '1.6.4.6' - ...TBD... ### VERSION 1.6.4.5 ### # see MODS*.diff for full patch (against 1.6.4) # made these "build" changes - edited pp*py to have version = '1.6.4.5' - redistribute package as 'ppft' ### VERSION 1.6.4.4 ### # see MODS*.diff for full patch (against 1.6.4) # made these functional changes - every file has __version__ # made these "build" changes - edited pp*py to have version = '1.6.4.4' - python2-only code in python2 (ported from 1.6.4.1) - python2/python3 code in python3 (split, due to minor functional bugs) ### VERSION 1.6.4.3 ### # see MODS.diff for full patch (against 1.6.4) # made these "build" changes - edited pp*py to have version = '1.6.4.3' - build with setuptools if available ### VERSION 1.6.4.2 ### # see MODS.diff for full patch (against 1.6.4) # made these functional changes - picks a random port instead of default port '60000' # made these "build" changes - edited pp*py to have version = '1.6.4.2' - added dependency 'six' (for python3.x support) - added python 3.x support ### VERSION 1.6.4.1 ### # see MODS.diff for full patch (against 1.6.4) # made these functional changes - provide dill as alternate install to cPickle in pp.py and ppworker.py . (this allows pp to unpickle with dill) - provide dill.source.importable as alternate to code using inspect in pp.py . (this allows pp to pickle interactively-defined functions and lambdas) - modify all references in pp.py to dill.source.getname instead of func_name . (this allows pp to interactively pickle builtin and imported functions) # made these "build" changes - edited pp*py to have version = '1.6.4.1' - edited setup.py to use 'pathos' download_url & version number - edited ppserver.py to be python2.5 compliant ppft-ppft-1.6.6.4/doc/000077500000000000000000000000001406150406700143565ustar00rootroot00000000000000ppft-ppft-1.6.6.4/doc/example.config000066400000000000000000000003051406150406700171760ustar00rootroot00000000000000[general] debug = True workers = 2 secret = epo20pdosl;dksldkmm proto = 0 restart = False [network] autodiscovery = False interface = 0.0.0.0 broadcast = 255.255.255.255 port = 60000 timeout = 10 ppft-ppft-1.6.6.4/doc/ppdoc.html000066400000000000000000001036161406150406700163600ustar00rootroot00000000000000 Parallel Python - Parallel Python documentation
Parallel Python
Nutrition facts and analysis
Parallel Python documentation PDF Print E-mail

 Module API
 Quick start guide, SMP
 Quick start guide, clusters
 Quick start guide, clusters with auto-discovery
 Advanced guide, clusters
 Command line arguments, ppserver.py
 Security and secret key
 PP FAQ

 

  pp 1.6.3 module API

 
class Server
    Parallel Python SMP execution server class
 
  Methods defined here:
__init__(self, ncpus='autodetect', ppservers=(), secret=None, restart=False, proto=2, socket_timeout=3600)
Creates Server instance
 
 
 
ncpus - the number of worker processes to start on the local 
        computer, if parameter is omitted it will be set to 
        the number of processors in the system
ppservers - list of active parallel python execution servers 
        to connect with
secret - passphrase for network connections, if omitted a default
        passphrase will be used. It's highly recommended to use a 
        custom passphrase for all network connections.
restart - whether to restart worker process after each task completion 
proto - protocol number for pickle module
socket_timeout - socket timeout in seconds which is also the maximum
        time a remote job could be executed. Increase this value
        if you have long running jobs or decrease if connectivity
        to remote ppservers is often lost.
 
With ncpus = 1 all tasks are executed consequently
For the best performance either use the default "autodetect" value
or set ncpus to the total number of processors in the system
destroy(self)
Kills ppworkers and closes open files
get_active_nodes(self)
Returns active nodes as a dictionary 
[keys - nodes, values - ncpus]
get_ncpus(self)
Returns the number of local worker processes (ppworkers)
get_stats(self)
Returns job execution statistics as a dictionary
print_stats(self)
Prints job execution statistics. Useful for benchmarking on 
clusters
set_ncpus(self, ncpus='autodetect')
Sets the number of local worker processes (ppworkers)
 
ncpus - the number of worker processes, if parammeter is omitted
        it will be set to the number of processors in the system
submit(self, func, args=(), depfuncs=(), modules=(), callback=None, callbackargs=(), group='default', globals=None)
Submits function to the execution queue
 
func - function to be executed
args - tuple with arguments of the 'func'
depfuncs - tuple with functions which might be called from 'func'
modules - tuple with module names to import
callback - callback function which will be called with argument 
        list equal to callbackargs+(result,) 
        as soon as calculation is done
callbackargs - additional arguments for callback function
group - job group, is used when wait(group) is called to wait for
jobs in a given group to finish
globals - dictionary from which all modules, functions and classes
will be imported, for instance: globals=globals()
wait(self, group=None)
Waits for all jobs in a given group to finish.
If group is omitted waits for all jobs to finish
default_port = 60000
default_secret = 'epo20pdosl;dksldkmm'

 
class Template
    Template class
 
  Methods defined here:
__init__(self, job_server, func, depfuncs=(), modules=(), callback=None, callbackargs=(), group='default', globals=None)
Creates Template instance
 
jobs_server - pp server for submitting jobs
func - function to be executed
depfuncs - tuple with functions which might be called from 'func'
modules - tuple with module names to import
callback - callback function which will be called with argument 
        list equal to callbackargs+(result,) 
        as soon as calculation is done
callbackargs - additional arguments for callback function
group - job group, is used when wait(group) is called to wait for
jobs in a given group to finish
globals - dictionary from which all modules, functions and classes
will be imported, for instance: globals=globals()
submit(self, *args)
Submits function with *arg arguments to the execution queue

 
Data
        copyright = 'Copyright (c) 2005-2012 Vitalii Vanovschi. All rights reserved'
version = '1.6.3'

  Quick start guide, SMP

1) Import pp module:

    import pp

2) Start pp execution server with the number of workers set to the number of processors in the system

    job_server = pp.Server() 

3) Submit all the tasks for parallel execution:

    f1 = job_server.submit(func1, args1, depfuncs1, modules1)

    f2 = job_server.submit(func1, args2, depfuncs1, modules1)

    f3 = job_server.submit(func2, args3, depfuncs2, modules2)

   ...etc...

4) Retrieve the results as needed:

    r1 = f1()

    r2 = f2()

    r3 = f3() 

    ...etc...

 To find out how to achieve efficient parallelization with pp please take a look at examples

  Quick start guide, clusters 

On the nodes

1) Start parallel python execution server on all your remote computational nodes:

    node-1> ./ppserver.py

    node-2> ./ppserver.py

    node-3> ./ppserver.py

On the client

2) Import pp module:

    import pp

3)  Create a list of all the nodes in your cluster (computers where you've run ppserver.py)

    ppservers=("node-1", "node-2", "node-3")

4) Start pp execution server with the number of workers set to the number of processors in the system and list of ppservers to connect with :

    job_server = pp.Server(ppservers=ppservers) 

5) Submit all the tasks for parallel execution:

    f1 = job_server.submit(func1, args1, depfuncs1, modules1)

    f2 = job_server.submit(func1, args2, depfuncs1, modules1)

    f3 = job_server.submit(func2, args3, depfuncs2, modules2)

   ...etc...

6) Retrieve the results as needed:

    r1 = f1()

    r2 = f2()

    r3 = f3() 

    ...etc...

 To find out how to achieve efficient parallelization with pp please take a look at examples

  Quick start guide, clusters with autodiscovery

On the nodes 

1) Start parallel python execution server on all your remote computational nodes:

    node-1> ./ppserver.py -a

    node-2> ./ppserver.py -a

    node-3> ./ppserver.py -a

On the client

2) Import pp module:

    import pp

3)  Set ppservers list to auto-discovery:

    ppservers=("*",)

4) Start pp execution server with the number of workers set to the number of processors in the system and list of ppservers to connect with :

    job_server = pp.Server(ppservers=ppservers) 

5) Submit all the tasks for parallel execution:

    f1 = job_server.submit(func1, args1, depfuncs1, modules1)

    f2 = job_server.submit(func1, args2, depfuncs1, modules1)

    f3 = job_server.submit(func2, args3, depfuncs2, modules2)

   ...etc...

6) Retrieve the results as needed:

    r1 = f1()

    r2 = f2()

    r3 = f3() 

    ...etc...

 To find out how to achieve efficient parallelization with pp please take a look at examples 

    Advanced guide, clusters 

On the nodes  

1) Start parallel python execution server on all your remote computational nodes (listen to a given port 35000,
and local network interface only, accept only connections which know correct secret):

    node-1> ./ppserver.py -p 35000 -i 192.168.0.101 -s "mysecret"

    node-2> ./ppserver.py -p 35000 -i 192.168.0.102 -s "mysecret"

    node-3> ./ppserver.py -p 35000 -i 192.168.0.103 -s "mysecret"

On the client

2) Import pp module:

    import pp

3)  Create a list of all the nodes in your cluster (computers where you've run ppserver.py)

    ppservers=("node-1:35000", "node-2:35000", "node-3:35000")

4) Start pp execution server with the number of workers set to the number of processors in the system,
list of ppservers to connect with and secret key to authorize the connection:

    job_server = pp.Server(ppservers=ppservers, secret="mysecret") 

5) Submit all the tasks for parallel execution:

    f1 = job_server.submit(func1, args1, depfuncs1, modules1)

    f2 = job_server.submit(func1, args2, depfuncs1, modules1)

    f3 = job_server.submit(func2, args3, depfuncs2, modules2)

   ...etc...

6) Retrieve the results as needed:

    r1 = f1()

    r2 = f2()

    r3 = f3() 

    ...etc...

 7) Print the execution statistics:

    job_server.print_stats()

To find out how to achieve efficient parallelization with pp please take a look at examples

  Command line options, ppserver.py 

Usage: ppserver.py [-hda] [-i interface] [-b broadcast] [-p port] [-w nworkers] [-s secret] [-t seconds]
 Options:
 -h                 : this help message
 -d                 : debug
 -a                 : enable auto-discovery service
 -i interface       : interface to listen
 -b broadcast       : broadcast address for auto-discovery service
 -p port            : port to listen
 -w nworkers        : number of workers to start
 -s secret          : secret for authentication
 -t seconds         : timeout to exit if no connections with clients exist
 -k seconds         : socket timeout in seconds 
 -P pid_file          : file to write PID to

  Security and secret key

 Due to the security concerns it is highly recommended to run ppserver.py with an non-trivial secret key (-s command line argument) which should be paired with the matching secret keyword of PP Server class constructor. Since PP 1.5.3 it is possible to set secret key by assigning pp_secret variable in the configuration file .pythonrc.py which should be located in the user home directory (please make this file readable and writable only by user). The key set in .pythonrc.py could be overridden by command line argument (for ppserver.py) and secret keyword (for PP Server class constructor).

 

  ppserver.py stats and PID file example

To print job execution statistics for ppserver.py send a SIGUSR1 signal (SIGUSR2, for jython) to its main process.
For instance on UNIX platform following commands will start a server and print its stats:
ppserver.py  -P /tmp/ppserver.pid

kill -s SIGUSR1 `cat /tmp/ppserver.pid`
 
Nutrition facts and analysis
Parallel Python Solutions | Parallel Python Forums | Parallel Python Community

Parallel Python © 2005 - 2012 Vitalii Vanovschi. All rights reserved
ppft-ppft-1.6.6.4/doc/ppserver.1000066400000000000000000000041211406150406700163040ustar00rootroot00000000000000.\" It was generated by help2man 1.36. .TH PPSERVER "1" "February 2010" "Parallel Python Network Server" "User Commands" .SH NAME ppserver \- manual page for Parallel Python Network Server .SH SYNOPSIS .B ppserver [\fI-hda\fR] [\fI-i interface\fR] [\fI-b broadcast\fR] [\fI-p port\fR] [\fI-w nworkers\fR] [\fI-s secret\fR] [\fI-t seconds\fR] .SH DESCRIPTION Parallel Python Network Server .SH OPTIONS .TP \fB\-h\fR this help message .TP \fB\-d\fR debug .TP \fB\-a\fR enable auto\-discovery service .TP \fB\-r\fR restart worker process after each task completion .TP \fB\-n\fR protocol number for pickle module .TP \fB\-c\fR path to config file .TP \fB\-i\fR interface to listen .TP \fB\-b\fR broadcast address for auto\-discovery service .TP \fB\-p\fR port to listen .TP \fB\-w\fR number of workers to start .TP \fB\-s\fR secret for authentication .TP \fB\-t\fR timeout to exit if no connections with clients exist .TP \fB\-k\fR socket timeout in seconds which is also the maximum time a remote job could be executed. Increase this value if you have long running jobs or decrease if connectivity to remote ppservers is often lost. .TP \fB\-P\fR pid_file file to write PID to .PP Please visit http://www.parallelpython.com for extended up\-to\-date documentation, examples and support forums .br .SH SECURITY Due to the security concerns it is highly recommended to run ppserver.py with an non-trivial secret key (-s command line argument) which should be paired with the matching secret keyword of PP Server class constructor. An alternative way to set a secret key is by assigning .B pp_secret variable in the configuration file .B .pythonrc.py which should be located in the user home directory (please make this file readable and writable only by user). The secret key set in .pythonrc.py could be overridden by command line argument (for ppserver.py) and secret keyword (for PP Server class constructor). .SH AUTHOR This manual page was written by Sandro Tosi , and Vitalii Vanovschi support@parallelpython.com ppft-ppft-1.6.6.4/examples/000077500000000000000000000000001406150406700154275ustar00rootroot00000000000000ppft-ppft-1.6.6.4/examples/__init__.py000066400000000000000000000007651406150406700175500ustar00rootroot00000000000000#!/usr/bin/env python # # Author: Mike McKerns (mmckerns @caltech and @uqfoundation) # Copyright (c) 2018-2021 The Uncertainty Quantification Foundation. # License: 3-clause BSD. The full license text is available at: # - https://github.com/uqfoundation/ppft/blob/master/LICENSE """ to run this test suite, first build and install `ppft`. $ python setup.py build $ python setup.py install then run the tests with: $ python -m ppft.tests or, if `nose` is installed: $ nosetests """ ppft-ppft-1.6.6.4/examples/__main__.py000066400000000000000000000015671406150406700175320ustar00rootroot00000000000000#!/usr/bin/env python # # Author: Mike McKerns (mmckerns @caltech and @uqfoundation) # Copyright (c) 2018-2021 The Uncertainty Quantification Foundation. # License: 3-clause BSD. The full license text is available at: # - https://github.com/uqfoundation/ppft/blob/master/LICENSE from __future__ import print_function import glob import os try: import pox python = pox.which_python(fullpath=False) or 'python' except ImportError: python = 'python' import subprocess as sp from sys import platform shell = platform[:3] == 'win' suite = os.path.dirname(__file__) or os.path.curdir tests = glob.glob(suite + os.path.sep + '*.py') tests = [f for f in tests if not os.path.basename(f).startswith('__')] if __name__ == '__main__': for test in tests: p = sp.Popen([python, test], shell=shell).wait() if not p: print('.', end='') print('') ppft-ppft-1.6.6.4/examples/auto_diff.py000077500000000000000000000071411406150406700177470ustar00rootroot00000000000000#!/usr/bin/env python # File: auto_diff.py # Author: Vitalii Vanovschi # Desc: This program demonstrates parallel computations with pp module # using class methods as parallel functions (available since pp 1.4). # Program calculates the partial sums of f(x) = x-x**2/2+x**3/3-x**4/4+... # and first derivatives f'(x) using automatic differentiation technique. # In the limit f(x) = ln(x+1) and f'(x) = 1/(x+1). # Parallel Python Software: http://www.parallelpython.com import math import sys if sys.version_info[0] == 2: import pp else: import ppft as pp # Partial implemenmtation of automatic differentiation class class AD(object): def __init__(self, x, dx=0.0): self.x = float(x) self.dx = float(dx) def __pow__(self, val): if isinstance(val, int): p = self.x**val return AD(self.x**val, val*self.x**(val-1)*self.dx) else: raise TypeError("Second argumnet must be an integer") def __add__(self, val): if isinstance(val, AD): return AD(self.x+val.x, self.dx+val.dx) else: return AD(self.x+val, self.dx) def __radd__(self, val): return self+val def __mul__(self, val): if isinstance(val, AD): return AD(self.x*val.x, self.x*val.dx+val.x*self.dx) else: return AD(self.x*val, val*self.dx) def __rmul__(self, val): return self*val def __div__(self, val): if isinstance(val, AD): return self*AD(1/val.x, -val.dx/val.x**2) else: return self*(1/float(val)) __truediv__ = __div__ # PYTHON 3 def __rdiv__(self, val): return AD(val)/self __rtruediv__ = __rdiv__ # PYTHON 3 def __sub__(self, val): if isinstance(val, AD): return AD(self.x-val.x, self.dx-val.dx) else: return AD(self.x-val, self.dx) def __repr__(self): return str((self.x, self.dx)) class PartialSum(object): def __init__(self, n): """ This class contains methods which will be executed in parallel """ self.n = n def t_log(self, x): """ truncated natural logarithm """ return self.partial_sum(x-1) def partial_sum(self, x): """ partial sum for truncated natural logarithm """ return sum([float(i%2 and 1 or -1)*x**i/i for i in range(1, self.n)]) print("""Usage: python auto_diff.py [ncpus] [ncpus] - the number of workers to run in parallel, if omitted it will be set to the number of processors in the system """) # tuple of all parallel python servers to connect with #ppservers = ("*",) # auto-discover #ppservers = ("10.0.0.1","10.0.0.2") # list of static IPs ppservers = () if len(sys.argv) > 1: ncpus = int(sys.argv[1]) # Creates jobserver with ncpus workers job_server = pp.Server(ncpus, ppservers=ppservers) else: # Creates jobserver with automatically detected number of workers job_server = pp.Server(ppservers=ppservers) print("Starting pp with %s workers" % job_server.get_ncpus()) proc = PartialSum(20000) results = [] for i in range(32): # Creates an object with x = float(i)/32+1 and dx = 1.0 ad_x = AD(float(i)/32+1, 1.0) # Submits a job of calulating proc.t_log(x). f = job_server.submit(proc.t_log, (ad_x, )) results.append((ad_x.x, f)) for x, f in results: # Retrieves the result of the calculation val = f() print("t_log(%lf) = %lf, t_log'(%lf) = %lf" % (x, val.x, x, val.dx)) # Print execution statistics job_server.print_stats() # Parallel Python Software: http://www.parallelpython.com ppft-ppft-1.6.6.4/examples/callback.py000077500000000000000000000051311406150406700175400ustar00rootroot00000000000000#!/usr/bin/env python # File: callback.py # Author: Vitalii Vanovschi # Desc: This program demonstrates parallel computations with pp module # using callbacks (available since pp 1.3). # Program calculates the partial sum 1-1/2+1/3-1/4+1/5-1/6+... # (in the limit it is ln(2)) # Parallel Python Software: http://www.parallelpython.com import math import time try: import _thread as thread except ImportError: import thread import sys if sys.version_info[0] == 2: import pp else: import ppft as pp class Sum(object): """Class for callbacks """ def __init__(self): self.value = 0.0 self.lock = thread.allocate_lock() def add(self, value): """ the callback function """ # we must use lock here because += is not atomic self.lock.acquire() self.value += value self.lock.release() def part_sum(start, end): """Calculates partial sum""" sum = 0 for x in range(start, end): if x % 2 == 0: sum -= 1.0 / x else: sum += 1.0 / x return sum print("""Usage: python callback.py [ncpus] [ncpus] - the number of workers to run in parallel, if omitted it will be set to the number of processors in the system """) start = 1 end = 20000000 # Divide the task into 128 subtasks parts = 128 step = int((end - start) / parts + 1) # tuple of all parallel python servers to connect with #ppservers = ("*",) # auto-discover #ppservers = ("10.0.0.1","10.0.0.2") # list of static IPs ppservers = () if len(sys.argv) > 1: ncpus = int(sys.argv[1]) # Creates jobserver with ncpus workers job_server = pp.Server(ncpus, ppservers=ppservers) else: # Creates jobserver with automatically detected number of workers job_server = pp.Server(ppservers=ppservers) print("Starting pp with %s workers" % job_server.get_ncpus()) # Create anm instance of callback class sum = Sum() # Execute the same task with different number # of active workers and measure the time start_time = time.time() for index in range(parts): starti = start+index*step endi = min(start+(index+1)*step, end) # Submit a job which will calculate partial sum # part_sum - the function # (starti, endi) - tuple with arguments for part_sum # callback=sum.add - callback function job_server.submit(part_sum, (starti, endi), callback=sum.add) #wait for jobs in all groups to finish job_server.wait() # Print the partial sum print("Partial sum is %s | diff = %s" % (sum.value, math.log(2) - sum.value)) job_server.print_stats() # Parallel Python Software: http://www.parallelpython.com ppft-ppft-1.6.6.4/examples/dynamic_ncpus.py000077500000000000000000000037171406150406700206500ustar00rootroot00000000000000#!/usr/bin/env python # File: dynamic_ncpus.py # Author: Vitalii Vanovschi # Desc: This program demonstrates parallel computations with pp module # and dynamic cpu allocation feature. # Program calculates the partial sum 1-1/2+1/3-1/4+1/5-1/6+... # (in the limit it is ln(2)) # Parallel Python Software: http://www.parallelpython.com import math import sys import time if sys.version_info[0] == 2: import pp else: import ppft as pp def part_sum(start, end): """Calculates partial sum""" sum = 0 for x in range(start, end): if x % 2 == 0: sum -= 1.0 / x else: sum += 1.0 / x return sum print("""Usage: python dynamic_ncpus.py""") print("") start = 1 end = 20000000 # Divide the task into 64 subtasks parts = 64 step = int((end - start) / parts + 1) # Create jobserver job_server = pp.Server() # Execute the same task with different amount of active workers # and measure the time for ncpus in (1, 2, 4, 8, 16, 1): job_server.set_ncpus(ncpus) jobs = [] start_time = time.time() print("Starting pp with %s workers" % job_server.get_ncpus()) for index in range(parts): starti = start+index*step endi = min(start+(index+1)*step, end) # Submit a job which will calculate partial sum # part_sum - the function # (starti, endi) - tuple with arguments for part_sum # () - tuple with functions on which function part_sum depends # () - tuple with module names which must be # imported before part_sum execution jobs.append(job_server.submit(part_sum, (starti, endi))) # Retrieve all the results and calculate their sum part_sum1 = sum([job() for job in jobs]) # Print the partial sum print("Partial sum is %s | diff = %s" % (part_sum1, math.log(2) - part_sum1)) print("Time elapsed: %s s" % (time.time() - start_time)) print("") job_server.print_stats() # Parallel Python Software: http://www.parallelpython.com ppft-ppft-1.6.6.4/examples/quicksort.py000077500000000000000000000035561406150406700200410ustar00rootroot00000000000000#!/usr/bin/env python # File: sum_primes.py # Author: Vitalii Vanovschi # Desc: This program demonstrates parallel version of quicksort algorithm # implemented using pp module # Parallel Python Software: http://www.parallelpython.com import sys, random if sys.version_info[0] == 2: import pp else: import ppft as pp try: callable(min) except NameError: def callable(x): import collections return isinstance(x, collections.Callable) def quicksort(a, n=-1, srv=None): if len(a) <= 1: return a if n: return quicksort([x for x in a if x < a[0]], n-1, srv) \ + [a[0]] + quicksort([x for x in a[1:] if x >= a[0]], n-1, srv) else: return [srv.submit(quicksort, (a,))] print("""Usage: python quicksort.py [ncpus] [ncpus] - the number of workers to run in parallel, if omitted it will be set to the number of processors in the system """) # tuple of all parallel python servers to connect with #ppservers = ("*",) #ppservers = ("10.0.0.1",) ppservers = () if len(sys.argv) > 1: ncpus = int(sys.argv[1]) # Creates jobserver with ncpus workers job_server = pp.Server(ncpus, ppservers=ppservers) else: # Creates jobserver with automatically detected number of workers job_server = pp.Server(ppservers=ppservers) print("Starting pp with %s workers" % job_server.get_ncpus()) n = 1000000 input = [] for i in range(n): input.append(random.randint(0,100000)) # set n to a positive integer to create 2^n PP jobs # or to -1 to avoid using PP # 32 PP jobs n = 5 # no PP #n = -1 outputraw = quicksort(input, n, job_server) output = [] for x in outputraw: if callable(x): output.extend(x()) else: output.append(x) print("first 30 numbers in increasing order: %s" % output[:30]) job_server.print_stats() # Parallel Python Software: http://www.parallelpython.com ppft-ppft-1.6.6.4/examples/reverse_md5.py000077500000000000000000000053731406150406700202340ustar00rootroot00000000000000#!/usr/bin/env python # File: reverse_md5.py # Author: Vitalii Vanovschi # Desc: This program demonstrates parallel computations with pp module # It tries to reverse an md5 hash in parallel # Parallel Python Software: http://www.parallelpython.com import math import sys import hashlib if sys.version_info[0] == 2: import pp else: import ppft as pp def md5test(hash, start, end): """Calculates md5 of the integers between 'start' and 'end' and compares it with 'hash'""" from pp.common import b_ for x in range(start, end): if hashlib.md5(b_(str(x))).hexdigest() == hash: return x print("""Usage: python reverse_md5.py [ncpus] [ncpus] - the number of workers to run in parallel, if omitted it will be set to the number of processors in the system """) # tuple of all parallel python servers to connect with #ppservers = ("*",) # auto-discover #ppservers = ("10.0.0.1","10.0.0.2") # list of static IPs ppservers = () if len(sys.argv) > 1: ncpus = int(sys.argv[1]) # Creates jobserver with ncpus workers job_server = pp.Server(ncpus, ppservers=ppservers) else: # Creates jobserver with automatically detected number of workers job_server = pp.Server(ppservers=ppservers) print("Starting pp with %s workers" % job_server.get_ncpus()) #Calculates md5 hash from the given number from pp.common import b_ hash = hashlib.md5(b_("1829182")).hexdigest() print("hash = %s" % hash) #Now we will try to find the number with this hash value start = 1 end = 2000000 # Since jobs are not equal in the execution time, division of the problem # into a 128 of small subproblems leads to a better load balancing parts = 128 step = int((end - start) / parts + 1) jobs = [] for index in range(parts): starti = start+index*step endi = min(start+(index+1)*step, end) # Submit a job which will test if a number in the range starti-endi # has given md5 hash # md5test - the function # (hash, starti, endi) - tuple with arguments for md5test # () - tuple with functions on which function md5test depends # ("hashlib", "_hashlib") - tuple with module names which must be # imported before md5test execution # jobs.append(job_server.submit(md5test, (hash, starti, endi), # globals=globals())) jobs.append(job_server.submit(md5test, (hash, starti, endi), (), ("hashlib", "_hashlib"))) # Retrieve results of all submited jobs for job in jobs: result = job() if result: break # Print the results if result: print("Reverse md5 for %s is %s" % (hash, result)) else: print("Reverse md5 for %s has not been found" % hash) job_server.print_stats() # Properly finalize all tasks (not necessary) job_server.wait() # Parallel Python Software: http://www.parallelpython.com ppft-ppft-1.6.6.4/examples/sum_primes.py000077500000000000000000000051411406150406700201700ustar00rootroot00000000000000#!/usr/bin/env python # File: sum_primes.py # Author: Vitalii Vanovschi # Desc: This program demonstrates parallel computations with pp module # It calculates the sum of prime numbers below a given integer in parallel # Parallel Python Software: http://www.parallelpython.com import math import sys if sys.version_info[0] == 2: import pp else: import ppft as pp def isprime(n): """Returns True if n is prime and False otherwise""" if not isinstance(n, int): raise TypeError("argument passed to is_prime is not of 'int' type") if n < 2: return False if n == 2: return True max = int(math.ceil(math.sqrt(n))) i = 2 while i <= max: if n % i == 0: return False i += 1 return True def sum_primes(n): """Calculates sum of all primes below given integer n""" return sum([x for x in range(2, n) if isprime(x)]) print("""Usage: python sum_primes.py [ncpus] [ncpus] - the number of workers to run in parallel, if omitted it will be set to the number of processors in the system""") # tuple of all parallel python servers to connect with ppservers = () #ppservers = ("127.0.0.1:60000", ) if len(sys.argv) > 1: ncpus = int(sys.argv[1]) # Creates jobserver with ncpus workers job_server = pp.Server(ncpus, ppservers=ppservers) else: # Creates jobserver with automatically detected number of workers job_server = pp.Server(ppservers=ppservers) print("Starting pp with %s workers" % job_server.get_ncpus()) # Submit a job of calulating sum_primes(100) for execution. # sum_primes - the function # (100,) - tuple with arguments for sum_primes # (isprime,) - tuple with functions on which function sum_primes depends # ("math",) - tuple with module names which must be imported before # sum_primes execution # Execution starts as soon as one of the workers will become available job1 = job_server.submit(sum_primes, (100, ), (isprime, ), ("math", )) # Retrieves the result calculated by job1 # The value of job1() is the same as sum_primes(100) # If the job has not been finished yet, execution will # wait here until result is available result = job1() print("Sum of primes below 100 is %s" % result) # The following submits 8 jobs and then retrieves the results inputs = (100000, 100100, 100200, 100300, 100400, 100500, 100600, 100700) jobs = [(input, job_server.submit(sum_primes, (input, ), (isprime, ), ("math", ))) for input in inputs] for input, job in jobs: print("Sum of primes below %s is %s" % (input, job())) job_server.print_stats() # Parallel Python Software: http://www.parallelpython.com ppft-ppft-1.6.6.4/examples/sum_primes_functor.py000077500000000000000000000053311406150406700217310ustar00rootroot00000000000000#!/usr/bin/env python # File: sum_primes_template.py # Author: Vitalii Vanovschi # Desc: This program demonstrates using pp template class # It calculates the sum of prime numbers below a given integer in parallel # Parallel Python Software: http://www.parallelpython.com import math, sys if sys.version_info[0] == 2: import pp else: import ppft as pp def isprime(n): """Returns True if n is prime and False otherwise""" if not isinstance(n, int): raise TypeError("argument passed to is_prime is not of 'int' type") if n < 2: return False if n == 2: return True max = int(math.ceil(math.sqrt(n))) i = 2 while i <= max: if n % i == 0: return False i += 1 return True def sum_primes(n): """Calculates sum of all primes below given integer n""" return sum([x for x in range(2,n) if isprime(x)]) print("""Usage: python sum_primes.py [ncpus] [ncpus] - the number of workers to run in parallel, if omitted it will be set to the number of processors in the system """) # tuple of all parallel python servers to connect with #ppservers = ("*",) # auto-discover #ppservers = ("10.0.0.1","10.0.0.2") # list of static IPs ppservers = () if len(sys.argv) > 1: ncpus = int(sys.argv[1]) # Creates jobserver with ncpus workers job_server = pp.Server(ncpus, ppservers=ppservers) else: # Creates jobserver with automatically detected number of workers job_server = pp.Server(ppservers=ppservers) print("Starting pp with %s workers" % job_server.get_ncpus()) # Creates a template # Template is created using all the parameters of the jobs except # the arguments of the function # sum_primes - the function # (isprime,) - tuple with functions on which function sum_primes depends # ("math",) - tuple with module names which must be imported # before sum_primes execution fn = pp.Template(job_server, sum_primes, (isprime,), ("math",)) # Submit a job of calulating sum_primes(100) for execution using # previously created template # Execution starts as soon as one of the workers will become available job1 = fn.submit(100) # Retrieves the result calculated by job1 # The value of job1() is the same as sum_primes(100) # If the job has not been finished yet, # execution will wait here until result is available result = job1() print("Sum of primes below 100 is %s" % result) # The following submits 8 jobs and then retrieves the results inputs = (100000, 100100, 100200, 100300, 100400, 100500, 100600, 100700) jobs = [(input, fn.submit(input)) for input in inputs] for input, job in jobs: print("Sum of primes below %s is %s" % (input, job())) job_server.print_stats() # Parallel Python Software: http://www.parallelpython.com ppft-ppft-1.6.6.4/pp/000077500000000000000000000000001406150406700142305ustar00rootroot00000000000000ppft-ppft-1.6.6.4/pp/__init__.py000066400000000000000000000017151406150406700163450ustar00rootroot00000000000000#!/usr/bin/env python # # Author: Mike McKerns (mmckerns @caltech and @uqfoundation) # Copyright (c) 2015-2016 California Institute of Technology. # Copyright (c) 2016-2021 The Uncertainty Quantification Foundation. # License: 3-clause BSD. The full license text is available at: # - https://github.com/uqfoundation/ppft/blob/master/LICENSE """ ppft -- a friendly Parallel Python fork """ from ._pp import * from ._pp import __version__, _USE_SUBPROCESS, \ _Task, _Worker, _RWorker, _Statistics from . import auto from . import common from . import transport from . import worker copyright = """Copyright (c) 2015-2016 California Institute of Technology. Copyright (c) 2016-2021 The Uncertainty Quantification Foundation.""" # comment out the following if this is a release #__version__ += '.dev0' #''' #__version__ = __version__.rsplit('.',1) #__version__[-1] = str(int(__version__[-1])+1) + '.dev0' #__version__ = ".".join(__version__) #''' # EOF ppft-ppft-1.6.6.4/pp/__main__.py000066400000000000000000000125211406150406700163230ustar00rootroot00000000000000# Parallel Python Software: http://www.parallelpython.com # Copyright (c) 2005-2012 Vitalii Vanovschi. # Copyright (c) 2015-2016 California Institute of Technology. # Copyright (c) 2016-2021 The Uncertainty Quantification Foundation. # All rights reserved. # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: # * Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of the author nor the names of its contributors # may be used to endorse or promote products derived from this software # without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE # ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF # THE POSSIBILITY OF SUCH DAMAGE. """ Parallel Python Software, PP Worker http://www.parallelpython.com - updates, documentation, examples and support forums """ import sys import os try: import io ioStringIO = io.StringIO #ioStringIO = io.BytesIO except ImportError: import StringIO as io ioStringIO = io.StringIO try: import dill as pickle except ImportError: try: import cPickle as pickle except ImportError: import pickle import six from . import transport as pptransport from . import common as ppc copyright = """Copyright (c) 2005-2012 Vitalii Vanovschi. Copyright (c) 2015-2016 California Institute of Technology. Copyright (c) 2016-2021 The Uncertainty Quantification Foundation.""" __version__ = version = "1.6.6.4" def preprocess(msg): fname, fsources, imports = pickle.loads(ppc.b_(msg)) fobjs = [compile(fsource, '', 'exec') for fsource in fsources] for module in imports: try: if not module.startswith("from ") and not module.startswith("import "): module = "import " + module six.exec_(module) globals().update(locals()) except: print("An error has occured during the module import") sys.excepthook(*sys.exc_info()) return fname, fobjs class _WorkerProcess(object): def __init__(self): self.hashmap = {} self.e = sys.__stderr__ self.sout = ioStringIO() # self.sout = open("/tmp/pp.debug","a+") sys.stdout = self.sout sys.stderr = self.sout self.t = pptransport.CPipeTransport(sys.stdin, sys.__stdout__) #open('/tmp/pp.debug', 'a+').write('Starting _WorkerProcess\n') #open('/tmp/pp.debug', 'a+').write('send... \n') self.t.send(str(os.getpid())) #open('/tmp/pp.debug', 'a+').write('send: %s\n' % str(os.getpid())) #open('/tmp/pp.debug', 'a+').write('receive... \n') self.pickle_proto = int(self.t.receive()) #open('/tmp/pp.debug', 'a+').write('receive: %s\n' % self.pickle_proto) def run(self): try: #execution cycle while 1: __fname, __fobjs = self.t.creceive(preprocess) __sargs = self.t.receive() for __fobj in __fobjs: try: six.exec_(__fobj) globals().update(locals()) except: print("An error has occured during the " + \ "function import") sys.excepthook(*sys.exc_info()) __args = pickle.loads(ppc.b_(__sargs)) __f = locals()[ppc.str_(__fname)] try: __result = __f(*__args) except: print("An error has occured during the function execution") sys.excepthook(*sys.exc_info()) __result = None __sresult = pickle.dumps((__result, self.sout.getvalue()), self.pickle_proto) self.t.send(__sresult) self.sout.truncate(0) except: print("A fatal error has occured during the function execution") sys.excepthook(*sys.exc_info()) __result = None __sresult = pickle.dumps((__result, self.sout.getvalue()), self.pickle_proto) self.t.send(__sresult) if __name__ == "__main__": # add the directory with ppworker.py to the path sys.path.append(os.path.dirname(__file__)) wp = _WorkerProcess() wp.run() # Parallel Python Software: http://www.parallelpython.com ppft-ppft-1.6.6.4/pp/_pp.py000066400000000000000000001030131406150406700153560ustar00rootroot00000000000000# Parallel Python Software: http://www.parallelpython.com # Copyright (c) 2005-2012 Vitalii Vanovschi. # Copyright (c) 2015-2016 California Institute of Technology. # Copyright (c) 2016-2021 The Uncertainty Quantification Foundation. # All rights reserved. # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: # * Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of the author nor the names of its contributors # may be used to endorse or promote products derived from this software # without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE # ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF # THE POSSIBILITY OF SUCH DAMAGE. """ Parallel Python Software, Execution Server http://www.parallelpython.com - updates, documentation, examples and support forums """ import os import threading import logging import inspect import sys import types import time import atexit try: import user except ImportError: user = types # using as a stub try: import dill as pickle from dill.source import importable from dill.source import getname except ImportError: try: import cPickle as pickle except ImportError: import pickle def importable(func): # the original code #get lines of the source and adjust indent sourcelines = inspect.getsourcelines(func)[0] #remove indentation from the first line sourcelines[0] = sourcelines[0].lstrip() return "".join(sourcelines) def getname(obj): # just get __name__ return obj.__name__ import six from . import transport as pptransport from . import auto as ppauto from . import common as ppc copyright = """Copyright (c) 2005-2012 Vitalii Vanovschi. Copyright (c) 2015-2016 California Institute of Technology. Copyright (c) 2016-2021 The Uncertainty Quantification Foundation.""" __version__ = version = "1.6.6.4" # Reconnect persistent rworkers in seconds. RECONNECT_WAIT_TIME = 5 # If set to true prints out the exceptions which are expected. SHOW_EXPECTED_EXCEPTIONS = False # we need to have set even in Python 2.3 try: set except NameError: from sets import Set as set _USE_SUBPROCESS = False try: import subprocess _USE_SUBPROCESS = True except ImportError: import popen2 class _Task(object): """Class describing single task (job) """ def __init__(self, server, tid, callback=None, callbackargs=(), group='default'): """Initializes the task""" self.lock = threading.Lock() self.lock.acquire() self.tid = tid self.server = server self.callback = callback self.callbackargs = callbackargs self.group = group self.finished = False self.unpickled = False def finalize(self, sresult): """Finalizes the task. For internal use only""" self.sresult = sresult if self.callback: self.__unpickle() self.lock.release() self.finished = True def __call__(self, raw_result=False): """Retrieves result of the task""" if not self.finished and self.server._exiting: raise DestroyedServerError("Server was destroyed before the job completion") self.wait() if not self.unpickled and not raw_result: self.__unpickle() if raw_result: return self.sresult else: return self.result def wait(self): """Waits for the task""" if not self.finished: self.lock.acquire() self.lock.release() def __unpickle(self): """Unpickles the result of the task""" self.result, sout = pickle.loads(ppc.b_(self.sresult)) self.unpickled = True if len(sout) > 0: six.print_(sout, end=' ') if self.callback: args = self.callbackargs + (self.result, ) self.callback(*args) class _Worker(object): """Local worker class """ command = [sys.executable, "-u", "-m", "ppft"] command.append("2>/dev/null") def __init__(self, restart_on_free, pickle_proto): """Initializes local worker""" self.restart_on_free = restart_on_free self.pickle_proto = pickle_proto self.start() def start(self): """Starts local worker""" if _USE_SUBPROCESS: proc = subprocess.Popen(self.command, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) self.t = pptransport.CPipeTransport(proc.stdout, proc.stdin) else: self.t = pptransport.CPipeTransport( *popen2.popen3(self.command)[:2]) #open('/tmp/pp.debug', 'a+').write('Starting _Worker\n') #open('/tmp/pp.debug', 'a+').write('receiving... \n') self.pid = int(self.t.receive()) #open('/tmp/pp.debug', 'a+').write('received: %s\n' % self.pid) #open('/tmp/pp.debug', 'a+').write('sending: %s\n' % self.pickle_proto) self.t.send(str(self.pickle_proto)) #open('/tmp/pp.debug', 'a+').write('...sent \n') self.is_free = True def stop(self): """Stops local worker""" self.is_free = False self.t.send('EXIT') # can send any string - it will exit self.t.close() def restart(self): """Restarts local worker""" self.stop() self.start() def free(self): """Frees local worker""" if self.restart_on_free: self.restart() else: self.is_free = True class _RWorker(pptransport.CSocketTransport): """Remote worker class """ def __init__(self, host, port, secret, server, message, persistent, socket_timeout): """Initializes remote worker""" self.server = server self.persistent = persistent self.host = host self.port = port self.secret = secret self.address = (host, port) self.id = host + ":" + str(port) self.server.logger.debug("Creating Rworker id=%s persistent=%s" % (self.id, persistent)) self.socket_timeout = socket_timeout self.connect(message) def __del__(self): """Closes connection with remote server""" self.close() def connect(self, message=None): """Connects to a remote server""" #open('/tmp/pp.debug', 'a+').write('connect: %s\n' % repr(message)) while True and not self.server._exiting: try: pptransport.SocketTransport.__init__(self, None, self.socket_timeout) self._connect(self.host, self.port) #open('/tmp/pp.debug', 'a+').write('connected: %s\n' % self.port) if not self.authenticate(self.secret): self.server.logger.error("Authentication failed for host=%s, port=%s" % (self.host, self.port)) return False #open('/tmp/pp.debug', 'a+').write('authenticated \n') if message: self.send(message) #open('/tmp/pp.debug', 'a+').write('sent: %s\n' % repr(message)) self.is_free = True return True except: if SHOW_EXPECTED_EXCEPTIONS: self.server.logger.debug("Exception in connect method " "(possibly expected)", exc_info=True) if not self.persistent: self.server.logger.debug("Deleting from queue Rworker %s" % (self.id, )) return False self.server.logger.info("Failed to reconnect with " \ "(host=%s, port=%i), will try again in %i s" % (self.host, self.port, RECONNECT_WAIT_TIME)) time.sleep(RECONNECT_WAIT_TIME) class _Statistics(object): """Class to hold execution statisitcs for a single node """ def __init__(self, ncpus, rworker=None): """Initializes statistics for a node""" self.ncpus = ncpus self.time = 0.0 self.njobs = 0 self.rworker = rworker class Template(object): """Template class """ def __init__(self, job_server, func, depfuncs=(), modules=(), callback=None, callbackargs=(), group='default', globals=None): """Creates Template instance jobs_server - pp server for submitting jobs func - function to be executed depfuncs - tuple with functions which might be called from 'func' modules - tuple with module names to import callback - callback function which will be called with argument list equal to callbackargs+(result,) as soon as calculation is done callbackargs - additional arguments for callback function group - job group, is used when wait(group) is called to wait for jobs in a given group to finish globals - dictionary from which all modules, functions and classes will be imported, for instance: globals=globals()""" self.job_server = job_server self.func = func self.depfuncs = depfuncs self.modules = modules self.callback = callback self.callbackargs = callbackargs self.group = group self.globals = globals def submit(self, *args): """Submits function with *arg arguments to the execution queue """ return self.job_server.submit(self.func, args, self.depfuncs, self.modules, self.callback, self.callbackargs, self.group, self.globals) class Server(object): """Parallel Python SMP execution server class """ default_port = 60000 default_secret = "epo20pdosl;dksldkmm" def __init__(self, ncpus="autodetect", ppservers=(), secret=None, restart=False, proto=2, socket_timeout=3600): """Creates Server instance ncpus - the number of worker processes to start on the local \ computer, if parameter is omitted it will be set to \ the number of processors in the system ppservers - list of active parallel python execution servers \ to connect with secret - passphrase for network connections, if omitted a default \ passphrase will be used. It's highly recommended to use a \ custom passphrase for all network connections. restart - restart the worker process after each task completion proto - protocol number for pickle module socket_timeout - socket timeout in seconds, which is the maximum \ time a remote job could be executed. Increase this value \ if you have long running jobs or decrease if connectivity \ to remote ppservers is often lost. With ncpus = 1 all tasks are executed consequently. For the best performance either use the default "autodetect" value or set ncpus to the total number of processors in the system. """ if not isinstance(ppservers, tuple): raise TypeError("ppservers argument must be a tuple") self.logger = logging.getLogger('pp') self.logger.info("Creating server instance (pp-" + version+")") self.logger.info("Running on Python %s %s", sys.version.split(" ")[0], sys.platform) self.__tid = 0 self.__active_tasks = 0 self.__active_tasks_lock = threading.Lock() self.__queue = [] self.__queue_lock = threading.Lock() self.__workers = [] self.__rworkers = [] self.__rworkers_reserved = [] self.__sourcesHM = {} self.__sfuncHM = {} self.__waittasks = [] self.__waittasks_lock = threading.Lock() self._exiting = False self.__accurate_stats = True self.autopp_list = {} self.__active_rworkers_list_lock = threading.Lock() self.__restart_on_free = restart self.__pickle_proto = proto self.__connect_locks = {} # add local directory and sys.path to PYTHONPATH pythondirs = [os.getcwd()] + sys.path if "PYTHONPATH" in os.environ and os.environ["PYTHONPATH"]: pythondirs = os.environ["PYTHONPATH"].split(os.pathsep) + pythondirs dirset = set() os.environ["PYTHONPATH"] = os.pathsep.join([dirset.add(x) or x for x in pythondirs if x not in dirset]) atexit.register(self.destroy) self.__stats = {"local": _Statistics(0)} self.set_ncpus(ncpus) self.ppservers = [] self.auto_ppservers = [] self.socket_timeout = socket_timeout for ppserver in ppservers: ppserver = ppserver.split(":") host = ppserver[0] if len(ppserver)>1: port = int(ppserver[1]) else: port = ppc.randomport() if host.find("*") == -1: self.ppservers.append((host, port)) else: if host == "*": host = "*.*.*.*" interface = host.replace("*", "0") broadcast = host.replace("*", "255") self.auto_ppservers.append(((interface, port), (broadcast, port))) self.__stats_lock = threading.Lock() if secret is not None: if not isinstance(secret, str): raise TypeError("secret must be of a string type") self.secret = str(secret) elif hasattr(user, "pp_secret"): secret = getattr(user, "pp_secret") if not isinstance(secret, str): raise TypeError("secret must be of a string type") self.secret = str(secret) else: self.secret = Server.default_secret self.__connect() self.__creation_time = time.time() self.logger.info("pp local server started with %d workers" % (self.__ncpus, )) def submit(self, func, args=(), depfuncs=(), modules=(), callback=None, callbackargs=(), group='default', globals=None): """Submits function to the execution queue func - function to be executed args - tuple with arguments of the 'func' depfuncs - tuple with functions which might be called from 'func' modules - tuple with module names to import callback - callback function which will be called with argument \ list equal to callbackargs+(result,) \ as soon as calculation is done callbackargs - additional arguments for callback function group - job group, is used when wait(group) is called to wait for jobs in a given group to finish globals - dict from which all modules, functions, and classes \ will be imported, for instance: globals=globals() """ # perform some checks for frequent mistakes if self._exiting: raise DestroyedServerError("Cannot submit jobs: server"\ " instance has been destroyed") if not isinstance(args, tuple): raise TypeError("args argument must be a tuple") if not isinstance(depfuncs, tuple): raise TypeError("depfuncs argument must be a tuple") if not isinstance(modules, tuple): raise TypeError("modules argument must be a tuple") if not isinstance(callbackargs, tuple): raise TypeError("callbackargs argument must be a tuple") if globals is not None and not isinstance(globals, dict): raise TypeError("globals argument must be a dictionary") for module in modules: if not isinstance(module, str): raise TypeError("modules argument must be a list of strings") tid = self.__gentid() other_type = types.FunctionType if six.PY3 else types.ClassType if globals: modules += tuple(self.__find_modules("", globals)) modules = tuple(set(modules)) self.logger.debug("Task %i will autoimport next modules: %s" % (tid, str(modules))) for object1 in globals.values(): if isinstance(object1, types.FunctionType) \ or isinstance(object1, other_type): depfuncs += (object1, ) task = _Task(self, tid, callback, callbackargs, group) self.__waittasks_lock.acquire() self.__waittasks.append(task) self.__waittasks_lock.release() # if the function is a method of a class add self to the arguments list if isinstance(func, types.MethodType): func_self = func.__self__ if six.PY3 else func.im_self if func_self is not None: args = (func_self, ) + args # if there is an instance of a user defined class in the arguments add # whole class to dependancies for arg in args: # Checks for both classic or new class instances if (six.PY2 and isinstance(arg, types.InstanceType)) \ or str(type(arg))[:6] == " 0") if ncpus > len(self.__workers): self.__workers.extend([_Worker(self.__restart_on_free, self.__pickle_proto) for x in\ range(ncpus - len(self.__workers))]) self.__stats["local"].ncpus = ncpus self.__ncpus = ncpus def get_active_nodes(self): """Returns active nodes as a dictionary [keys - nodes, values - ncpus]""" active_nodes = {} for node, stat in self.__stats.items(): if node == "local" or node in self.autopp_list \ and self.autopp_list[node]: active_nodes[node] = stat.ncpus return active_nodes def get_stats(self): """Returns job execution statistics as a dictionary""" for node, stat in self.__stats.items(): if stat.rworker: try: stat.rworker.send("TIME") stat.time = float(stat.rworker.receive()) except: self.__accurate_stats = False stat.time = 0.0 return self.__stats def print_stats(self): """Prints job execution statistics. Useful for benchmarking on clusters""" print("Job execution statistics:") walltime = time.time() - self.__creation_time statistics = list(self.get_stats().items()) totaljobs = 0.0 for ppserver, stat in statistics: totaljobs += stat.njobs print(" job count | % of all jobs | job time sum | " \ "time per job | job server") for ppserver, stat in statistics: if stat.njobs: print(" %6i | %6.2f | %8.4f | %11.6f | %s" \ % (stat.njobs, 100.0*stat.njobs/totaljobs, stat.time, stat.time/stat.njobs, ppserver, )) print("Time elapsed since server creation %s" % walltime) print("%s active tasks, %s cores" % (self.__active_tasks, \ self.get_ncpus())) if not self.__accurate_stats: print("WARNING: statistics provided above is not accurate" \ " due to job rescheduling") print("") # all methods below are for internal use only def insert(self, sfunc, sargs, task=None): """Inserts function into the execution queue. It's intended for internal use only (in ppserver). """ if not task: tid = self.__gentid() task = _Task(self, tid) self.__queue_lock.acquire() self.__queue.append((task, sfunc, sargs)) self.__queue_lock.release() self.logger.debug("Task %i inserted" % (task.tid, )) self.__scheduler() return task def connect1(self, host, port, persistent=True): """Conects to a remote ppserver specified by host and port""" hostid = host+":"+str(port) lock = self.__connect_locks.setdefault(hostid, threading.Lock()) lock.acquire() try: if hostid in self.autopp_list: return #open('/tmp/pp.debug', 'a+').write('_RWorker(STAT)\n') rworker = _RWorker(host, port, self.secret, self, "STAT", persistent, self.socket_timeout) ncpus = int(rworker.receive()) #open('/tmp/pp.debug', 'a+').write('_RWorker: %s\n' % ncpus) self.__stats[hostid] = _Statistics(ncpus, rworker) for x in range(ncpus): #open('/tmp/pp.debug', 'a+').write('_RWorker(EXEC)\n') rworker = _RWorker(host, port, self.secret, self, "EXEC", persistent, self.socket_timeout) self.__update_active_rworkers(rworker.id, 1) # append is atomic - no need to lock self.__rworkers self.__rworkers.append(rworker) #creating reserved rworkers for x in range(ncpus): #open('/tmp/pp.debug', 'a+').write('_RWorker(EXEC)_\n') rworker = _RWorker(host, port, self.secret, self, "EXEC", persistent, self.socket_timeout) self.__update_active_rworkers(rworker.id, 1) self.__rworkers_reserved.append(rworker) self.logger.debug("Connected to ppserver (host=%s, port=%i) \ with %i workers" % (host, port, ncpus)) #open('/tmp/pp.debug', 'a+').write('_RWorker(sched)\n') self.__scheduler() except: if SHOW_EXPECTED_EXCEPTIONS: self.logger.debug("Exception in connect1 method (possibly expected)", exc_info=True) finally: lock.release() def __connect(self): """Connects to all remote ppservers""" for ppserver in self.ppservers: ppc.start_thread("connect1", self.connect1, ppserver) self.discover = ppauto.Discover(self, True) for ppserver in self.auto_ppservers: ppc.start_thread("discover.run", self.discover.run, ppserver) def __detect_ncpus(self): """Detects the number of effective CPUs in the system""" #for Linux, Unix and MacOS if hasattr(os, "sysconf"): if "SC_NPROCESSORS_ONLN" in os.sysconf_names: #Linux and Unix ncpus = os.sysconf("SC_NPROCESSORS_ONLN") if isinstance(ncpus, int) and ncpus > 0: return ncpus else: #MacOS X return int(os.popen2("sysctl -n hw.ncpu")[1].read()) #for Windows if "NUMBER_OF_PROCESSORS" in os.environ: ncpus = int(os.environ["NUMBER_OF_PROCESSORS"]) if ncpus > 0: return ncpus #return the default value return 1 def __dumpsfunc(self, funcs, modules): """Serializes functions and modules""" hashs = hash(funcs + modules) if hashs not in self.__sfuncHM: sources = [self.__get_source(func) for func in funcs] # should probably just 'try' above, if fail rely on dill.dumps self.__sfuncHM[hashs] = pickle.dumps( (getname(funcs[0]), sources, modules), self.__pickle_proto) return self.__sfuncHM[hashs] def __find_modules(self, prefix, dict): """recursively finds all the modules in dict""" modules = [] for name, object in dict.items(): if isinstance(object, types.ModuleType) \ and name not in ("__builtins__", "pp"): if object.__name__ == prefix+name or prefix == "": modules.append(object.__name__) modules.extend(self.__find_modules( object.__name__+".", object.__dict__)) return modules def __scheduler(self): """Schedules jobs for execution""" self.__queue_lock.acquire() while self.__queue: if self.__active_tasks < self.__ncpus: #TODO: select a job number on the basis of heuristic task = self.__queue.pop(0) for worker in self.__workers: if worker.is_free: worker.is_free = False break else: self.logger.error("There are no free workers left") raise RuntimeError("Error: No free workers") self.__add_to_active_tasks(1) try: self.__stats["local"].njobs += 1 ppc.start_thread("run_local", self._run_local, task+(worker, )) except: pass else: for rworker in self.__rworkers: if rworker.is_free: rworker.is_free = False task = self.__queue.pop(0) self.__stats[rworker.id].njobs += 1 ppc.start_thread("run_remote", self._run_remote, task+(rworker, )) break else: if len(self.__queue) > self.__ncpus: for rworker in self.__rworkers_reserved: if rworker.is_free: rworker.is_free = False task = self.__queue.pop(0) self.__stats[rworker.id].njobs += 1 ppc.start_thread("run_remote", self._run_remote, task+(rworker, )) break else: break else: break self.__queue_lock.release() def __get_source(self, func): """Fetches source of the function""" hashf = hash(func) if hashf not in self.__sourcesHM: self.__sourcesHM[hashf] = importable(func) return self.__sourcesHM[hashf] def _run_local(self, job, sfunc, sargs, worker): """Runs a job locally""" if self._exiting: return self.logger.info("Task %i started", job.tid) start_time = time.time() try: #open('/tmp/pp.debug', 'a+').write('_local: %s\n' % repr(sfunc)) worker.t.csend(sfunc) #open('/tmp/pp.debug', 'a+').write('_local: %s\n' % repr(sargs)) worker.t.send(sargs) #open('/tmp/pp.debug', 'a+').write('_local: \n') sresult = worker.t.receive() #open('/tmp/pp.debug', 'a+').write('_local: %s\n' % repr(sresult)) job.finalize(sresult) #open('/tmp/pp.debug', 'a+').write('_local: _\n') except: if self._exiting: return if SHOW_EXPECTED_EXCEPTIONS: self.logger.debug("Exception in _run_local (possibly expected)", exc_info=True) # remove the job from the waiting list if self.__waittasks: self.__waittasks_lock.acquire() self.__waittasks.remove(job) self.__waittasks_lock.release() worker.free() self.__add_to_active_tasks(-1) if not self._exiting: self.__stat_add_time("local", time.time()-start_time) self.logger.debug("Task %i ended", job.tid) self.__scheduler() def _run_remote(self, job, sfunc, sargs, rworker): """Runs a job remotelly""" self.logger.debug("Task (remote) %i started", job.tid) try: #open('/tmp/pp.debug', 'a+').write('_remote: %s\n' % repr(sfunc)) rworker.csend(sfunc) #open('/tmp/pp.debug', 'a+').write('_remote: %s\n' % repr(sargs)) rworker.send(sargs) #open('/tmp/pp.debug', 'a+').write('_remote: %s\n') sresult = rworker.receive() #open('/tmp/pp.debug', 'a+').write('_remote: %s\n' % repr(sresult)) rworker.is_free = True job.finalize(sresult) #open('/tmp/pp.debug', 'a+').write('_remote: _%s\n') except: self.logger.debug("Task %i failed due to broken network " \ "connection - rescheduling", job.tid) self.insert(sfunc, sargs, job) self.__scheduler() self.__update_active_rworkers(rworker.id, -1) if rworker.connect("EXEC"): self.__update_active_rworkers(rworker.id, 1) self.__scheduler() return # remove the job from the waiting list if self.__waittasks: self.__waittasks_lock.acquire() self.__waittasks.remove(job) self.__waittasks_lock.release() self.logger.debug("Task (remote) %i ended", job.tid) self.__scheduler() def __add_to_active_tasks(self, num): """Updates the number of active tasks""" self.__active_tasks_lock.acquire() self.__active_tasks += num self.__active_tasks_lock.release() def __stat_add_time(self, node, time_add): """Updates total runtime on the node""" self.__stats_lock.acquire() self.__stats[node].time += time_add self.__stats_lock.release() def __stat_add_job(self, node): """Increments job count on the node""" self.__stats_lock.acquire() self.__stats[node].njobs += 1 self.__stats_lock.release() def __update_active_rworkers(self, id, count): """Updates list of active rworkers""" self.__active_rworkers_list_lock.acquire() if id not in self.autopp_list: self.autopp_list[id] = 0 self.autopp_list[id] += count self.__active_rworkers_list_lock.release() def __gentid(self): """Generates a unique job ID number""" self.__tid += 1 return self.__tid - 1 def __del__(self): self._exiting = True def destroy(self): """Kills ppworkers and closes open files""" self._exiting = True self.__queue_lock.acquire() self.__queue = [] self.__queue_lock.release() for worker in self.__workers: try: worker.t.close() if sys.platform.startswith("win"): os.popen('TASKKILL /PID '+str(worker.pid)+' /F 2>NUL') else: os.kill(worker.pid, 9) os.waitpid(worker.pid, 0) except: pass class DestroyedServerError(RuntimeError): pass # Parallel Python Software: http://www.parallelpython.com ppft-ppft-1.6.6.4/pp/auto.py000066400000000000000000000123751406150406700155620ustar00rootroot00000000000000# Parallel Python Software: http://www.parallelpython.com # Copyright (c) 2005-2012 Vitalii Vanovschi. # Copyright (c) 2015-2016 California Institute of Technology. # Copyright (c) 2016-2021 The Uncertainty Quantification Foundation. # All rights reserved. # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: # * Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of the author nor the names of its contributors # may be used to endorse or promote products derived from this software # without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE # ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF # THE POSSIBILITY OF SUCH DAMAGE. """Parallel Python Software, Auto-Discovery Service http://www.parallelpython.com - updates, documentation, examples and support forums """ import socket import sys import time import threading from . import common as ppc copyright = """Copyright (c) 2005-2012 Vitalii Vanovschi. Copyright (c) 2015-2016 California Institute of Technology. Copyright (c) 2016-2021 The Uncertainty Quantification Foundation.""" __version__ = version = "1.6.6.4" # broadcast every 10 sec BROADCAST_INTERVAL = 10 class Discover(object): """Auto-discovery service class""" def __init__(self, base, isclient=False): self.base = base self.hosts = [] self.isclient = isclient def run(self, interface_addr, broadcast_addr): """Starts auto-discovery""" self.interface_addr = interface_addr self.broadcast_addr = broadcast_addr self.bsocket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) self.bsocket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) self.bsocket.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1) try: self.listen() except: sys.excepthook(*sys.exc_info()) def broadcast(self): """Sends a broadcast""" if self.isclient: self.base.logger.debug("Client sends initial broadcast to (%s, %i)" % self.broadcast_addr) self.bsocket.sendto(ppc.b_("C"), self.broadcast_addr) else: while True: if self.base._exiting: return self.base.logger.debug("Server sends broadcast to (%s, %i)" % self.broadcast_addr) self.bsocket.sendto(ppc.b_("S"), self.broadcast_addr) time.sleep(BROADCAST_INTERVAL) def listen(self): """Listens for broadcasts from other clients/servers""" self.base.logger.debug("Listening (%s, %i)" % self.interface_addr) self.socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1) self.socket.settimeout(5) self.socket.bind(self.interface_addr) ppc.start_thread("broadcast", self.broadcast) while True: try: if self.base._exiting: return message, (host, port) = self.socket.recvfrom(1024) message = ppc.str_(message) remote_address = (host, self.broadcast_addr[1]) hostid = host + ":" + str(self.broadcast_addr[1]) self.base.logger.debug("Discovered host (%s, %i) message=%c" % (remote_address + (message[0], ))) if not self.base.autopp_list.get(hostid, 0) and self.isclient \ and message[0] == 'S': self.base.logger.debug("Connecting to host %s" % (hostid, )) ppc.start_thread("ppauto_connect1", self.base.connect1, remote_address+(False, )) if not self.isclient and message[0] == 'C': self.base.logger.debug("Replying to host %s" % (hostid, )) self.bsocket.sendto(ppc.b_("S"), self.broadcast_addr) except socket.timeout: pass except: self.base.logger.error("An error has occured during execution of " "Discover.listen") sys.excepthook(*sys.exc_info()) ppft-ppft-1.6.6.4/pp/common.py000066400000000000000000000117651406150406700161040ustar00rootroot00000000000000# Parallel Python Software: http://www.parallelpython.com # Copyright (c) 2005-2012 Vitalii Vanovschi. # Copyright (c) 2015-2016 California Institute of Technology. # Copyright (c) 2016-2021 The Uncertainty Quantification Foundation. # All rights reserved. # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: # * Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of the author nor the names of its contributors # may be used to endorse or promote products derived from this software # without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE # ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF # THE POSSIBILITY OF SUCH DAMAGE. """ Parallel Python Software, Execution Server http://www.parallelpython.com - updates, documentation, examples and support forums """ import threading try: # workaround for install process import six except ImportError: import types import sys six = types.ModuleType('six') six.PY3 = sys.version_info[0] == 3 six.b = lambda x:x if six.PY3: long = int import io file = io.IOBase def str_(byte): # convert to unicode if not hasattr(byte, 'decode'): return byte try: return byte.decode('ascii') except UnicodeDecodeError: # non-ascii needs special handling return repr([i for i in byte])+'{B}' def b_(string): if not hasattr(string, 'encode'): return string if not string.endswith(']{B}'): return six.b(string) return bytes(eval(string[:-3])) # special handling for non-ascii else: long = long file = file def str_(string): # is already str return string def b_(string): return six.b(string) copyright = """Copyright (c) 2005-2012 Vitalii Vanovschi. Copyright (c) 2015-2016 California Institute of Technology. Copyright (c) 2016-2021 The Uncertainty Quantification Foundation.""" __version__ = version = "1.6.6.4" def start_thread(name, target, args=(), kwargs={}, daemon=True): """Starts a thread""" thread = threading.Thread(name=name, target=target, args=args, kwargs=kwargs) thread.daemon = daemon thread.start() return thread def get_class_hierarchy(clazz): classes = [] if clazz is type(object()): return classes for base_class in clazz.__bases__: classes.extend(get_class_hierarchy(base_class)) classes.append(clazz) return classes def is_not_imported(arg, modules): args_module = str(arg.__module__) for module in modules: if args_module == module or args_module.startswith(module + "."): return False return True class portnumber(object): '''port selector Usage: >>> pick = portnumber(min=1024,max=65535) >>> print( pick() ) ''' def __init__(self, min=0, max=64*1024): '''select a port number from a given range. The first call will return a random number from the available range, and each subsequent call will return the next number in the range. Inputs: min -- minimum port number [default = 0] max -- maximum port number [default = 65536] ''' self.min = min self.max = max self.first = -1 self.current = -1 return def __call__(self): import random if self.current < 0: #first call self.current = random.randint(self.min, self.max) self.first = self.current return self.current else: self.current += 1 if self.current > self.max: self.current = self.min if self.current == self.first: raise RuntimeError( 'Range exhausted' ) return self.current return def randomport(min=1024, max=65536): '''select a random port number Inputs: min -- minimum port number [default = 1024] max -- maximum port number [default = 65536] ''' return portnumber(min, max)() # Parallel Python Software: http://www.parallelpython.com ppft-ppft-1.6.6.4/pp/server/000077500000000000000000000000001406150406700155365ustar00rootroot00000000000000ppft-ppft-1.6.6.4/pp/server/__init__.py000066400000000000000000000003231406150406700176450ustar00rootroot00000000000000""" (Remote) ppft servers can be created with ``ppserver`` (or with ``python -m ppserver``), and then jobs can be distributed to remote workers. See ``--help`` for more details on how to configure a server. """ ppft-ppft-1.6.6.4/pp/server/__main__.py000077500000000000000000000362361406150406700176450ustar00rootroot00000000000000#!/usr/bin/env python # Parallel Python Software: http://www.parallelpython.com # Copyright (c) 2005-2012 Vitalii Vanovschi. # Copyright (c) 2015-2016 California Institute of Technology. # Copyright (c) 2016-2021 The Uncertainty Quantification Foundation. # All rights reserved. # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: # * Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of the author nor the names of its contributors # may be used to endorse or promote products derived from this software # without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE # ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF # THE POSSIBILITY OF SUCH DAMAGE. """ Parallel Python Software, Network Server http://www.parallelpython.com - updates, documentation, examples and support forums """ import atexit import logging import errno import getopt import sys import socket import threading import random import string import signal import time import os import six import pp import pp.auto as ppauto import pp.common as ppc import pp.transport as pptransport copyright = """Copyright (c) 2005-2012 Vitalii Vanovschi. Copyright (c) 2015-2016 California Institute of Technology. Copyright (c) 2016-2021 The Uncertainty Quantification Foundation.""" __version__ = version = "1.6.6.4" LISTEN_SOCKET_TIMEOUT = 20 # compatibility with Jython STAT_SIGNAL = 'SIGUSR1' if 'java' not in sys.platform else 'SIGUSR2' # compatibility with Python 2.6 try: import hashlib sha_new = hashlib.sha1 except ImportError: import sha sha_new = sha.new class _NetworkServer(pp.Server): """Network Server Class """ def __init__(self, ncpus="autodetect", interface="0.0.0.0", broadcast="255.255.255.255", port=None, secret=None, timeout=None, restart=False, proto=2, socket_timeout=3600, pid_file=None): pp.Server.__init__(self, ncpus, (), secret, restart, proto, socket_timeout) if pid_file: with open(pid_file, 'w') as pfile: six.print_(os.getpid(), file=pfile) atexit.register(os.remove, pid_file) self.host = interface self.bcast = broadcast if port is not None: self.port = port else: self.port = ppc.randomport() self.timeout = timeout self.ncon = 0 self.last_con_time = time.time() self.ncon_lock = threading.Lock() self.logger.debug("Starting network server interface=%s port=%i" % (self.host, self.port)) if self.timeout is not None: self.logger.debug("ppserver will exit in %i seconds if no "\ "connections with clients exist" % (self.timeout)) ppc.start_thread("timeout_check", self.check_timeout) def ncon_add(self, val): """Keeps track of the number of connections and time of the last one""" self.ncon_lock.acquire() self.ncon += val self.last_con_time = time.time() self.ncon_lock.release() def check_timeout(self): """Checks if timeout happened and shutdowns server if it did""" while True: if self.ncon == 0: idle_time = time.time() - self.last_con_time if idle_time < self.timeout: time.sleep(self.timeout - idle_time) else: self.logger.debug("exiting ppserver due to timeout (no client"\ " connections in last %i sec)", self.timeout) os._exit(0) else: time.sleep(self.timeout) def listen(self): """Initiates listenting to incoming connections""" try: self.ssocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) # following allows ppserver to restart faster on the same port self.ssocket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) self.ssocket.settimeout(LISTEN_SOCKET_TIMEOUT) self.ssocket.bind((self.host, self.port)) self.ssocket.listen(5) except socket.error: e = sys.exc_info()[1] self.logger.error("Cannot create socket for %s:%s, %s", self.host, self.port, e) try: while 1: csocket = None # accept connections from outside try: (csocket, address) = self.ssocket.accept() except socket.timeout: pass # don't exit on an interupt due to a signal except socket.error: e = sys.exc_info()[1] if e.errno == errno.EINTR: pass if self._exiting: return # now do something with the clientsocket # in this case, we'll pretend this is a threaded server if csocket: ppc.start_thread("client_socket", self.crun, (csocket, )) except KeyboardInterrupt: pass except: self.logger.debug("Exception in listen method (possibly expected)", exc_info=True) finally: self.logger.debug("Closing server socket") self.ssocket.close() def crun(self, csocket): """Authenticates client and handles its jobs""" mysocket = pptransport.CSocketTransport(csocket, self.socket_timeout) #send PP version mysocket.send(version) #generate a random string srandom = "".join([random.choice(string.ascii_letters) for i in range(16)]) mysocket.send(srandom) answer = sha_new(ppc.b_(srandom+self.secret)).hexdigest() clientanswer = ppc.str_(mysocket.receive()) if answer != clientanswer: self.logger.warning("Authentication failed, client host=%s, port=%i" % csocket.getpeername()) mysocket.send("FAILED") csocket.close() return else: mysocket.send("OK") ctype = ppc.str_(mysocket.receive()) self.logger.debug("Control message received: " + ctype) self.ncon_add(1) try: if ctype == "STAT": #reset time at each new connection self.get_stats()["local"].time = 0.0 #open('/tmp/pp.debug', 'a+').write('STAT: \n') mysocket.send(str(self.get_ncpus())) #open('/tmp/pp.debug', 'a+').write('STAT: get_ncpus\n') while 1: mysocket.receive() #open('/tmp/pp.debug', 'a+').write('STAT: recvd\n') mysocket.send(str(self.get_stats()["local"].time)) #open('/tmp/pp.debug', 'a+').write('STAT: _\n') elif ctype=="EXEC": while 1: #open('/tmp/pp.debug', 'a+').write('EXEC: \n') sfunc = mysocket.creceive() #open('/tmp/pp.debug', 'a+').write('EXEC: '+repr((sfunc,))+'\n') sargs = mysocket.receive() #open('/tmp/pp.debug', 'a+').write('EXEC: '+repr((sargs,))+'\n') fun = self.insert(sfunc, sargs) sresult = fun(True) #open('/tmp/pp.debug', 'a+').write('EXEC: '+repr((sresult,))+'\n') mysocket.send(sresult) #open('/tmp/pp.debug', 'a+').write('EXEC: _\n') except: if self._exiting: return if pp.SHOW_EXPECTED_EXCEPTIONS: self.logger.debug("Exception in crun method (possibly expected)", exc_info=True) self.logger.debug("Closing client socket") csocket.close() self.ncon_add(-1) def broadcast(self): """Initiaates auto-discovery mechanism""" discover = ppauto.Discover(self) ppc.start_thread("server_broadcast", discover.run, ((self.host, self.port), (self.bcast, self.port))) def parse_config(file_loc): """ Parses a config file in a very forgiving way. """ # If we don't have configobj installed then let the user know and exit try: from configobj import ConfigObj except ImportError: ie = sys.exc_info()[1] #sysstderr = getattr(sys.stderr, 'buffer', sys.stderr) six.print_(("ERROR: You must have config obj installed to use" "configuration files. You can still use command line switches."), file=sys.stderr) sys.exit(1) if not os.access(file_loc, os.F_OK): six.print_("ERROR: Can not access %s." % arg, file=sys.stderr) sys.exit(1) args = {} autodiscovery = False debug = False # Load the configuration file config = ConfigObj(file_loc) # try each config item and use the result if it exists. If it doesn't # then simply pass and move along try: args['secret'] = config['general'].get('secret') except: pass try: autodiscovery = config['network'].as_bool('autodiscovery') except: pass try: args['interface'] = config['network'].get('interface', default="0.0.0.0") except: pass try: args['broadcast'] = config['network'].get('broadcast') except: pass try: args['port'] = config['network'].as_int('port') except: pass try: debug = config['general'].as_bool('debug') except: pass try: args['ncpus'] = config['general'].as_int('workers') except: pass try: args['proto'] = config['general'].as_int('proto') except: pass try: args['restart'] = config['general'].as_bool('restart') except: pass try: args['timeout'] = config['network'].as_int('timeout') except: pass try: args['socket_timeout'] = config['network'].as_int('socket_timeout') except: pass try: args['pid_file'] = config['general'].get('pid_file') except: pass # Return a tuple of the args dict and autodiscovery variable return args, autodiscovery, debug def print_usage(): """Prints help""" print("Parallel Python Network Server (pp-" + version + ")") print("Usage: ppserver [-hdar] [-f format] [-n proto]"\ " [-c config_path] [-i interface] [-b broadcast]"\ " [-p port] [-w nworkers] [-s secret] [-t seconds]"\ " [-k seconds] [-P pid_file]") print("") print("Options: ") print("-h : this help message") print("-d : set log level to debug") print("-f format : log format") print("-a : enable auto-discovery service") print("-r : restart worker process after each"\ " task completion") print("-n proto : protocol number for pickle module") print("-c path : path to config file") print("-i interface : interface to listen") print("-b broadcast : broadcast address for auto-discovery service") print("-p port : port to listen") print("-w nworkers : number of workers to start") print("-s secret : secret for authentication") print("-t seconds : timeout to exit if no connections with "\ "clients exist") print("-k seconds : socket timeout in seconds") print("-P pid_file : file to write PID to") print("") print("To print server stats send %s to its main process (unix only). " % STAT_SIGNAL) print("") print("Due to the security concerns always use a non-trivial secret key.") print("Secret key set by -s switch will override secret key assigned by") print("pp_secret variable in .pythonrc.py") print("") print("Please visit http://www.parallelpython.com for extended up-to-date") print("documentation, examples and support forums") def create_network_server(argv): try: opts, args = getopt.getopt(argv, "hdarn:c:b:i:p:w:s:t:f:k:P:", ["help"]) except getopt.GetoptError: print_usage() raise args = {} autodiscovery = False debug = False log_level = logging.WARNING log_format = "%(asctime)s - %(name)s - %(levelname)s - %(message)s" for opt, arg in opts: if opt in ("-h", "--help"): print_usage() sys.exit() elif opt == "-c": args, autodiscovery, debug = parse_config(arg) elif opt == "-d": debug = True elif opt == "-f": log_format = arg elif opt == "-i": args["interface"] = arg elif opt == "-s": args["secret"] = arg elif opt == "-p": args["port"] = int(arg) elif opt == "-w": args["ncpus"] = int(arg) elif opt == "-a": autodiscovery = True elif opt == "-r": args["restart"] = True elif opt == "-b": args["broadcast"] = arg elif opt == "-n": args["proto"] = int(arg) elif opt == "-t": args["timeout"] = int(arg) elif opt == "-k": args["socket_timeout"] = int(arg) elif opt == "-P": args["pid_file"] = arg if debug: log_level = logging.DEBUG pp.SHOW_EXPECTED_EXCEPTIONS = True log_handler = logging.StreamHandler() log_handler.setFormatter(logging.Formatter(log_format)) logging.getLogger("pp").setLevel(log_level) logging.getLogger("pp").addHandler(log_handler) server = _NetworkServer(**args) if autodiscovery: server.broadcast() return server def signal_handler(signum, stack): """Prints server stats when %s is received (unix only). """ % STAT_SIGNAL server.print_stats() if __name__ == "__main__": server = create_network_server(sys.argv[1:]) statsignal = getattr(signal, STAT_SIGNAL, None) if statsignal: signal.signal(statsignal, signal_handler) server.listen() #have to destroy it here explicitly otherwise an exception #comes out in Python 2.4 del server # Parallel Python Software: http://www.parallelpython.com ppft-ppft-1.6.6.4/pp/server/ppserver000077500000000000000000000361251406150406700173410ustar00rootroot00000000000000#!/usr/bin/env python # Parallel Python Software: http://www.parallelpython.com # Copyright (c) 2005-2012 Vitalii Vanovschi. # Copyright (c) 2015-2016 California Institute of Technology. # Copyright (c) 2016-2021 The Uncertainty Quantification Foundation. # All rights reserved. # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: # * Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of the author nor the names of its contributors # may be used to endorse or promote products derived from this software # without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE # ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF # THE POSSIBILITY OF SUCH DAMAGE. """ Parallel Python Software, Network Server http://www.parallelpython.com - updates, documentation, examples and support forums """ from __future__ import with_statement import atexit import logging import errno import getopt import sys import socket import threading import random import string import signal import time import os import six import pp import pp.auto as ppauto import pp.common as ppc import pp.transport as pptransport copyright = """Copyright (c) 2005-2012 Vitalii Vanovschi. Copyright (c) 2015-2016 California Institute of Technology. Copyright (c) 2016-2021 The Uncertainty Quantification Foundation.""" __version__ = version = "1.6.6.4" LISTEN_SOCKET_TIMEOUT = 20 # compatibility with Jython STAT_SIGNAL = 'SIGUSR1' if 'java' not in sys.platform else 'SIGUSR2' # compatibility with Python 2.6 try: import hashlib sha_new = hashlib.sha1 except ImportError: import sha sha_new = sha.new class _NetworkServer(pp.Server): """Network Server Class """ def __init__(self, ncpus="autodetect", interface="0.0.0.0", broadcast="255.255.255.255", port=None, secret=None, timeout=None, restart=False, proto=2, socket_timeout=3600, pid_file=None): pp.Server.__init__(self, ncpus, (), secret, restart, proto, socket_timeout) if pid_file: with open(pid_file, 'w') as pfile: six.print_(os.getpid(), file=pfile) atexit.register(os.remove, pid_file) self.host = interface self.bcast = broadcast if port is not None: self.port = port else: self.port = ppc.randomport() self.timeout = timeout self.ncon = 0 self.last_con_time = time.time() self.ncon_lock = threading.Lock() self.logger.debug("Starting network server interface=%s port=%i" % (self.host, self.port)) if self.timeout is not None: self.logger.debug("ppserver will exit in %i seconds if no "\ "connections with clients exist" % (self.timeout)) ppc.start_thread("timeout_check", self.check_timeout) def ncon_add(self, val): """Keeps track of the number of connections and time of the last one""" self.ncon_lock.acquire() self.ncon += val self.last_con_time = time.time() self.ncon_lock.release() def check_timeout(self): """Checks if timeout happened and shutdowns server if it did""" while True: if self.ncon == 0: idle_time = time.time() - self.last_con_time if idle_time < self.timeout: time.sleep(self.timeout - idle_time) else: self.logger.debug("exiting ppserver due to timeout (no client"\ " connections in last %i sec)", self.timeout) os._exit(0) else: time.sleep(self.timeout) def listen(self): """Initiates listenting to incoming connections""" try: self.ssocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) # following allows ppserver to restart faster on the same port self.ssocket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) self.ssocket.settimeout(LISTEN_SOCKET_TIMEOUT) self.ssocket.bind((self.host, self.port)) self.ssocket.listen(5) except socket.error: e = sys.exc_info()[1] self.logger.error("Cannot create socket for %s:%s, %s", self.host, self.port, e) try: while 1: csocket = None # accept connections from outside try: (csocket, address) = self.ssocket.accept() except socket.timeout: pass # don't exit on an interupt due to a signal except socket.error: e = sys.exc_info()[1] if e.errno == errno.EINTR: pass if self._exiting: return # now do something with the clientsocket # in this case, we'll pretend this is a threaded server if csocket: ppc.start_thread("client_socket", self.crun, (csocket, )) except KeyboardInterrupt: pass except: self.logger.debug("Exception in listen method (possibly expected)", exc_info=True) finally: self.logger.debug("Closing server socket") self.ssocket.close() def crun(self, csocket): """Authenticates client and handles its jobs""" mysocket = pptransport.CSocketTransport(csocket, self.socket_timeout) #send PP version mysocket.send(version) #generate a random string srandom = "".join([random.choice(string.ascii_letters) for i in range(16)]) mysocket.send(srandom) answer = sha_new(ppc.b_(srandom+self.secret)).hexdigest() clientanswer = ppc.str_(mysocket.receive()) if answer != clientanswer: self.logger.warning("Authentication failed, client host=%s, port=%i" % csocket.getpeername()) mysocket.send("FAILED") csocket.close() return else: mysocket.send("OK") ctype = ppc.str_(mysocket.receive()) self.logger.debug("Control message received: " + ctype) self.ncon_add(1) try: if ctype == "STAT": #reset time at each new connection self.get_stats()["local"].time = 0.0 #open('/tmp/pp.debug', 'a+').write('STAT: \n') mysocket.send(str(self.get_ncpus())) #open('/tmp/pp.debug', 'a+').write('STAT: get_ncpus\n') while 1: mysocket.receive() #open('/tmp/pp.debug', 'a+').write('STAT: recvd\n') mysocket.send(str(self.get_stats()["local"].time)) #open('/tmp/pp.debug', 'a+').write('STAT: _\n') elif ctype=="EXEC": while 1: #open('/tmp/pp.debug', 'a+').write('EXEC: \n') sfunc = mysocket.creceive() #open('/tmp/pp.debug', 'a+').write('EXEC: '+repr((sfunc,))+'\n') sargs = mysocket.receive() #open('/tmp/pp.debug', 'a+').write('EXEC: '+repr((sargs,))+'\n') fun = self.insert(sfunc, sargs) sresult = fun(True) #open('/tmp/pp.debug', 'a+').write('EXEC: '+repr((sresult,))+'\n') mysocket.send(sresult) #open('/tmp/pp.debug', 'a+').write('EXEC: _\n') except: if self._exiting: return if pp.SHOW_EXPECTED_EXCEPTIONS: self.logger.debug("Exception in crun method (possibly expected)", exc_info=True) self.logger.debug("Closing client socket") csocket.close() self.ncon_add(-1) def broadcast(self): """Initiaates auto-discovery mechanism""" discover = ppauto.Discover(self) ppc.start_thread("server_broadcast", discover.run, ((self.host, self.port), (self.bcast, self.port))) def parse_config(file_loc): """ Parses a config file in a very forgiving way. """ # If we don't have configobj installed then let the user know and exit try: from configobj import ConfigObj except ImportError: ie = sys.exc_info()[1] #sysstderr = getattr(sys.stderr, 'buffer', sys.stderr) six.print_(("ERROR: You must have config obj installed to use" "configuration files. You can still use command line switches."), file=sys.stderr) sys.exit(1) if not os.access(file_loc, os.F_OK): six.print_("ERROR: Can not access %s." % arg, file=sys.stderr) sys.exit(1) # Load the configuration file config = ConfigObj(file_loc) # try each config item and use the result if it exists. If it doesn't # then simply pass and move along try: args['secret'] = config['general'].get('secret') except: pass try: autodiscovery = config['network'].as_bool('autodiscovery') except: pass try: args['interface'] = config['network'].get('interface', default="0.0.0.0") except: pass try: args['broadcast'] = config['network'].get('broadcast') except: pass try: args['port'] = config['network'].as_int('port') except: pass try: args['loglevel'] = config['general'].as_bool('debug') except: pass try: args['ncpus'] = config['general'].as_int('workers') except: pass try: args['proto'] = config['general'].as_int('proto') except: pass try: args['restart'] = config['general'].as_bool('restart') except: pass try: args['timeout'] = config['network'].as_int('timeout') except: pass try: args['socket_timeout'] = config['network'].as_int('socket_timeout') except: pass try: args['pid_file'] = config['general'].get('pid_file') except: pass # Return a tuple of the args dict and autodiscovery variable return args, autodiscovery def print_usage(): """Prints help""" print("Parallel Python Network Server (pp-" + version + ")") print("Usage: ppserver [-hdar] [-f format] [-n proto]"\ " [-c config_path] [-i interface] [-b broadcast]"\ " [-p port] [-w nworkers] [-s secret] [-t seconds]"\ " [-k seconds] [-P pid_file]") print("") print("Options: ") print("-h : this help message") print("-d : set log level to debug") print("-f format : log format") print("-a : enable auto-discovery service") print("-r : restart worker process after each"\ " task completion") print("-n proto : protocol number for pickle module") print("-c path : path to config file") print("-i interface : interface to listen") print("-b broadcast : broadcast address for auto-discovery service") print("-p port : port to listen") print("-w nworkers : number of workers to start") print("-s secret : secret for authentication") print("-t seconds : timeout to exit if no connections with "\ "clients exist") print("-k seconds : socket timeout in seconds") print("-P pid_file : file to write PID to") print("") print("To print server stats send %s to its main process (unix only). " % STAT_SIGNAL) print("") print("Due to the security concerns always use a non-trivial secret key.") print("Secret key set by -s switch will override secret key assigned by") print("pp_secret variable in .pythonrc.py") print("") print("Please visit http://www.parallelpython.com for extended up-to-date") print("documentation, examples and support forums") def create_network_server(argv): try: opts, args = getopt.getopt(argv, "hdarn:c:b:i:p:w:s:t:f:k:P:", ["help"]) except getopt.GetoptError: print_usage() raise args = {} autodiscovery = False log_level = logging.WARNING log_format = "%(asctime)s - %(name)s - %(levelname)s - %(message)s" for opt, arg in opts: if opt in ("-h", "--help"): print_usage() sys.exit() elif opt == "-c": args, autodiscovery = parse_config(arg) elif opt == "-d": log_level = logging.DEBUG pp.SHOW_EXPECTED_EXCEPTIONS = True elif opt == "-f": log_format = arg elif opt == "-i": args["interface"] = arg elif opt == "-s": args["secret"] = arg elif opt == "-p": args["port"] = int(arg) elif opt == "-w": args["ncpus"] = int(arg) elif opt == "-a": autodiscovery = True elif opt == "-r": args["restart"] = True elif opt == "-b": args["broadcast"] = arg elif opt == "-n": args["proto"] = int(arg) elif opt == "-t": args["timeout"] = int(arg) elif opt == "-k": args["socket_timeout"] = int(arg) elif opt == "-P": args["pid_file"] = arg log_handler = logging.StreamHandler() log_handler.setFormatter(logging.Formatter(log_format)) logging.getLogger("pp").setLevel(log_level) logging.getLogger("pp").addHandler(log_handler) server = _NetworkServer(**args) if autodiscovery: server.broadcast() return server def signal_handler(signum, stack): """Prints server stats when %s is received (unix only). """ % STAT_SIGNAL server.print_stats() if __name__ == "__main__": server = create_network_server(sys.argv[1:]) statsignal = getattr(signal, STAT_SIGNAL, None) if statsignal: signal.signal(statsignal, signal_handler) server.listen() #have to destroy it here explicitly otherwise an exception #comes out in Python 2.4 del server # Parallel Python Software: http://www.parallelpython.com ppft-ppft-1.6.6.4/pp/transport.py000066400000000000000000000231651406150406700166450ustar00rootroot00000000000000# Parallel Python Software: http://www.parallelpython.com # Copyright (c) 2005-2012 Vitalii Vanovschi. # Copyright (c) 2015-2016 California Institute of Technology. # Copyright (c) 2016-2021 The Uncertainty Quantification Foundation. # All rights reserved. # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: # * Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of the author nor the names of its contributors # may be used to endorse or promote products derived from this software # without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE # ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF # THE POSSIBILITY OF SUCH DAMAGE. """ Parallel Python Software, PP Transport http://www.parallelpython.com - updates, documentation, examples and support forums """ import logging import os import socket import struct import six copyright = """Copyright (c) 2005-2012 Vitalii Vanovschi. Copyright (c) 2015-2016 California Institute of Technology. Copyright (c) 2016-2021 The Uncertainty Quantification Foundation.""" __version__ = version = "1.6.6.4" # compatibility with Python 2.6 try: import hashlib sha_new = hashlib.sha1 md5_new = hashlib.md5 except ImportError: import sha import md5 sha_new = sha.new md5_new = md5.new from . import common as ppc class Transport(object): def send(self, msg): raise NotImplemented("abstact function 'send' must be implemented "\ "in a subclass") def receive(self, preprocess=None): raise NotImplemented("abstact function 'receive' must be implemented "\ "in a subclass") def authenticate(self, secret): remote_version = ppc.str_(self.receive()) if version != remote_version: logging.error("PP version mismatch (local: pp-%s, remote: pp-%s)" % (version, remote_version)) logging.error("Please install the same version of PP on all nodes") return False srandom = ppc.b_(self.receive()) secret = ppc.b_(secret) answer = sha_new(srandom+secret).hexdigest() self.send(answer) response = ppc.b_(self.receive()) if response == ppc.b_("OK"): return True else: return False def close(self): pass def _connect(self, host, port): pass class CTransport(Transport): """Cached transport """ rcache = {} def hash(self, msg): return md5_new(ppc.b_(msg)).hexdigest() def csend(self, msg): #if hasattr(self, 'w'): # open('/tmp/pp.debug', 'a+').write(repr(('cs', self.w, msg))+'\n') #else: # open('/tmp/pp.debug', 'a+').write(repr(('cs', self.socket, msg))+'\n') msg = ppc.b_(msg) hash1 = self.hash(msg) if hash1 in self.scache: self.send(ppc.b_("H" + hash1)) else: self.send(ppc.b_("N") + msg) self.scache[hash1] = True def creceive(self, preprocess=None): msg = self.receive() #if hasattr(self, 'r'): # open('/tmp/pp.debug', 'a+').write(repr(('cr', self.r, msg))+'\n') #else: # open('/tmp/pp.debug', 'a+').write(repr(('cr', self.socket, msg))+'\n') msg = ppc.b_(msg) if msg[:1] == ppc.b_('H'): hash1 = ppc.str_(msg[1:]) else: msg = msg[1:] hash1 = self.hash(msg) if preprocess is None: preprocess = lambda x:x self.rcache[hash1] = tuple(map(preprocess, (msg, )))[0] return self.rcache[hash1] class PipeTransport(Transport): def __init__(self, r, w): #open('/tmp/pp.debug', 'a+').write(repr((r,w))+'\n') self.scache = {} self.exiting = False if isinstance(r, ppc.file) and isinstance(w, ppc.file): self.r = r self.w = w else: raise TypeError("Both arguments of PipeTransport constructor " \ "must be file objects") if six.PY3 and hasattr(self.w, 'buffer'): self.wb = self.w.buffer self.has_wb = True else: self.wb = self.w self.has_wb = False if six.PY3 and hasattr(self.r, 'buffer'): self.rb = self.r.buffer self.has_rb = True else: self.rb = self.r self.has_rb = False def send(self, msg): #l = len(ppc.b_(msg)) if (self.has_wb or self.w.mode == 'wb') else len(ppc.str_(msg)) #open('/tmp/pp.debug', 'a+').write(repr(('s', l, self.w, msg))+'\n') if self.has_wb or self.w.mode == 'wb': msg = ppc.b_(msg) self.wb.write(struct.pack("!Q", len(msg))) self.w.flush() else: #HACK: following may be > 8 bytes, needed for len(msg) >= 256 msg = ppc.str_(msg) self.wb.write(ppc.str_(struct.pack("!Q", len(msg)))) self.w.flush() self.wb.write(msg) self.w.flush() def receive(self, preprocess=None): e_size = struct.calcsize("!Q") # 8 c_size = struct.calcsize("!c") # 1 r_size = 0 stub = ppc.b_("") if (self.has_rb or self.r.mode == 'rb') else "" data = stub while r_size < e_size: msg = self.rb.read(e_size-r_size) #l = len(msg) #open('/tmp/pp.debug', 'a+').write(repr(('_r', l, self.r, msg))+'\n') if msg == stub: raise RuntimeError("Communication pipe read error") if stub == "" and msg.startswith('['): #HACK to get str_ length while not msg.endswith('{B}'): msg += self.rb.read(c_size) r_size += len(msg) data += msg e_size = struct.unpack("!Q", ppc.b_(data))[0] # get size of msg r_size = 0 data = stub while r_size < e_size: msg = self.rb.read(e_size-r_size) #l = len(msg) #open('/tmp/pp.debug', 'a+').write(repr(('r_', l, self.r, msg))+'\n') if msg == stub: raise RuntimeError("Communication pipe read error") r_size += len(msg) data += msg data = ppc.b_(data) if preprocess is None: preprocess = lambda x:x return tuple(map(preprocess, (data, )))[0] def close(self): self.w.close() self.r.close() class SocketTransport(Transport): def __init__(self, socket1, socket_timeout): if socket1: self.socket = socket1 else: self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self.socket.settimeout(socket_timeout) self.scache = {} def send(self, data): #l = len(ppc.b_(data)) #open('/tmp/pp.debug', 'a+').write(repr(('ss', l, self.socket, data))+'\n') data = ppc.b_(data) size = struct.pack("!Q", len(data)) t_size = struct.calcsize("!Q") s_size = ppc.long(0) while s_size < t_size: p_size = self.socket.send(size[s_size:]) if p_size == 0: raise RuntimeError("Socket connection is broken") s_size += p_size t_size = len(data) s_size = ppc.long(0) while s_size < t_size: p_size = self.socket.send(data[s_size:]) if p_size == 0: raise RuntimeError("Socket connection is broken") s_size += p_size def receive(self, preprocess=None): e_size = struct.calcsize("!Q") r_size = 0 stub = ppc.b_("") data = stub while r_size < e_size: msg = self.socket.recv(e_size-r_size) #l = len(msg) #open('/tmp/pp.debug', 'a+').write(repr(('_sr', l, self.socket, msg))+'\n') if msg == stub: raise RuntimeError("Socket connection is broken") r_size += len(msg) data += msg e_size = struct.unpack("!Q", ppc.b_(data))[0] # get size of msg r_size = 0 data = stub while r_size < e_size: msg = self.socket.recv(e_size-r_size) #l = len(msg) #open('/tmp/pp.debug', 'a+').write(repr(('sr_', l, self.socket, msg))+'\n') if msg == stub: raise RuntimeError("Socket connection is broken") r_size += len(msg) data += msg data = ppc.b_(data) return data def close(self): self.socket.close() def _connect(self, host, port): self.socket.connect((host, port)) class CPipeTransport(PipeTransport, CTransport): pass class CSocketTransport(SocketTransport, CTransport): pass # Parallel Python Software: http://www.parallelpython.com ppft-ppft-1.6.6.4/pp/worker.py000066400000000000000000000014561406150406700161210ustar00rootroot00000000000000#!/usr/bin/env python # # Author: Mike McKerns (mmckerns @caltech and @uqfoundation) # Copyright (c) 2015-2016 California Institute of Technology. # Copyright (c) 2016-2021 The Uncertainty Quantification Foundation. # License: 3-clause BSD. The full license text is available at: # - https://github.com/uqfoundation/ppft/blob/master/LICENSE """ ppft worker: ppworker to communicate with ppserver """ import sys if sys.version_info[0] == 2: from pp.__main__ import * from pp.__main__ import _WorkerProcess, __version__ else: from ppft.__main__ import * from ppft.__main__ import _WorkerProcess, __version__ if __name__ == "__main__": # add the directory with worker.py to the path sys.path.append(os.path.dirname(__file__)) wp = _WorkerProcess() wp.run() # EOF ppft-ppft-1.6.6.4/ppft/000077500000000000000000000000001406150406700145625ustar00rootroot00000000000000ppft-ppft-1.6.6.4/ppft/__init__.py000066400000000000000000000017151406150406700166770ustar00rootroot00000000000000#!/usr/bin/env python # # Author: Mike McKerns (mmckerns @caltech and @uqfoundation) # Copyright (c) 2015-2016 California Institute of Technology. # Copyright (c) 2016-2021 The Uncertainty Quantification Foundation. # License: 3-clause BSD. The full license text is available at: # - https://github.com/uqfoundation/ppft/blob/master/LICENSE """ ppft -- a friendly Parallel Python fork """ from ._pp import * from ._pp import __version__, _USE_SUBPROCESS, \ _Task, _Worker, _RWorker, _Statistics from . import auto from . import common from . import transport from . import worker copyright = """Copyright (c) 2015-2016 California Institute of Technology. Copyright (c) 2016-2021 The Uncertainty Quantification Foundation.""" # comment out the following if this is a release #__version__ += '.dev0' #''' #__version__ = __version__.rsplit('.',1) #__version__[-1] = str(int(__version__[-1])+1) + '.dev0' #__version__ = ".".join(__version__) #''' # EOF ppft-ppft-1.6.6.4/ppft/__main__.py000066400000000000000000000125211406150406700166550ustar00rootroot00000000000000# Parallel Python Software: http://www.parallelpython.com # Copyright (c) 2005-2012 Vitalii Vanovschi. # Copyright (c) 2015-2016 California Institute of Technology. # Copyright (c) 2016-2021 The Uncertainty Quantification Foundation. # All rights reserved. # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: # * Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of the author nor the names of its contributors # may be used to endorse or promote products derived from this software # without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE # ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF # THE POSSIBILITY OF SUCH DAMAGE. """ Parallel Python Software, PP Worker http://www.parallelpython.com - updates, documentation, examples and support forums """ import sys import os try: import io ioStringIO = io.StringIO #ioStringIO = io.BytesIO except ImportError: import StringIO as io ioStringIO = io.StringIO try: import dill as pickle except ImportError: try: import cPickle as pickle except ImportError: import pickle import six from . import transport as pptransport from . import common as ppc copyright = """Copyright (c) 2005-2012 Vitalii Vanovschi. Copyright (c) 2015-2016 California Institute of Technology. Copyright (c) 2016-2021 The Uncertainty Quantification Foundation.""" __version__ = version = "1.6.6.4" def preprocess(msg): fname, fsources, imports = pickle.loads(ppc.b_(msg)) fobjs = [compile(fsource, '', 'exec') for fsource in fsources] for module in imports: try: if not module.startswith("from ") and not module.startswith("import "): module = "import " + module six.exec_(module) globals().update(locals()) except: print("An error has occured during the module import") sys.excepthook(*sys.exc_info()) return fname, fobjs class _WorkerProcess(object): def __init__(self): self.hashmap = {} self.e = sys.__stderr__ self.sout = ioStringIO() # self.sout = open("/tmp/pp.debug","a+") sys.stdout = self.sout sys.stderr = self.sout self.t = pptransport.CPipeTransport(sys.stdin, sys.__stdout__) #open('/tmp/pp.debug', 'a+').write('Starting _WorkerProcess\n') #open('/tmp/pp.debug', 'a+').write('send... \n') self.t.send(str(os.getpid())) #open('/tmp/pp.debug', 'a+').write('send: %s\n' % str(os.getpid())) #open('/tmp/pp.debug', 'a+').write('receive... \n') self.pickle_proto = int(self.t.receive()) #open('/tmp/pp.debug', 'a+').write('receive: %s\n' % self.pickle_proto) def run(self): try: #execution cycle while 1: __fname, __fobjs = self.t.creceive(preprocess) __sargs = self.t.receive() for __fobj in __fobjs: try: six.exec_(__fobj) globals().update(locals()) except: print("An error has occured during the " + \ "function import") sys.excepthook(*sys.exc_info()) __args = pickle.loads(ppc.b_(__sargs)) __f = locals()[ppc.str_(__fname)] try: __result = __f(*__args) except: print("An error has occured during the function execution") sys.excepthook(*sys.exc_info()) __result = None __sresult = pickle.dumps((__result, self.sout.getvalue()), self.pickle_proto) self.t.send(__sresult) self.sout.truncate(0) except: print("A fatal error has occured during the function execution") sys.excepthook(*sys.exc_info()) __result = None __sresult = pickle.dumps((__result, self.sout.getvalue()), self.pickle_proto) self.t.send(__sresult) if __name__ == "__main__": # add the directory with ppworker.py to the path sys.path.append(os.path.dirname(__file__)) wp = _WorkerProcess() wp.run() # Parallel Python Software: http://www.parallelpython.com ppft-ppft-1.6.6.4/ppft/_pp.py000066400000000000000000001030131406150406700157100ustar00rootroot00000000000000# Parallel Python Software: http://www.parallelpython.com # Copyright (c) 2005-2012 Vitalii Vanovschi. # Copyright (c) 2015-2016 California Institute of Technology. # Copyright (c) 2016-2021 The Uncertainty Quantification Foundation. # All rights reserved. # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: # * Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of the author nor the names of its contributors # may be used to endorse or promote products derived from this software # without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE # ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF # THE POSSIBILITY OF SUCH DAMAGE. """ Parallel Python Software, Execution Server http://www.parallelpython.com - updates, documentation, examples and support forums """ import os import threading import logging import inspect import sys import types import time import atexit try: import user except ImportError: user = types # using as a stub try: import dill as pickle from dill.source import importable from dill.source import getname except ImportError: try: import cPickle as pickle except ImportError: import pickle def importable(func): # the original code #get lines of the source and adjust indent sourcelines = inspect.getsourcelines(func)[0] #remove indentation from the first line sourcelines[0] = sourcelines[0].lstrip() return "".join(sourcelines) def getname(obj): # just get __name__ return obj.__name__ import six from . import transport as pptransport from . import auto as ppauto from . import common as ppc copyright = """Copyright (c) 2005-2012 Vitalii Vanovschi. Copyright (c) 2015-2016 California Institute of Technology. Copyright (c) 2016-2021 The Uncertainty Quantification Foundation.""" __version__ = version = "1.6.6.4" # Reconnect persistent rworkers in seconds. RECONNECT_WAIT_TIME = 5 # If set to true prints out the exceptions which are expected. SHOW_EXPECTED_EXCEPTIONS = False # we need to have set even in Python 2.3 try: set except NameError: from sets import Set as set _USE_SUBPROCESS = False try: import subprocess _USE_SUBPROCESS = True except ImportError: import popen2 class _Task(object): """Class describing single task (job) """ def __init__(self, server, tid, callback=None, callbackargs=(), group='default'): """Initializes the task""" self.lock = threading.Lock() self.lock.acquire() self.tid = tid self.server = server self.callback = callback self.callbackargs = callbackargs self.group = group self.finished = False self.unpickled = False def finalize(self, sresult): """Finalizes the task. For internal use only""" self.sresult = sresult if self.callback: self.__unpickle() self.lock.release() self.finished = True def __call__(self, raw_result=False): """Retrieves result of the task""" if not self.finished and self.server._exiting: raise DestroyedServerError("Server was destroyed before the job completion") self.wait() if not self.unpickled and not raw_result: self.__unpickle() if raw_result: return self.sresult else: return self.result def wait(self): """Waits for the task""" if not self.finished: self.lock.acquire() self.lock.release() def __unpickle(self): """Unpickles the result of the task""" self.result, sout = pickle.loads(ppc.b_(self.sresult)) self.unpickled = True if len(sout) > 0: six.print_(sout, end=' ') if self.callback: args = self.callbackargs + (self.result, ) self.callback(*args) class _Worker(object): """Local worker class """ command = [sys.executable, "-u", "-m", "ppft"] command.append("2>/dev/null") def __init__(self, restart_on_free, pickle_proto): """Initializes local worker""" self.restart_on_free = restart_on_free self.pickle_proto = pickle_proto self.start() def start(self): """Starts local worker""" if _USE_SUBPROCESS: proc = subprocess.Popen(self.command, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) self.t = pptransport.CPipeTransport(proc.stdout, proc.stdin) else: self.t = pptransport.CPipeTransport( *popen2.popen3(self.command)[:2]) #open('/tmp/pp.debug', 'a+').write('Starting _Worker\n') #open('/tmp/pp.debug', 'a+').write('receiving... \n') self.pid = int(self.t.receive()) #open('/tmp/pp.debug', 'a+').write('received: %s\n' % self.pid) #open('/tmp/pp.debug', 'a+').write('sending: %s\n' % self.pickle_proto) self.t.send(str(self.pickle_proto)) #open('/tmp/pp.debug', 'a+').write('...sent \n') self.is_free = True def stop(self): """Stops local worker""" self.is_free = False self.t.send('EXIT') # can send any string - it will exit self.t.close() def restart(self): """Restarts local worker""" self.stop() self.start() def free(self): """Frees local worker""" if self.restart_on_free: self.restart() else: self.is_free = True class _RWorker(pptransport.CSocketTransport): """Remote worker class """ def __init__(self, host, port, secret, server, message, persistent, socket_timeout): """Initializes remote worker""" self.server = server self.persistent = persistent self.host = host self.port = port self.secret = secret self.address = (host, port) self.id = host + ":" + str(port) self.server.logger.debug("Creating Rworker id=%s persistent=%s" % (self.id, persistent)) self.socket_timeout = socket_timeout self.connect(message) def __del__(self): """Closes connection with remote server""" self.close() def connect(self, message=None): """Connects to a remote server""" #open('/tmp/pp.debug', 'a+').write('connect: %s\n' % repr(message)) while True and not self.server._exiting: try: pptransport.SocketTransport.__init__(self, None, self.socket_timeout) self._connect(self.host, self.port) #open('/tmp/pp.debug', 'a+').write('connected: %s\n' % self.port) if not self.authenticate(self.secret): self.server.logger.error("Authentication failed for host=%s, port=%s" % (self.host, self.port)) return False #open('/tmp/pp.debug', 'a+').write('authenticated \n') if message: self.send(message) #open('/tmp/pp.debug', 'a+').write('sent: %s\n' % repr(message)) self.is_free = True return True except: if SHOW_EXPECTED_EXCEPTIONS: self.server.logger.debug("Exception in connect method " "(possibly expected)", exc_info=True) if not self.persistent: self.server.logger.debug("Deleting from queue Rworker %s" % (self.id, )) return False self.server.logger.info("Failed to reconnect with " \ "(host=%s, port=%i), will try again in %i s" % (self.host, self.port, RECONNECT_WAIT_TIME)) time.sleep(RECONNECT_WAIT_TIME) class _Statistics(object): """Class to hold execution statisitcs for a single node """ def __init__(self, ncpus, rworker=None): """Initializes statistics for a node""" self.ncpus = ncpus self.time = 0.0 self.njobs = 0 self.rworker = rworker class Template(object): """Template class """ def __init__(self, job_server, func, depfuncs=(), modules=(), callback=None, callbackargs=(), group='default', globals=None): """Creates Template instance jobs_server - pp server for submitting jobs func - function to be executed depfuncs - tuple with functions which might be called from 'func' modules - tuple with module names to import callback - callback function which will be called with argument list equal to callbackargs+(result,) as soon as calculation is done callbackargs - additional arguments for callback function group - job group, is used when wait(group) is called to wait for jobs in a given group to finish globals - dictionary from which all modules, functions and classes will be imported, for instance: globals=globals()""" self.job_server = job_server self.func = func self.depfuncs = depfuncs self.modules = modules self.callback = callback self.callbackargs = callbackargs self.group = group self.globals = globals def submit(self, *args): """Submits function with *arg arguments to the execution queue """ return self.job_server.submit(self.func, args, self.depfuncs, self.modules, self.callback, self.callbackargs, self.group, self.globals) class Server(object): """Parallel Python SMP execution server class """ default_port = 60000 default_secret = "epo20pdosl;dksldkmm" def __init__(self, ncpus="autodetect", ppservers=(), secret=None, restart=False, proto=2, socket_timeout=3600): """Creates Server instance ncpus - the number of worker processes to start on the local \ computer, if parameter is omitted it will be set to \ the number of processors in the system ppservers - list of active parallel python execution servers \ to connect with secret - passphrase for network connections, if omitted a default \ passphrase will be used. It's highly recommended to use a \ custom passphrase for all network connections. restart - restart the worker process after each task completion proto - protocol number for pickle module socket_timeout - socket timeout in seconds, which is the maximum \ time a remote job could be executed. Increase this value \ if you have long running jobs or decrease if connectivity \ to remote ppservers is often lost. With ncpus = 1 all tasks are executed consequently. For the best performance either use the default "autodetect" value or set ncpus to the total number of processors in the system. """ if not isinstance(ppservers, tuple): raise TypeError("ppservers argument must be a tuple") self.logger = logging.getLogger('pp') self.logger.info("Creating server instance (pp-" + version+")") self.logger.info("Running on Python %s %s", sys.version.split(" ")[0], sys.platform) self.__tid = 0 self.__active_tasks = 0 self.__active_tasks_lock = threading.Lock() self.__queue = [] self.__queue_lock = threading.Lock() self.__workers = [] self.__rworkers = [] self.__rworkers_reserved = [] self.__sourcesHM = {} self.__sfuncHM = {} self.__waittasks = [] self.__waittasks_lock = threading.Lock() self._exiting = False self.__accurate_stats = True self.autopp_list = {} self.__active_rworkers_list_lock = threading.Lock() self.__restart_on_free = restart self.__pickle_proto = proto self.__connect_locks = {} # add local directory and sys.path to PYTHONPATH pythondirs = [os.getcwd()] + sys.path if "PYTHONPATH" in os.environ and os.environ["PYTHONPATH"]: pythondirs = os.environ["PYTHONPATH"].split(os.pathsep) + pythondirs dirset = set() os.environ["PYTHONPATH"] = os.pathsep.join([dirset.add(x) or x for x in pythondirs if x not in dirset]) atexit.register(self.destroy) self.__stats = {"local": _Statistics(0)} self.set_ncpus(ncpus) self.ppservers = [] self.auto_ppservers = [] self.socket_timeout = socket_timeout for ppserver in ppservers: ppserver = ppserver.split(":") host = ppserver[0] if len(ppserver)>1: port = int(ppserver[1]) else: port = ppc.randomport() if host.find("*") == -1: self.ppservers.append((host, port)) else: if host == "*": host = "*.*.*.*" interface = host.replace("*", "0") broadcast = host.replace("*", "255") self.auto_ppservers.append(((interface, port), (broadcast, port))) self.__stats_lock = threading.Lock() if secret is not None: if not isinstance(secret, str): raise TypeError("secret must be of a string type") self.secret = str(secret) elif hasattr(user, "pp_secret"): secret = getattr(user, "pp_secret") if not isinstance(secret, str): raise TypeError("secret must be of a string type") self.secret = str(secret) else: self.secret = Server.default_secret self.__connect() self.__creation_time = time.time() self.logger.info("pp local server started with %d workers" % (self.__ncpus, )) def submit(self, func, args=(), depfuncs=(), modules=(), callback=None, callbackargs=(), group='default', globals=None): """Submits function to the execution queue func - function to be executed args - tuple with arguments of the 'func' depfuncs - tuple with functions which might be called from 'func' modules - tuple with module names to import callback - callback function which will be called with argument \ list equal to callbackargs+(result,) \ as soon as calculation is done callbackargs - additional arguments for callback function group - job group, is used when wait(group) is called to wait for jobs in a given group to finish globals - dict from which all modules, functions, and classes \ will be imported, for instance: globals=globals() """ # perform some checks for frequent mistakes if self._exiting: raise DestroyedServerError("Cannot submit jobs: server"\ " instance has been destroyed") if not isinstance(args, tuple): raise TypeError("args argument must be a tuple") if not isinstance(depfuncs, tuple): raise TypeError("depfuncs argument must be a tuple") if not isinstance(modules, tuple): raise TypeError("modules argument must be a tuple") if not isinstance(callbackargs, tuple): raise TypeError("callbackargs argument must be a tuple") if globals is not None and not isinstance(globals, dict): raise TypeError("globals argument must be a dictionary") for module in modules: if not isinstance(module, str): raise TypeError("modules argument must be a list of strings") tid = self.__gentid() other_type = types.FunctionType if six.PY3 else types.ClassType if globals: modules += tuple(self.__find_modules("", globals)) modules = tuple(set(modules)) self.logger.debug("Task %i will autoimport next modules: %s" % (tid, str(modules))) for object1 in globals.values(): if isinstance(object1, types.FunctionType) \ or isinstance(object1, other_type): depfuncs += (object1, ) task = _Task(self, tid, callback, callbackargs, group) self.__waittasks_lock.acquire() self.__waittasks.append(task) self.__waittasks_lock.release() # if the function is a method of a class add self to the arguments list if isinstance(func, types.MethodType): func_self = func.__self__ if six.PY3 else func.im_self if func_self is not None: args = (func_self, ) + args # if there is an instance of a user defined class in the arguments add # whole class to dependancies for arg in args: # Checks for both classic or new class instances if (six.PY2 and isinstance(arg, types.InstanceType)) \ or str(type(arg))[:6] == " 0") if ncpus > len(self.__workers): self.__workers.extend([_Worker(self.__restart_on_free, self.__pickle_proto) for x in\ range(ncpus - len(self.__workers))]) self.__stats["local"].ncpus = ncpus self.__ncpus = ncpus def get_active_nodes(self): """Returns active nodes as a dictionary [keys - nodes, values - ncpus]""" active_nodes = {} for node, stat in self.__stats.items(): if node == "local" or node in self.autopp_list \ and self.autopp_list[node]: active_nodes[node] = stat.ncpus return active_nodes def get_stats(self): """Returns job execution statistics as a dictionary""" for node, stat in self.__stats.items(): if stat.rworker: try: stat.rworker.send("TIME") stat.time = float(stat.rworker.receive()) except: self.__accurate_stats = False stat.time = 0.0 return self.__stats def print_stats(self): """Prints job execution statistics. Useful for benchmarking on clusters""" print("Job execution statistics:") walltime = time.time() - self.__creation_time statistics = list(self.get_stats().items()) totaljobs = 0.0 for ppserver, stat in statistics: totaljobs += stat.njobs print(" job count | % of all jobs | job time sum | " \ "time per job | job server") for ppserver, stat in statistics: if stat.njobs: print(" %6i | %6.2f | %8.4f | %11.6f | %s" \ % (stat.njobs, 100.0*stat.njobs/totaljobs, stat.time, stat.time/stat.njobs, ppserver, )) print("Time elapsed since server creation %s" % walltime) print("%s active tasks, %s cores" % (self.__active_tasks, \ self.get_ncpus())) if not self.__accurate_stats: print("WARNING: statistics provided above is not accurate" \ " due to job rescheduling") print("") # all methods below are for internal use only def insert(self, sfunc, sargs, task=None): """Inserts function into the execution queue. It's intended for internal use only (in ppserver). """ if not task: tid = self.__gentid() task = _Task(self, tid) self.__queue_lock.acquire() self.__queue.append((task, sfunc, sargs)) self.__queue_lock.release() self.logger.debug("Task %i inserted" % (task.tid, )) self.__scheduler() return task def connect1(self, host, port, persistent=True): """Conects to a remote ppserver specified by host and port""" hostid = host+":"+str(port) lock = self.__connect_locks.setdefault(hostid, threading.Lock()) lock.acquire() try: if hostid in self.autopp_list: return #open('/tmp/pp.debug', 'a+').write('_RWorker(STAT)\n') rworker = _RWorker(host, port, self.secret, self, "STAT", persistent, self.socket_timeout) ncpus = int(rworker.receive()) #open('/tmp/pp.debug', 'a+').write('_RWorker: %s\n' % ncpus) self.__stats[hostid] = _Statistics(ncpus, rworker) for x in range(ncpus): #open('/tmp/pp.debug', 'a+').write('_RWorker(EXEC)\n') rworker = _RWorker(host, port, self.secret, self, "EXEC", persistent, self.socket_timeout) self.__update_active_rworkers(rworker.id, 1) # append is atomic - no need to lock self.__rworkers self.__rworkers.append(rworker) #creating reserved rworkers for x in range(ncpus): #open('/tmp/pp.debug', 'a+').write('_RWorker(EXEC)_\n') rworker = _RWorker(host, port, self.secret, self, "EXEC", persistent, self.socket_timeout) self.__update_active_rworkers(rworker.id, 1) self.__rworkers_reserved.append(rworker) self.logger.debug("Connected to ppserver (host=%s, port=%i) \ with %i workers" % (host, port, ncpus)) #open('/tmp/pp.debug', 'a+').write('_RWorker(sched)\n') self.__scheduler() except: if SHOW_EXPECTED_EXCEPTIONS: self.logger.debug("Exception in connect1 method (possibly expected)", exc_info=True) finally: lock.release() def __connect(self): """Connects to all remote ppservers""" for ppserver in self.ppservers: ppc.start_thread("connect1", self.connect1, ppserver) self.discover = ppauto.Discover(self, True) for ppserver in self.auto_ppservers: ppc.start_thread("discover.run", self.discover.run, ppserver) def __detect_ncpus(self): """Detects the number of effective CPUs in the system""" #for Linux, Unix and MacOS if hasattr(os, "sysconf"): if "SC_NPROCESSORS_ONLN" in os.sysconf_names: #Linux and Unix ncpus = os.sysconf("SC_NPROCESSORS_ONLN") if isinstance(ncpus, int) and ncpus > 0: return ncpus else: #MacOS X return int(os.popen2("sysctl -n hw.ncpu")[1].read()) #for Windows if "NUMBER_OF_PROCESSORS" in os.environ: ncpus = int(os.environ["NUMBER_OF_PROCESSORS"]) if ncpus > 0: return ncpus #return the default value return 1 def __dumpsfunc(self, funcs, modules): """Serializes functions and modules""" hashs = hash(funcs + modules) if hashs not in self.__sfuncHM: sources = [self.__get_source(func) for func in funcs] # should probably just 'try' above, if fail rely on dill.dumps self.__sfuncHM[hashs] = pickle.dumps( (getname(funcs[0]), sources, modules), self.__pickle_proto) return self.__sfuncHM[hashs] def __find_modules(self, prefix, dict): """recursively finds all the modules in dict""" modules = [] for name, object in dict.items(): if isinstance(object, types.ModuleType) \ and name not in ("__builtins__", "pp"): if object.__name__ == prefix+name or prefix == "": modules.append(object.__name__) modules.extend(self.__find_modules( object.__name__+".", object.__dict__)) return modules def __scheduler(self): """Schedules jobs for execution""" self.__queue_lock.acquire() while self.__queue: if self.__active_tasks < self.__ncpus: #TODO: select a job number on the basis of heuristic task = self.__queue.pop(0) for worker in self.__workers: if worker.is_free: worker.is_free = False break else: self.logger.error("There are no free workers left") raise RuntimeError("Error: No free workers") self.__add_to_active_tasks(1) try: self.__stats["local"].njobs += 1 ppc.start_thread("run_local", self._run_local, task+(worker, )) except: pass else: for rworker in self.__rworkers: if rworker.is_free: rworker.is_free = False task = self.__queue.pop(0) self.__stats[rworker.id].njobs += 1 ppc.start_thread("run_remote", self._run_remote, task+(rworker, )) break else: if len(self.__queue) > self.__ncpus: for rworker in self.__rworkers_reserved: if rworker.is_free: rworker.is_free = False task = self.__queue.pop(0) self.__stats[rworker.id].njobs += 1 ppc.start_thread("run_remote", self._run_remote, task+(rworker, )) break else: break else: break self.__queue_lock.release() def __get_source(self, func): """Fetches source of the function""" hashf = hash(func) if hashf not in self.__sourcesHM: self.__sourcesHM[hashf] = importable(func) return self.__sourcesHM[hashf] def _run_local(self, job, sfunc, sargs, worker): """Runs a job locally""" if self._exiting: return self.logger.info("Task %i started", job.tid) start_time = time.time() try: #open('/tmp/pp.debug', 'a+').write('_local: %s\n' % repr(sfunc)) worker.t.csend(sfunc) #open('/tmp/pp.debug', 'a+').write('_local: %s\n' % repr(sargs)) worker.t.send(sargs) #open('/tmp/pp.debug', 'a+').write('_local: \n') sresult = worker.t.receive() #open('/tmp/pp.debug', 'a+').write('_local: %s\n' % repr(sresult)) job.finalize(sresult) #open('/tmp/pp.debug', 'a+').write('_local: _\n') except: if self._exiting: return if SHOW_EXPECTED_EXCEPTIONS: self.logger.debug("Exception in _run_local (possibly expected)", exc_info=True) # remove the job from the waiting list if self.__waittasks: self.__waittasks_lock.acquire() self.__waittasks.remove(job) self.__waittasks_lock.release() worker.free() self.__add_to_active_tasks(-1) if not self._exiting: self.__stat_add_time("local", time.time()-start_time) self.logger.debug("Task %i ended", job.tid) self.__scheduler() def _run_remote(self, job, sfunc, sargs, rworker): """Runs a job remotelly""" self.logger.debug("Task (remote) %i started", job.tid) try: #open('/tmp/pp.debug', 'a+').write('_remote: %s\n' % repr(sfunc)) rworker.csend(sfunc) #open('/tmp/pp.debug', 'a+').write('_remote: %s\n' % repr(sargs)) rworker.send(sargs) #open('/tmp/pp.debug', 'a+').write('_remote: %s\n') sresult = rworker.receive() #open('/tmp/pp.debug', 'a+').write('_remote: %s\n' % repr(sresult)) rworker.is_free = True job.finalize(sresult) #open('/tmp/pp.debug', 'a+').write('_remote: _%s\n') except: self.logger.debug("Task %i failed due to broken network " \ "connection - rescheduling", job.tid) self.insert(sfunc, sargs, job) self.__scheduler() self.__update_active_rworkers(rworker.id, -1) if rworker.connect("EXEC"): self.__update_active_rworkers(rworker.id, 1) self.__scheduler() return # remove the job from the waiting list if self.__waittasks: self.__waittasks_lock.acquire() self.__waittasks.remove(job) self.__waittasks_lock.release() self.logger.debug("Task (remote) %i ended", job.tid) self.__scheduler() def __add_to_active_tasks(self, num): """Updates the number of active tasks""" self.__active_tasks_lock.acquire() self.__active_tasks += num self.__active_tasks_lock.release() def __stat_add_time(self, node, time_add): """Updates total runtime on the node""" self.__stats_lock.acquire() self.__stats[node].time += time_add self.__stats_lock.release() def __stat_add_job(self, node): """Increments job count on the node""" self.__stats_lock.acquire() self.__stats[node].njobs += 1 self.__stats_lock.release() def __update_active_rworkers(self, id, count): """Updates list of active rworkers""" self.__active_rworkers_list_lock.acquire() if id not in self.autopp_list: self.autopp_list[id] = 0 self.autopp_list[id] += count self.__active_rworkers_list_lock.release() def __gentid(self): """Generates a unique job ID number""" self.__tid += 1 return self.__tid - 1 def __del__(self): self._exiting = True def destroy(self): """Kills ppworkers and closes open files""" self._exiting = True self.__queue_lock.acquire() self.__queue = [] self.__queue_lock.release() for worker in self.__workers: try: worker.t.close() if sys.platform.startswith("win"): os.popen('TASKKILL /PID '+str(worker.pid)+' /F 2>NUL') else: os.kill(worker.pid, 9) os.waitpid(worker.pid, 0) except: pass class DestroyedServerError(RuntimeError): pass # Parallel Python Software: http://www.parallelpython.com ppft-ppft-1.6.6.4/ppft/auto.py000066400000000000000000000123751406150406700161140ustar00rootroot00000000000000# Parallel Python Software: http://www.parallelpython.com # Copyright (c) 2005-2012 Vitalii Vanovschi. # Copyright (c) 2015-2016 California Institute of Technology. # Copyright (c) 2016-2021 The Uncertainty Quantification Foundation. # All rights reserved. # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: # * Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of the author nor the names of its contributors # may be used to endorse or promote products derived from this software # without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE # ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF # THE POSSIBILITY OF SUCH DAMAGE. """Parallel Python Software, Auto-Discovery Service http://www.parallelpython.com - updates, documentation, examples and support forums """ import socket import sys import time import threading from . import common as ppc copyright = """Copyright (c) 2005-2012 Vitalii Vanovschi. Copyright (c) 2015-2016 California Institute of Technology. Copyright (c) 2016-2021 The Uncertainty Quantification Foundation.""" __version__ = version = "1.6.6.4" # broadcast every 10 sec BROADCAST_INTERVAL = 10 class Discover(object): """Auto-discovery service class""" def __init__(self, base, isclient=False): self.base = base self.hosts = [] self.isclient = isclient def run(self, interface_addr, broadcast_addr): """Starts auto-discovery""" self.interface_addr = interface_addr self.broadcast_addr = broadcast_addr self.bsocket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) self.bsocket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) self.bsocket.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1) try: self.listen() except: sys.excepthook(*sys.exc_info()) def broadcast(self): """Sends a broadcast""" if self.isclient: self.base.logger.debug("Client sends initial broadcast to (%s, %i)" % self.broadcast_addr) self.bsocket.sendto(ppc.b_("C"), self.broadcast_addr) else: while True: if self.base._exiting: return self.base.logger.debug("Server sends broadcast to (%s, %i)" % self.broadcast_addr) self.bsocket.sendto(ppc.b_("S"), self.broadcast_addr) time.sleep(BROADCAST_INTERVAL) def listen(self): """Listens for broadcasts from other clients/servers""" self.base.logger.debug("Listening (%s, %i)" % self.interface_addr) self.socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1) self.socket.settimeout(5) self.socket.bind(self.interface_addr) ppc.start_thread("broadcast", self.broadcast) while True: try: if self.base._exiting: return message, (host, port) = self.socket.recvfrom(1024) message = ppc.str_(message) remote_address = (host, self.broadcast_addr[1]) hostid = host + ":" + str(self.broadcast_addr[1]) self.base.logger.debug("Discovered host (%s, %i) message=%c" % (remote_address + (message[0], ))) if not self.base.autopp_list.get(hostid, 0) and self.isclient \ and message[0] == 'S': self.base.logger.debug("Connecting to host %s" % (hostid, )) ppc.start_thread("ppauto_connect1", self.base.connect1, remote_address+(False, )) if not self.isclient and message[0] == 'C': self.base.logger.debug("Replying to host %s" % (hostid, )) self.bsocket.sendto(ppc.b_("S"), self.broadcast_addr) except socket.timeout: pass except: self.base.logger.error("An error has occured during execution of " "Discover.listen") sys.excepthook(*sys.exc_info()) ppft-ppft-1.6.6.4/ppft/common.py000066400000000000000000000117651406150406700164360ustar00rootroot00000000000000# Parallel Python Software: http://www.parallelpython.com # Copyright (c) 2005-2012 Vitalii Vanovschi. # Copyright (c) 2015-2016 California Institute of Technology. # Copyright (c) 2016-2021 The Uncertainty Quantification Foundation. # All rights reserved. # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: # * Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of the author nor the names of its contributors # may be used to endorse or promote products derived from this software # without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE # ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF # THE POSSIBILITY OF SUCH DAMAGE. """ Parallel Python Software, Execution Server http://www.parallelpython.com - updates, documentation, examples and support forums """ import threading try: # workaround for install process import six except ImportError: import types import sys six = types.ModuleType('six') six.PY3 = sys.version_info[0] == 3 six.b = lambda x:x if six.PY3: long = int import io file = io.IOBase def str_(byte): # convert to unicode if not hasattr(byte, 'decode'): return byte try: return byte.decode('ascii') except UnicodeDecodeError: # non-ascii needs special handling return repr([i for i in byte])+'{B}' def b_(string): if not hasattr(string, 'encode'): return string if not string.endswith(']{B}'): return six.b(string) return bytes(eval(string[:-3])) # special handling for non-ascii else: long = long file = file def str_(string): # is already str return string def b_(string): return six.b(string) copyright = """Copyright (c) 2005-2012 Vitalii Vanovschi. Copyright (c) 2015-2016 California Institute of Technology. Copyright (c) 2016-2021 The Uncertainty Quantification Foundation.""" __version__ = version = "1.6.6.4" def start_thread(name, target, args=(), kwargs={}, daemon=True): """Starts a thread""" thread = threading.Thread(name=name, target=target, args=args, kwargs=kwargs) thread.daemon = daemon thread.start() return thread def get_class_hierarchy(clazz): classes = [] if clazz is type(object()): return classes for base_class in clazz.__bases__: classes.extend(get_class_hierarchy(base_class)) classes.append(clazz) return classes def is_not_imported(arg, modules): args_module = str(arg.__module__) for module in modules: if args_module == module or args_module.startswith(module + "."): return False return True class portnumber(object): '''port selector Usage: >>> pick = portnumber(min=1024,max=65535) >>> print( pick() ) ''' def __init__(self, min=0, max=64*1024): '''select a port number from a given range. The first call will return a random number from the available range, and each subsequent call will return the next number in the range. Inputs: min -- minimum port number [default = 0] max -- maximum port number [default = 65536] ''' self.min = min self.max = max self.first = -1 self.current = -1 return def __call__(self): import random if self.current < 0: #first call self.current = random.randint(self.min, self.max) self.first = self.current return self.current else: self.current += 1 if self.current > self.max: self.current = self.min if self.current == self.first: raise RuntimeError( 'Range exhausted' ) return self.current return def randomport(min=1024, max=65536): '''select a random port number Inputs: min -- minimum port number [default = 1024] max -- maximum port number [default = 65536] ''' return portnumber(min, max)() # Parallel Python Software: http://www.parallelpython.com ppft-ppft-1.6.6.4/ppft/server/000077500000000000000000000000001406150406700160705ustar00rootroot00000000000000ppft-ppft-1.6.6.4/ppft/server/__init__.py000066400000000000000000000003231406150406700201770ustar00rootroot00000000000000""" (Remote) ppft servers can be created with ``ppserver`` (or with ``python -m ppserver``), and then jobs can be distributed to remote workers. See ``--help`` for more details on how to configure a server. """ ppft-ppft-1.6.6.4/ppft/server/__main__.py000077500000000000000000000362361406150406700201770ustar00rootroot00000000000000#!/usr/bin/env python # Parallel Python Software: http://www.parallelpython.com # Copyright (c) 2005-2012 Vitalii Vanovschi. # Copyright (c) 2015-2016 California Institute of Technology. # Copyright (c) 2016-2021 The Uncertainty Quantification Foundation. # All rights reserved. # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: # * Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of the author nor the names of its contributors # may be used to endorse or promote products derived from this software # without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE # ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF # THE POSSIBILITY OF SUCH DAMAGE. """ Parallel Python Software, Network Server http://www.parallelpython.com - updates, documentation, examples and support forums """ import atexit import logging import errno import getopt import sys import socket import threading import random import string import signal import time import os import six import pp import pp.auto as ppauto import pp.common as ppc import pp.transport as pptransport copyright = """Copyright (c) 2005-2012 Vitalii Vanovschi. Copyright (c) 2015-2016 California Institute of Technology. Copyright (c) 2016-2021 The Uncertainty Quantification Foundation.""" __version__ = version = "1.6.6.4" LISTEN_SOCKET_TIMEOUT = 20 # compatibility with Jython STAT_SIGNAL = 'SIGUSR1' if 'java' not in sys.platform else 'SIGUSR2' # compatibility with Python 2.6 try: import hashlib sha_new = hashlib.sha1 except ImportError: import sha sha_new = sha.new class _NetworkServer(pp.Server): """Network Server Class """ def __init__(self, ncpus="autodetect", interface="0.0.0.0", broadcast="255.255.255.255", port=None, secret=None, timeout=None, restart=False, proto=2, socket_timeout=3600, pid_file=None): pp.Server.__init__(self, ncpus, (), secret, restart, proto, socket_timeout) if pid_file: with open(pid_file, 'w') as pfile: six.print_(os.getpid(), file=pfile) atexit.register(os.remove, pid_file) self.host = interface self.bcast = broadcast if port is not None: self.port = port else: self.port = ppc.randomport() self.timeout = timeout self.ncon = 0 self.last_con_time = time.time() self.ncon_lock = threading.Lock() self.logger.debug("Starting network server interface=%s port=%i" % (self.host, self.port)) if self.timeout is not None: self.logger.debug("ppserver will exit in %i seconds if no "\ "connections with clients exist" % (self.timeout)) ppc.start_thread("timeout_check", self.check_timeout) def ncon_add(self, val): """Keeps track of the number of connections and time of the last one""" self.ncon_lock.acquire() self.ncon += val self.last_con_time = time.time() self.ncon_lock.release() def check_timeout(self): """Checks if timeout happened and shutdowns server if it did""" while True: if self.ncon == 0: idle_time = time.time() - self.last_con_time if idle_time < self.timeout: time.sleep(self.timeout - idle_time) else: self.logger.debug("exiting ppserver due to timeout (no client"\ " connections in last %i sec)", self.timeout) os._exit(0) else: time.sleep(self.timeout) def listen(self): """Initiates listenting to incoming connections""" try: self.ssocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) # following allows ppserver to restart faster on the same port self.ssocket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) self.ssocket.settimeout(LISTEN_SOCKET_TIMEOUT) self.ssocket.bind((self.host, self.port)) self.ssocket.listen(5) except socket.error: e = sys.exc_info()[1] self.logger.error("Cannot create socket for %s:%s, %s", self.host, self.port, e) try: while 1: csocket = None # accept connections from outside try: (csocket, address) = self.ssocket.accept() except socket.timeout: pass # don't exit on an interupt due to a signal except socket.error: e = sys.exc_info()[1] if e.errno == errno.EINTR: pass if self._exiting: return # now do something with the clientsocket # in this case, we'll pretend this is a threaded server if csocket: ppc.start_thread("client_socket", self.crun, (csocket, )) except KeyboardInterrupt: pass except: self.logger.debug("Exception in listen method (possibly expected)", exc_info=True) finally: self.logger.debug("Closing server socket") self.ssocket.close() def crun(self, csocket): """Authenticates client and handles its jobs""" mysocket = pptransport.CSocketTransport(csocket, self.socket_timeout) #send PP version mysocket.send(version) #generate a random string srandom = "".join([random.choice(string.ascii_letters) for i in range(16)]) mysocket.send(srandom) answer = sha_new(ppc.b_(srandom+self.secret)).hexdigest() clientanswer = ppc.str_(mysocket.receive()) if answer != clientanswer: self.logger.warning("Authentication failed, client host=%s, port=%i" % csocket.getpeername()) mysocket.send("FAILED") csocket.close() return else: mysocket.send("OK") ctype = ppc.str_(mysocket.receive()) self.logger.debug("Control message received: " + ctype) self.ncon_add(1) try: if ctype == "STAT": #reset time at each new connection self.get_stats()["local"].time = 0.0 #open('/tmp/pp.debug', 'a+').write('STAT: \n') mysocket.send(str(self.get_ncpus())) #open('/tmp/pp.debug', 'a+').write('STAT: get_ncpus\n') while 1: mysocket.receive() #open('/tmp/pp.debug', 'a+').write('STAT: recvd\n') mysocket.send(str(self.get_stats()["local"].time)) #open('/tmp/pp.debug', 'a+').write('STAT: _\n') elif ctype=="EXEC": while 1: #open('/tmp/pp.debug', 'a+').write('EXEC: \n') sfunc = mysocket.creceive() #open('/tmp/pp.debug', 'a+').write('EXEC: '+repr((sfunc,))+'\n') sargs = mysocket.receive() #open('/tmp/pp.debug', 'a+').write('EXEC: '+repr((sargs,))+'\n') fun = self.insert(sfunc, sargs) sresult = fun(True) #open('/tmp/pp.debug', 'a+').write('EXEC: '+repr((sresult,))+'\n') mysocket.send(sresult) #open('/tmp/pp.debug', 'a+').write('EXEC: _\n') except: if self._exiting: return if pp.SHOW_EXPECTED_EXCEPTIONS: self.logger.debug("Exception in crun method (possibly expected)", exc_info=True) self.logger.debug("Closing client socket") csocket.close() self.ncon_add(-1) def broadcast(self): """Initiaates auto-discovery mechanism""" discover = ppauto.Discover(self) ppc.start_thread("server_broadcast", discover.run, ((self.host, self.port), (self.bcast, self.port))) def parse_config(file_loc): """ Parses a config file in a very forgiving way. """ # If we don't have configobj installed then let the user know and exit try: from configobj import ConfigObj except ImportError: ie = sys.exc_info()[1] #sysstderr = getattr(sys.stderr, 'buffer', sys.stderr) six.print_(("ERROR: You must have config obj installed to use" "configuration files. You can still use command line switches."), file=sys.stderr) sys.exit(1) if not os.access(file_loc, os.F_OK): six.print_("ERROR: Can not access %s." % arg, file=sys.stderr) sys.exit(1) args = {} autodiscovery = False debug = False # Load the configuration file config = ConfigObj(file_loc) # try each config item and use the result if it exists. If it doesn't # then simply pass and move along try: args['secret'] = config['general'].get('secret') except: pass try: autodiscovery = config['network'].as_bool('autodiscovery') except: pass try: args['interface'] = config['network'].get('interface', default="0.0.0.0") except: pass try: args['broadcast'] = config['network'].get('broadcast') except: pass try: args['port'] = config['network'].as_int('port') except: pass try: debug = config['general'].as_bool('debug') except: pass try: args['ncpus'] = config['general'].as_int('workers') except: pass try: args['proto'] = config['general'].as_int('proto') except: pass try: args['restart'] = config['general'].as_bool('restart') except: pass try: args['timeout'] = config['network'].as_int('timeout') except: pass try: args['socket_timeout'] = config['network'].as_int('socket_timeout') except: pass try: args['pid_file'] = config['general'].get('pid_file') except: pass # Return a tuple of the args dict and autodiscovery variable return args, autodiscovery, debug def print_usage(): """Prints help""" print("Parallel Python Network Server (pp-" + version + ")") print("Usage: ppserver [-hdar] [-f format] [-n proto]"\ " [-c config_path] [-i interface] [-b broadcast]"\ " [-p port] [-w nworkers] [-s secret] [-t seconds]"\ " [-k seconds] [-P pid_file]") print("") print("Options: ") print("-h : this help message") print("-d : set log level to debug") print("-f format : log format") print("-a : enable auto-discovery service") print("-r : restart worker process after each"\ " task completion") print("-n proto : protocol number for pickle module") print("-c path : path to config file") print("-i interface : interface to listen") print("-b broadcast : broadcast address for auto-discovery service") print("-p port : port to listen") print("-w nworkers : number of workers to start") print("-s secret : secret for authentication") print("-t seconds : timeout to exit if no connections with "\ "clients exist") print("-k seconds : socket timeout in seconds") print("-P pid_file : file to write PID to") print("") print("To print server stats send %s to its main process (unix only). " % STAT_SIGNAL) print("") print("Due to the security concerns always use a non-trivial secret key.") print("Secret key set by -s switch will override secret key assigned by") print("pp_secret variable in .pythonrc.py") print("") print("Please visit http://www.parallelpython.com for extended up-to-date") print("documentation, examples and support forums") def create_network_server(argv): try: opts, args = getopt.getopt(argv, "hdarn:c:b:i:p:w:s:t:f:k:P:", ["help"]) except getopt.GetoptError: print_usage() raise args = {} autodiscovery = False debug = False log_level = logging.WARNING log_format = "%(asctime)s - %(name)s - %(levelname)s - %(message)s" for opt, arg in opts: if opt in ("-h", "--help"): print_usage() sys.exit() elif opt == "-c": args, autodiscovery, debug = parse_config(arg) elif opt == "-d": debug = True elif opt == "-f": log_format = arg elif opt == "-i": args["interface"] = arg elif opt == "-s": args["secret"] = arg elif opt == "-p": args["port"] = int(arg) elif opt == "-w": args["ncpus"] = int(arg) elif opt == "-a": autodiscovery = True elif opt == "-r": args["restart"] = True elif opt == "-b": args["broadcast"] = arg elif opt == "-n": args["proto"] = int(arg) elif opt == "-t": args["timeout"] = int(arg) elif opt == "-k": args["socket_timeout"] = int(arg) elif opt == "-P": args["pid_file"] = arg if debug: log_level = logging.DEBUG pp.SHOW_EXPECTED_EXCEPTIONS = True log_handler = logging.StreamHandler() log_handler.setFormatter(logging.Formatter(log_format)) logging.getLogger("pp").setLevel(log_level) logging.getLogger("pp").addHandler(log_handler) server = _NetworkServer(**args) if autodiscovery: server.broadcast() return server def signal_handler(signum, stack): """Prints server stats when %s is received (unix only). """ % STAT_SIGNAL server.print_stats() if __name__ == "__main__": server = create_network_server(sys.argv[1:]) statsignal = getattr(signal, STAT_SIGNAL, None) if statsignal: signal.signal(statsignal, signal_handler) server.listen() #have to destroy it here explicitly otherwise an exception #comes out in Python 2.4 del server # Parallel Python Software: http://www.parallelpython.com ppft-ppft-1.6.6.4/ppft/server/ppserver000077500000000000000000000361251406150406700176730ustar00rootroot00000000000000#!/usr/bin/env python # Parallel Python Software: http://www.parallelpython.com # Copyright (c) 2005-2012 Vitalii Vanovschi. # Copyright (c) 2015-2016 California Institute of Technology. # Copyright (c) 2016-2021 The Uncertainty Quantification Foundation. # All rights reserved. # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: # * Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of the author nor the names of its contributors # may be used to endorse or promote products derived from this software # without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE # ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF # THE POSSIBILITY OF SUCH DAMAGE. """ Parallel Python Software, Network Server http://www.parallelpython.com - updates, documentation, examples and support forums """ from __future__ import with_statement import atexit import logging import errno import getopt import sys import socket import threading import random import string import signal import time import os import six import pp import pp.auto as ppauto import pp.common as ppc import pp.transport as pptransport copyright = """Copyright (c) 2005-2012 Vitalii Vanovschi. Copyright (c) 2015-2016 California Institute of Technology. Copyright (c) 2016-2021 The Uncertainty Quantification Foundation.""" __version__ = version = "1.6.6.4" LISTEN_SOCKET_TIMEOUT = 20 # compatibility with Jython STAT_SIGNAL = 'SIGUSR1' if 'java' not in sys.platform else 'SIGUSR2' # compatibility with Python 2.6 try: import hashlib sha_new = hashlib.sha1 except ImportError: import sha sha_new = sha.new class _NetworkServer(pp.Server): """Network Server Class """ def __init__(self, ncpus="autodetect", interface="0.0.0.0", broadcast="255.255.255.255", port=None, secret=None, timeout=None, restart=False, proto=2, socket_timeout=3600, pid_file=None): pp.Server.__init__(self, ncpus, (), secret, restart, proto, socket_timeout) if pid_file: with open(pid_file, 'w') as pfile: six.print_(os.getpid(), file=pfile) atexit.register(os.remove, pid_file) self.host = interface self.bcast = broadcast if port is not None: self.port = port else: self.port = ppc.randomport() self.timeout = timeout self.ncon = 0 self.last_con_time = time.time() self.ncon_lock = threading.Lock() self.logger.debug("Starting network server interface=%s port=%i" % (self.host, self.port)) if self.timeout is not None: self.logger.debug("ppserver will exit in %i seconds if no "\ "connections with clients exist" % (self.timeout)) ppc.start_thread("timeout_check", self.check_timeout) def ncon_add(self, val): """Keeps track of the number of connections and time of the last one""" self.ncon_lock.acquire() self.ncon += val self.last_con_time = time.time() self.ncon_lock.release() def check_timeout(self): """Checks if timeout happened and shutdowns server if it did""" while True: if self.ncon == 0: idle_time = time.time() - self.last_con_time if idle_time < self.timeout: time.sleep(self.timeout - idle_time) else: self.logger.debug("exiting ppserver due to timeout (no client"\ " connections in last %i sec)", self.timeout) os._exit(0) else: time.sleep(self.timeout) def listen(self): """Initiates listenting to incoming connections""" try: self.ssocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) # following allows ppserver to restart faster on the same port self.ssocket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) self.ssocket.settimeout(LISTEN_SOCKET_TIMEOUT) self.ssocket.bind((self.host, self.port)) self.ssocket.listen(5) except socket.error: e = sys.exc_info()[1] self.logger.error("Cannot create socket for %s:%s, %s", self.host, self.port, e) try: while 1: csocket = None # accept connections from outside try: (csocket, address) = self.ssocket.accept() except socket.timeout: pass # don't exit on an interupt due to a signal except socket.error: e = sys.exc_info()[1] if e.errno == errno.EINTR: pass if self._exiting: return # now do something with the clientsocket # in this case, we'll pretend this is a threaded server if csocket: ppc.start_thread("client_socket", self.crun, (csocket, )) except KeyboardInterrupt: pass except: self.logger.debug("Exception in listen method (possibly expected)", exc_info=True) finally: self.logger.debug("Closing server socket") self.ssocket.close() def crun(self, csocket): """Authenticates client and handles its jobs""" mysocket = pptransport.CSocketTransport(csocket, self.socket_timeout) #send PP version mysocket.send(version) #generate a random string srandom = "".join([random.choice(string.ascii_letters) for i in range(16)]) mysocket.send(srandom) answer = sha_new(ppc.b_(srandom+self.secret)).hexdigest() clientanswer = ppc.str_(mysocket.receive()) if answer != clientanswer: self.logger.warning("Authentication failed, client host=%s, port=%i" % csocket.getpeername()) mysocket.send("FAILED") csocket.close() return else: mysocket.send("OK") ctype = ppc.str_(mysocket.receive()) self.logger.debug("Control message received: " + ctype) self.ncon_add(1) try: if ctype == "STAT": #reset time at each new connection self.get_stats()["local"].time = 0.0 #open('/tmp/pp.debug', 'a+').write('STAT: \n') mysocket.send(str(self.get_ncpus())) #open('/tmp/pp.debug', 'a+').write('STAT: get_ncpus\n') while 1: mysocket.receive() #open('/tmp/pp.debug', 'a+').write('STAT: recvd\n') mysocket.send(str(self.get_stats()["local"].time)) #open('/tmp/pp.debug', 'a+').write('STAT: _\n') elif ctype=="EXEC": while 1: #open('/tmp/pp.debug', 'a+').write('EXEC: \n') sfunc = mysocket.creceive() #open('/tmp/pp.debug', 'a+').write('EXEC: '+repr((sfunc,))+'\n') sargs = mysocket.receive() #open('/tmp/pp.debug', 'a+').write('EXEC: '+repr((sargs,))+'\n') fun = self.insert(sfunc, sargs) sresult = fun(True) #open('/tmp/pp.debug', 'a+').write('EXEC: '+repr((sresult,))+'\n') mysocket.send(sresult) #open('/tmp/pp.debug', 'a+').write('EXEC: _\n') except: if self._exiting: return if pp.SHOW_EXPECTED_EXCEPTIONS: self.logger.debug("Exception in crun method (possibly expected)", exc_info=True) self.logger.debug("Closing client socket") csocket.close() self.ncon_add(-1) def broadcast(self): """Initiaates auto-discovery mechanism""" discover = ppauto.Discover(self) ppc.start_thread("server_broadcast", discover.run, ((self.host, self.port), (self.bcast, self.port))) def parse_config(file_loc): """ Parses a config file in a very forgiving way. """ # If we don't have configobj installed then let the user know and exit try: from configobj import ConfigObj except ImportError: ie = sys.exc_info()[1] #sysstderr = getattr(sys.stderr, 'buffer', sys.stderr) six.print_(("ERROR: You must have config obj installed to use" "configuration files. You can still use command line switches."), file=sys.stderr) sys.exit(1) if not os.access(file_loc, os.F_OK): six.print_("ERROR: Can not access %s." % arg, file=sys.stderr) sys.exit(1) # Load the configuration file config = ConfigObj(file_loc) # try each config item and use the result if it exists. If it doesn't # then simply pass and move along try: args['secret'] = config['general'].get('secret') except: pass try: autodiscovery = config['network'].as_bool('autodiscovery') except: pass try: args['interface'] = config['network'].get('interface', default="0.0.0.0") except: pass try: args['broadcast'] = config['network'].get('broadcast') except: pass try: args['port'] = config['network'].as_int('port') except: pass try: args['loglevel'] = config['general'].as_bool('debug') except: pass try: args['ncpus'] = config['general'].as_int('workers') except: pass try: args['proto'] = config['general'].as_int('proto') except: pass try: args['restart'] = config['general'].as_bool('restart') except: pass try: args['timeout'] = config['network'].as_int('timeout') except: pass try: args['socket_timeout'] = config['network'].as_int('socket_timeout') except: pass try: args['pid_file'] = config['general'].get('pid_file') except: pass # Return a tuple of the args dict and autodiscovery variable return args, autodiscovery def print_usage(): """Prints help""" print("Parallel Python Network Server (pp-" + version + ")") print("Usage: ppserver [-hdar] [-f format] [-n proto]"\ " [-c config_path] [-i interface] [-b broadcast]"\ " [-p port] [-w nworkers] [-s secret] [-t seconds]"\ " [-k seconds] [-P pid_file]") print("") print("Options: ") print("-h : this help message") print("-d : set log level to debug") print("-f format : log format") print("-a : enable auto-discovery service") print("-r : restart worker process after each"\ " task completion") print("-n proto : protocol number for pickle module") print("-c path : path to config file") print("-i interface : interface to listen") print("-b broadcast : broadcast address for auto-discovery service") print("-p port : port to listen") print("-w nworkers : number of workers to start") print("-s secret : secret for authentication") print("-t seconds : timeout to exit if no connections with "\ "clients exist") print("-k seconds : socket timeout in seconds") print("-P pid_file : file to write PID to") print("") print("To print server stats send %s to its main process (unix only). " % STAT_SIGNAL) print("") print("Due to the security concerns always use a non-trivial secret key.") print("Secret key set by -s switch will override secret key assigned by") print("pp_secret variable in .pythonrc.py") print("") print("Please visit http://www.parallelpython.com for extended up-to-date") print("documentation, examples and support forums") def create_network_server(argv): try: opts, args = getopt.getopt(argv, "hdarn:c:b:i:p:w:s:t:f:k:P:", ["help"]) except getopt.GetoptError: print_usage() raise args = {} autodiscovery = False log_level = logging.WARNING log_format = "%(asctime)s - %(name)s - %(levelname)s - %(message)s" for opt, arg in opts: if opt in ("-h", "--help"): print_usage() sys.exit() elif opt == "-c": args, autodiscovery = parse_config(arg) elif opt == "-d": log_level = logging.DEBUG pp.SHOW_EXPECTED_EXCEPTIONS = True elif opt == "-f": log_format = arg elif opt == "-i": args["interface"] = arg elif opt == "-s": args["secret"] = arg elif opt == "-p": args["port"] = int(arg) elif opt == "-w": args["ncpus"] = int(arg) elif opt == "-a": autodiscovery = True elif opt == "-r": args["restart"] = True elif opt == "-b": args["broadcast"] = arg elif opt == "-n": args["proto"] = int(arg) elif opt == "-t": args["timeout"] = int(arg) elif opt == "-k": args["socket_timeout"] = int(arg) elif opt == "-P": args["pid_file"] = arg log_handler = logging.StreamHandler() log_handler.setFormatter(logging.Formatter(log_format)) logging.getLogger("pp").setLevel(log_level) logging.getLogger("pp").addHandler(log_handler) server = _NetworkServer(**args) if autodiscovery: server.broadcast() return server def signal_handler(signum, stack): """Prints server stats when %s is received (unix only). """ % STAT_SIGNAL server.print_stats() if __name__ == "__main__": server = create_network_server(sys.argv[1:]) statsignal = getattr(signal, STAT_SIGNAL, None) if statsignal: signal.signal(statsignal, signal_handler) server.listen() #have to destroy it here explicitly otherwise an exception #comes out in Python 2.4 del server # Parallel Python Software: http://www.parallelpython.com ppft-ppft-1.6.6.4/ppft/transport.py000066400000000000000000000231651406150406700171770ustar00rootroot00000000000000# Parallel Python Software: http://www.parallelpython.com # Copyright (c) 2005-2012 Vitalii Vanovschi. # Copyright (c) 2015-2016 California Institute of Technology. # Copyright (c) 2016-2021 The Uncertainty Quantification Foundation. # All rights reserved. # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: # * Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # * Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # * Neither the name of the author nor the names of its contributors # may be used to endorse or promote products derived from this software # without specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE # ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF # THE POSSIBILITY OF SUCH DAMAGE. """ Parallel Python Software, PP Transport http://www.parallelpython.com - updates, documentation, examples and support forums """ import logging import os import socket import struct import six copyright = """Copyright (c) 2005-2012 Vitalii Vanovschi. Copyright (c) 2015-2016 California Institute of Technology. Copyright (c) 2016-2021 The Uncertainty Quantification Foundation.""" __version__ = version = "1.6.6.4" # compatibility with Python 2.6 try: import hashlib sha_new = hashlib.sha1 md5_new = hashlib.md5 except ImportError: import sha import md5 sha_new = sha.new md5_new = md5.new from . import common as ppc class Transport(object): def send(self, msg): raise NotImplemented("abstact function 'send' must be implemented "\ "in a subclass") def receive(self, preprocess=None): raise NotImplemented("abstact function 'receive' must be implemented "\ "in a subclass") def authenticate(self, secret): remote_version = ppc.str_(self.receive()) if version != remote_version: logging.error("PP version mismatch (local: pp-%s, remote: pp-%s)" % (version, remote_version)) logging.error("Please install the same version of PP on all nodes") return False srandom = ppc.b_(self.receive()) secret = ppc.b_(secret) answer = sha_new(srandom+secret).hexdigest() self.send(answer) response = ppc.b_(self.receive()) if response == ppc.b_("OK"): return True else: return False def close(self): pass def _connect(self, host, port): pass class CTransport(Transport): """Cached transport """ rcache = {} def hash(self, msg): return md5_new(ppc.b_(msg)).hexdigest() def csend(self, msg): #if hasattr(self, 'w'): # open('/tmp/pp.debug', 'a+').write(repr(('cs', self.w, msg))+'\n') #else: # open('/tmp/pp.debug', 'a+').write(repr(('cs', self.socket, msg))+'\n') msg = ppc.b_(msg) hash1 = self.hash(msg) if hash1 in self.scache: self.send(ppc.b_("H" + hash1)) else: self.send(ppc.b_("N") + msg) self.scache[hash1] = True def creceive(self, preprocess=None): msg = self.receive() #if hasattr(self, 'r'): # open('/tmp/pp.debug', 'a+').write(repr(('cr', self.r, msg))+'\n') #else: # open('/tmp/pp.debug', 'a+').write(repr(('cr', self.socket, msg))+'\n') msg = ppc.b_(msg) if msg[:1] == ppc.b_('H'): hash1 = ppc.str_(msg[1:]) else: msg = msg[1:] hash1 = self.hash(msg) if preprocess is None: preprocess = lambda x:x self.rcache[hash1] = tuple(map(preprocess, (msg, )))[0] return self.rcache[hash1] class PipeTransport(Transport): def __init__(self, r, w): #open('/tmp/pp.debug', 'a+').write(repr((r,w))+'\n') self.scache = {} self.exiting = False if isinstance(r, ppc.file) and isinstance(w, ppc.file): self.r = r self.w = w else: raise TypeError("Both arguments of PipeTransport constructor " \ "must be file objects") if six.PY3 and hasattr(self.w, 'buffer'): self.wb = self.w.buffer self.has_wb = True else: self.wb = self.w self.has_wb = False if six.PY3 and hasattr(self.r, 'buffer'): self.rb = self.r.buffer self.has_rb = True else: self.rb = self.r self.has_rb = False def send(self, msg): #l = len(ppc.b_(msg)) if (self.has_wb or self.w.mode == 'wb') else len(ppc.str_(msg)) #open('/tmp/pp.debug', 'a+').write(repr(('s', l, self.w, msg))+'\n') if self.has_wb or self.w.mode == 'wb': msg = ppc.b_(msg) self.wb.write(struct.pack("!Q", len(msg))) self.w.flush() else: #HACK: following may be > 8 bytes, needed for len(msg) >= 256 msg = ppc.str_(msg) self.wb.write(ppc.str_(struct.pack("!Q", len(msg)))) self.w.flush() self.wb.write(msg) self.w.flush() def receive(self, preprocess=None): e_size = struct.calcsize("!Q") # 8 c_size = struct.calcsize("!c") # 1 r_size = 0 stub = ppc.b_("") if (self.has_rb or self.r.mode == 'rb') else "" data = stub while r_size < e_size: msg = self.rb.read(e_size-r_size) #l = len(msg) #open('/tmp/pp.debug', 'a+').write(repr(('_r', l, self.r, msg))+'\n') if msg == stub: raise RuntimeError("Communication pipe read error") if stub == "" and msg.startswith('['): #HACK to get str_ length while not msg.endswith('{B}'): msg += self.rb.read(c_size) r_size += len(msg) data += msg e_size = struct.unpack("!Q", ppc.b_(data))[0] # get size of msg r_size = 0 data = stub while r_size < e_size: msg = self.rb.read(e_size-r_size) #l = len(msg) #open('/tmp/pp.debug', 'a+').write(repr(('r_', l, self.r, msg))+'\n') if msg == stub: raise RuntimeError("Communication pipe read error") r_size += len(msg) data += msg data = ppc.b_(data) if preprocess is None: preprocess = lambda x:x return tuple(map(preprocess, (data, )))[0] def close(self): self.w.close() self.r.close() class SocketTransport(Transport): def __init__(self, socket1, socket_timeout): if socket1: self.socket = socket1 else: self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self.socket.settimeout(socket_timeout) self.scache = {} def send(self, data): #l = len(ppc.b_(data)) #open('/tmp/pp.debug', 'a+').write(repr(('ss', l, self.socket, data))+'\n') data = ppc.b_(data) size = struct.pack("!Q", len(data)) t_size = struct.calcsize("!Q") s_size = ppc.long(0) while s_size < t_size: p_size = self.socket.send(size[s_size:]) if p_size == 0: raise RuntimeError("Socket connection is broken") s_size += p_size t_size = len(data) s_size = ppc.long(0) while s_size < t_size: p_size = self.socket.send(data[s_size:]) if p_size == 0: raise RuntimeError("Socket connection is broken") s_size += p_size def receive(self, preprocess=None): e_size = struct.calcsize("!Q") r_size = 0 stub = ppc.b_("") data = stub while r_size < e_size: msg = self.socket.recv(e_size-r_size) #l = len(msg) #open('/tmp/pp.debug', 'a+').write(repr(('_sr', l, self.socket, msg))+'\n') if msg == stub: raise RuntimeError("Socket connection is broken") r_size += len(msg) data += msg e_size = struct.unpack("!Q", ppc.b_(data))[0] # get size of msg r_size = 0 data = stub while r_size < e_size: msg = self.socket.recv(e_size-r_size) #l = len(msg) #open('/tmp/pp.debug', 'a+').write(repr(('sr_', l, self.socket, msg))+'\n') if msg == stub: raise RuntimeError("Socket connection is broken") r_size += len(msg) data += msg data = ppc.b_(data) return data def close(self): self.socket.close() def _connect(self, host, port): self.socket.connect((host, port)) class CPipeTransport(PipeTransport, CTransport): pass class CSocketTransport(SocketTransport, CTransport): pass # Parallel Python Software: http://www.parallelpython.com ppft-ppft-1.6.6.4/ppft/worker.py000066400000000000000000000014561406150406700164530ustar00rootroot00000000000000#!/usr/bin/env python # # Author: Mike McKerns (mmckerns @caltech and @uqfoundation) # Copyright (c) 2015-2016 California Institute of Technology. # Copyright (c) 2016-2021 The Uncertainty Quantification Foundation. # License: 3-clause BSD. The full license text is available at: # - https://github.com/uqfoundation/ppft/blob/master/LICENSE """ ppft worker: ppworker to communicate with ppserver """ import sys if sys.version_info[0] == 2: from pp.__main__ import * from pp.__main__ import _WorkerProcess, __version__ else: from ppft.__main__ import * from ppft.__main__ import _WorkerProcess, __version__ if __name__ == "__main__": # add the directory with worker.py to the path sys.path.append(os.path.dirname(__file__)) wp = _WorkerProcess() wp.run() # EOF ppft-ppft-1.6.6.4/setup.py000077500000000000000000000157711406150406700153410ustar00rootroot00000000000000#!/usr/bin/env python # Parallel Python setup script # For the latest version of the Parallel Python # software visit: http://www.parallelpython.com """ Standard build tool for python libraries. """ import os, os.path import sys # drop support for older python unsupported = None if sys.version_info < (2, 7): unsupported = 'Versions of Python before 2.7 are not supported' elif (3, 0) <= sys.version_info < (3, 6): unsupported = 'Versions of Python before 3.6 are not supported' if unsupported: raise ValueError(unsupported) pyversion = sys.version_info[0] pkgdir = 'pp' if pyversion == 2 else 'ppft' _pkgdir = pkgdir # 'src' _server = os.path.join('ppft', 'server') # import shutil # shutil.copytree(pkgdir, _pkgdir) try: from setuptools import setup has_setuptools = True except ImportError: from distutils.core import setup has_setuptools = False stable_version = '1.6.6.4' target_version = '1.6.6.4' is_release = stable_version == target_version VERSION = stable_version if is_release else target_version + '.dev0' # os.chdir(pkgdir) # sys.path.insert(0, '.') # from ppcommon import __version__ as VERSION # sys.path.pop(0) # if os.path.exists('ppcommon.pyc'): os.remove('ppcommon.pyc') # os.chdir('..') LONG_DESCRIPTION = \ """------------------------------------- ppft: distributed and parallel python ------------------------------------- About Ppft ========== ``ppft`` is a fork of Parallel Python, and is developed as part of ``pathos``: https://github.com/uqfoundation/pathos Parallel Python module (``pp``) provides an easy and efficient way to create parallel-enabled applications for SMP computers and clusters. ``pp`` module features cross-platform portability and dynamic load balancing. Thus application written with ``pp`` will parallelize efficiently even on heterogeneous and multi-platform clusters (including clusters running other application with variable CPU loads). Visit http://www.parallelpython.com for further information. ``ppft`` is part of ``pathos``, a python framework for heterogeneous computing. ``ppft`` is in active development, so any user feedback, bug reports, comments, or suggestions are highly appreciated. A list of issues is located at https://github.com/uqfoundation/ppft/issues, with a legacy list maintained at https://uqfoundation.github.io/project/pathos/query. NOTE: ``ppft`` installs as ``pp``. If ``pp`` is installed, it should be uninstalled before ``ppft`` is installed -- otherwise, ``import pp`` will likely not find the ``ppft`` fork. Major Changes: ============== - ``pip`` and ``setuptools`` support - support for python 3 - enhanced serialization, using ``dill.source`` Current Release =============== This documentation is for version ``ppft-1.6.6.4`` (a fork of ``pp-1.6.6``). The latest released version of ``ppft`` is available from:: https://pypi.org/project/ppft ``pp`` and ``ppft`` are distributed under a BSD-like license. Development Version =================== You can get the latest development version with all the shiny new features at:: https://github.com/uqfoundation If you have a new contribution, please submit a pull request. Installation ============ ``ppft`` is packaged to install from source, so you must download the tarball, unzip, and run the installer:: [download] $ tar -xvzf ppft-1.6.6.4.tgz $ cd ppft-1.6.6.4 $ python setup.py build $ python setup.py install You will be warned of any missing dependencies and/or settings after you run the "build" step above. Alternately, ``ppft`` can be installed with ``pip`` or ``easy_install``:: $ pip install ppft NOTE: ``ppft`` installs as ``pp``. If ``pp`` is installed, it should be uninstalled before ``ppft`` is installed -- otherwise, ``import pp`` will likely not find the ``ppft`` fork. Requirements ============ ``ppft`` requires:: - ``python``, **version == 2.7** or **version >= 3.6**, or ``pypy`` - ``six``, **version >= 1.7.3** Optional requirements:: - ``setuptools``, **version >= 0.6** - ``dill``, **version >= 0.3.4** More Information ================ Probably the best way to get started is to look at the set of example scripts in ``ppft.examples``. You can run the test suite with ``python -m ppft.tests``. ``ppft`` will create and execute jobs on local workers (automatically created using ``python -u -m ppft``). Additionally, remote servers can be created with ``ppserver`` (or ``python -m ppft.server``), and then jobs can be distributed to remote workers. See ``--help`` for more details on how to configure a server. Please feel free to submit a ticket on github, or ask a question on stackoverflow (**@Mike McKerns**). If you would like to share how you use ``ppft`` in your work, please send an email (to **mmckerns at uqfoundation dot org**). Citation ======== If you use ``ppft`` to do research that leads to publication, we ask that you acknowledge use of ``ppft`` by citing the following in your publication:: M.M. McKerns, L. Strand, T. Sullivan, A. Fang, M.A.G. Aivazis, "Building a framework for predictive science", Proceedings of the 10th Python in Science Conference, 2011; http://arxiv.org/pdf/1202.1056 Michael McKerns and Michael Aivazis, "pathos: a framework for heterogeneous computing", 2010- ; https://uqfoundation.github.io/project/pathos Please see https://uqfoundation.github.io/project/pathos or http://arxiv.org/pdf/1202.1056 for further information. """ kwds = { "name" : "ppft", "url" : "https://github.com/uqfoundation/ppft", "version" : VERSION, "download_url" : "https://github.com/uqfoundation/ppft/releases/download/ppft-%s/ppft-%s.tar.gz" % (stable_version, stable_version), "author" : "Mike McKerns", "maintainer" : "Mike McKerns", "packages" : ["ppft","pp","ppft.tests","ppft.server"], "package_dir" : {'ppft': _pkgdir,'pp': _pkgdir,'ppft.server': _server,'ppft.tests':'examples'}, "scripts" : ["%s/ppserver" % _server], "description" : "distributed and parallel python", "platforms" : ["Windows", "Linux", "Mac"], "long_description" : LONG_DESCRIPTION, "license" : "BSD-like", "classifiers" : [ "Development Status :: 5 - Production/Stable", "Intended Audience :: Developers", "Intended Audience :: Science/Research", "Programming Language :: Python :: 2", "Programming Language :: Python :: 2.7", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Topic :: Software Development", "Topic :: Scientific/Engineering", ], } if has_setuptools: kwds.update({ "zip_safe" : False, "extras_require" : {'dill': ['dill>=0.3.4']}, }) if has_setuptools and pyversion > 2: kwds.update({ "install_requires" : ['six>=1.7.3'], }) setup(**kwds) # if os.path.exists(_pkgdir): # shutil.rmtree(_pkgdir) ppft-ppft-1.6.6.4/tox.ini000066400000000000000000000011511406150406700151220ustar00rootroot00000000000000[tox] skip_missing_interpreters= True envlist = # py26 # py27 # py33 # py34 # py35 py36 py37 py38 py39 py310 # pypy27 pypy36 pypy37 [testenv] setenv = # recreate = True deps = # dill whitelist_externals = bash commands = {envpython} setup.py build {envpython} setup.py install bash -c "failed=0; for test in examples/__main__.py; do echo $test; \ {envpython} $test || failed=1; done; exit $failed" # bash -c "failed=0; for test in examples/reverse*md5.py; do echo $test; \ # {envpython} $test || failed=1; done; exit $failed"