pax_global_header00006660000000000000000000000064146046014210014510gustar00rootroot0000000000000052 comment=e1332b1b02b82bb0639e398bd1290cc2f6a3e536 execnet-2.1.1/000077500000000000000000000000001460460142100131445ustar00rootroot00000000000000execnet-2.1.1/.gitattributes000066400000000000000000000001251460460142100160350ustar00rootroot00000000000000# restructured text files forced to LF because of doc8 pre-commit hook. *.rst eol=lf execnet-2.1.1/.github/000077500000000000000000000000001460460142100145045ustar00rootroot00000000000000execnet-2.1.1/.github/dependabot.yaml000066400000000000000000000001661460460142100175000ustar00rootroot00000000000000version: 2 updates: - package-ecosystem: "github-actions" directory: "/" schedule: interval: "weekly" execnet-2.1.1/.github/workflows/000077500000000000000000000000001460460142100165415ustar00rootroot00000000000000execnet-2.1.1/.github/workflows/deploy.yml000066400000000000000000000022261460460142100205620ustar00rootroot00000000000000name: deploy on: workflow_dispatch: inputs: version: description: 'Release version' required: true default: '1.2.3' jobs: package: runs-on: ubuntu-latest env: SETUPTOOLS_SCM_PRETEND_VERSION: ${{ github.event.inputs.version }} steps: - uses: actions/checkout@v4 - name: Build and Check Package uses: hynek/build-and-inspect-python-package@v2.2 deploy: needs: package runs-on: ubuntu-latest environment: deploy permissions: id-token: write # For PyPI trusted publishers. contents: write # For tag. steps: - uses: actions/checkout@v4 - name: Download Package uses: actions/download-artifact@v4 with: name: Packages path: dist - name: Publish package to PyPI uses: pypa/gh-action-pypi-publish@v1.8.14 - name: Push tag run: | git config user.name "pytest bot" git config user.email "pytestbot@gmail.com" git tag --annotate --message=v${{ github.event.inputs.version }} v${{ github.event.inputs.version }} ${{ github.sha }} git push origin v${{ github.event.inputs.version }} execnet-2.1.1/.github/workflows/test.yml000066400000000000000000000022031460460142100202400ustar00rootroot00000000000000name: test on: push: branches: - "master" - "test-me-*" pull_request: branches: - "master" # Cancel running jobs for the same workflow and branch. concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: package: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - name: Build and Check Package uses: hynek/build-and-inspect-python-package@v2.2 test: needs: [package] runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: os: [ windows-latest, ubuntu-latest ] python: [ "3.8","3.10","3.11","3.12", "pypy-3.8" ] steps: - uses: actions/checkout@v4 with: fetch-depth: 0 - name: Download Package uses: actions/download-artifact@v4.1.4 with: name: Packages path: dist - name: Set up Python uses: actions/setup-python@v5 with: python-version: ${{ matrix.python }} - name: Install tox run: pip install tox - name: Test shell: bash run: | tox run -e py --installpkg `find dist/*.tar.gz` execnet-2.1.1/.gitignore000066400000000000000000000002741460460142100151370ustar00rootroot00000000000000doc/_build build/ src/execnet/_version.py dist/ .pytest_cache/ .eggs/ *.pyc *$py.class *.orig *~ *.swp .tox lib/ bin/ include/ .Python .env/ .cache/ .vagrant/ .vagrant.d/ .config/ .local/ execnet-2.1.1/.pre-commit-config.yaml000066400000000000000000000015451460460142100174320ustar00rootroot00000000000000repos: - repo: https://github.com/codespell-project/codespell rev: v2.2.6 hooks: - id: codespell - repo: https://github.com/asottile/blacken-docs rev: 1.16.0 hooks: - id: blacken-docs additional_dependencies: [black==22.12.0] - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.5.0 hooks: - id: check-yaml - repo: https://github.com/astral-sh/ruff-pre-commit rev: v0.3.5 hooks: - id: ruff args: [ --fix ] exclude: "^doc/" - id: ruff-format - repo: https://github.com/PyCQA/doc8 rev: 'v1.1.1' hooks: - id: doc8 args: ["--ignore", "D001"] - repo: https://github.com/pre-commit/mirrors-mypy rev: 'v1.9.0' hooks: - id: mypy additional_dependencies: - pytest - types-pywin32 - types-gevent execnet-2.1.1/.readthedocs.yaml000066400000000000000000000001631460460142100163730ustar00rootroot00000000000000version: 2 build: os: ubuntu-22.04 tools: python: "3" python: install: - method: pip path: . execnet-2.1.1/CHANGELOG.rst000066400000000000000000000421771460460142100152000ustar00rootroot000000000000002.1.1 (2024-04-08) ------------------ * `#267 `__ Fixed regression in 2.1.0 where the ``strconfig`` argument to ``load``/``loads`` is ignored. 2.1.0 (2024-04-05) ------------------ * `#243 `__: Added ``main_thread_only`` execmodel which is derived from the thread execmodel and only executes ``remote_exec`` calls in the main thread. Callers of ``remote_exec`` must use the returned channel to wait for a task to complete before they call remote_exec again, otherwise the ``remote_exec`` call will fail with a ``concurrent remote_exec would cause deadlock`` error. The main_thread_only execmodel provides solutions for `#96 `__ and `pytest-dev/pytest-xdist#620 `__ (pending a new `pytest-xdist` release). Also fixed ``init_popen_io`` to use ``closefd=False`` for shared stdin and stdout file descriptors, preventing ``Bad file descriptor`` errors triggered by test_stdouterrin_setnull. * The library is now typed and the typing is exposed to type-checkers. * Re-exported ``Gateway``, ``Channel``, ``DumpError`` and ``LoadError`` from ``execnet``. The constructors are private. * Fixed ``GatewayBase.join()`` timeout argument getting ignored. * Removed support for Python 3.7. * Added official support for Python 3.12. 2.0.2 (2023-07-09) ------------------ * Re-release without code changes, just to include ``tox.ini`` into the source distribution. 2.0.1 (2023-07-08) ------------------ * Re-release without code changes, just to include docs and tests into the source distribution. 2.0.0 (2023-07-06) ------------------ * Removed support for Python < 3.7. - Applied ``pyupgrade --py37-plus``. - Minimal ``mypy`` fixes and dropped Python 2 support code. * Migrated packaging to ``hatch``. * Dropped deprecated APIs of old makegateway names. * Removed ``py`` testing dependency. * Explicitly pass ``encoding`` when opening files in the gateway to get rid of warnings when using ``PYTHONWARNDEFAULTENCODING=1`` (#195). * Fixed error when loading source code files from a path containing non-ascii characters. 1.9.0 (2021-06-13) ------------------ * Removed the ``apipkg`` dependency. 1.8.1 (2021-05-27) ------------------ * Update calls of `threading.Event.isSet()` to `threading.Event.is_set()`, which avoids a deprecation warning with Python 3.10. 1.8.0 (2021-01-27) ------------------ * Dropped support for Python 3.4. * `#118 `__: Fixed internal leak that should make ``execnet`` execute remote code in the main thread more often; previously it would sometimes spawn a thread to execute a ``remote_exec`` call, even when the caller didn't issue multiple ``remote_exec`` calls at the same time. Some frameworks require code to execute in the main thread, so the previous behavior would break them on occasion (see `pytest-dev/pytest-xdist#620 `__ for an example). * `#115 `__: Current working directory is now restored when calling ``script/socketserver.py``. The script now also loops by default when called from the command-line. 1.7.1 (2019-08-28) ------------------ * `#108 `__: Revert ``linecache`` optimization introduced in ``1.7.0`` which broke remote execution. 1.7.0 (2019-08-08) ------------------ * `#102 `__: Show paths in stack traces generated by ``remote_exec()``. * `#100 `__: Fix flaky hangs in ``workerpool.waitall``. 1.6.1 (2019-07-22) ------------------ * `#98 `__: Internal change to avoid using deprecated ``funcargs`` name in pytest 5+. 1.6.0 (2019-03-31) ------------------ * ``execnet`` no longer supports Python 2.6 and 3.3 (#85). Users of those Python versions using a recent enough ``pip`` should not be affected, as ``pip`` will only install ``1.5.0`` for them. * Update test suite to support ``pytest>4``. 1.5.0 (2017-10-16) ------------------ - support shell escaping in python pathnames of popen. Eugene Ciurana discovered that execnet breaks if you use pathnames with spaces in a "python=" part of a spec. We now use shlex.split to split the string. There is a potential for regressions if you used quote or escape sequences as part of your python command. - Only insert importdir into sys.path if it is not already in the path. This prevents a bug when using enum34 with python 3.6 and pytest-xdist. The issue is that enum34 installs an 'enum' module in site-packages which is normally shadowed by the stdlib version of enum, however in gateway_bootstrap.py site-packages is added at the front the the search path. This means on the workers enum34 is hit for import enum which in turn causes import re to fail (as it makes use of the new enum features in 3.6). - fix #49 - use inspect.getfullargspec if possible to avoid deprecationwarnings - fix #56 - use partials in safe_terminate to avoid a bad carried binding - fix spec parsing on Windows due to path containing '\' characters. 1.4.1 (2015-09-02) ------------------ - fix a regression of the Serializer created by the implied opcode ordering which resulted in a incompatible opcode mapping *warning* stored serialized objects created with 1.4.0 are incompatible with previous versions and future versions additionally stored serialized objects containing complex objects will have a incompatible opcode when read with execnet < 1.4.0 and won't be loadable with execnet 1.4.0 either its strongly suggested to avoid using the Serializer of execnet 1.4.0 this affects devpi and the external pytest-cache plugin 1.4 ---- - de-vendor apipkg and use the pypi dependency instead (this also fixes the bpython interaction issues) - Fix issue38: provide ability to connect to Vagrant VMs easily using :code:`vagrant_ssh=default` or :code:`vagrant_ssh=machinename` this feature is experimental and will be refined in future releases. Thanks Christian Theune for the discussion and the initial pull request. - add support for serializing the "complex" type. Thanks Sebastian Koslowski. 1.3 -------------------------------- - fix issue33: index.txt to correctly mention MIT instead of GPL. - fix issue35: adapt some doctests, fix some channel tests for py3. - use subprocess32 when available for python < 3. - try to be a bit more careful when interpreter is shutting down to avoid random exceptions, thanks Alfredo Deza. - ignore errors on orphan file removal when rsyncing - fix issue34: limit use of import based bootstrap 1.2 -------------------------------- - fix issue22 -- during interpreter shutdown don't throw an exception when we can't send a termination sequence anymore as we are about to die anyway. - fix issue24 -- allow concurrent creation of gateways by guarding automatic id creation by a look. Thanks tlecomte. - majorly refactor internal thread and IO handling. execnet can now operate on different thread models, defaults to "thread" but allows for eventlet and gevent if it is installed. - gateway.remote_exec() will now execute in multiple threads on the other side by default. The previous necessity of running "gateway.remote_init_threads()" to allow for such concurrency is gone. The latter method is now a no-op and will be removed in future versions of execnet. - fix issue20: prevent AttributError at interpreter shutdown by not trying to send close/last_message messages if the world around is half destroyed. - fix issue21: allow to create local gateways with sudo aka makegateway("popen//python=sudo python"). Thanks Alfredo Deza for the PR. - streamline gateway termination and simplify proxy implementation. add more internal tracing. - if execution hangs in computation, we now try to send a SIGINT to ourselves on Unix platforms instead of just calling thread.interrupt_main() - change license from GPL to MIT - introduce execnet.dump/load variants of dumps/loads serializing/unserializing mechanism. - improve channel.receive() communication latency on python2 by changing the default timeout of the underlying Queue.get to a regular None instead of the previous default -1 which caused an internal positive timeout value (a hack probably introduced to allow CTRL-C to pass through for 30%) - refine internal gateway exit/termination procedure and introduce group.terminate(timeout) which will attempt to kill all subprocesses that did not terminate within time. - EOFError on channel.receive/waitclose if the other side unexpectedly went away. When a gateway exits it now internally sends an explicit termination message instead of abruptly closing. - introduce a timeout parameter to channel.receive() and default to periodically internally wake up to let KeyboardInterrupts pass through. - EXECNET_DEBUG=2 will cause tracing to go to stderr, which with popen worker gateways will relay back tracing to the instantiator process. 1.0.0 -------------------------------- * introduce execnet.Group for managing gateway creation and termination. Introduce execnet.default_group through which all "global" calls are routed. cleanup gateway termination. All Gateways get an id through which they can be retrieved from a group object. * deprecate execnet.XYZGateway in favour of direct makegateway() calls. * refine socketserver-examples, experimentally introduce a way to indirectly setup a socket server ("installvia") through a gateway url. * refine and automatically test documentation examples 1.0.0b3 -------------------------------- * fix EXECNET_DEBUG to work with win32 * add support for serializing longs, sets and frozensets (thanks Benjamin Peterson) * introduce remote_status() method which on the low level gives information about the remote side of a gateway * disallow explicit close in remote_exec situation * perform some more detailed tracing with EXECNET_DEBUG 1.0.0b2 -------------------------------- * make internal protocols more robust against serialization failures * fix a serialization bug with nested tuples containing empty tuples (thanks to ronny for discovering it) * setting the environment variable EXECNET_DEBUG will generate per process trace-files for debugging 1.0.0b1 ---------------------------- * added new examples for NumPy, Jython, IronPython * improved documentation * include apipkg.py for lazy-importing * integrated new serializer code from Benjamin Peterson * improved support for Jython-2.5.1 1.0.0alpha2 ---------------------------- * improve documentation, new website * use sphinx for documentation, added boilerplate files and setup.py * fixes for standalone usage, adding boilerplate files * imported py/execnet and made it work standalone execnet-2.1.1/LICENSE000066400000000000000000000020361460460142100141520ustar00rootroot00000000000000 Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. execnet-2.1.1/README.rst000066400000000000000000000030461460460142100146360ustar00rootroot00000000000000execnet: distributed Python deployment and communication ======================================================== .. image:: https://img.shields.io/pypi/v/execnet.svg :target: https://pypi.org/project/execnet/ .. image:: https://anaconda.org/conda-forge/execnet/badges/version.svg :target: https://anaconda.org/conda-forge/execnet .. image:: https://img.shields.io/pypi/pyversions/execnet.svg :target: https://pypi.org/project/execnet/ .. image:: https://github.com/pytest-dev/execnet/workflows/test/badge.svg :target: https://github.com/pytest-dev/execnet/actions?query=workflow%3Atest .. image:: https://img.shields.io/badge/code%20style-black-000000.svg :target: https://github.com/python/black .. _execnet: https://execnet.readthedocs.io execnet_ provides carefully tested means to ad-hoc interact with Python interpreters across version, platform and network barriers. It provides a minimal and fast API targeting the following uses: * distribute tasks to local or remote processes * write and deploy hybrid multi-process applications * write scripts to administer multiple hosts Features -------- * zero-install bootstrapping: no remote installation required! * flexible communication: send/receive as well as callback/queue mechanisms supported * simple serialization of python builtin types (no pickling) * grouped creation and robust termination of processes * interoperable between Windows and Unix-ish systems. * integrates with different threading models, including standard os threads, eventlet and gevent based systems. execnet-2.1.1/RELEASING.rst000066400000000000000000000020111460460142100152010ustar00rootroot00000000000000================= Releasing execnet ================= This document describes the steps to make a new ``execnet`` release. Version ------- ``master`` should always be green and a potential release candidate. ``execnet`` follows semantic versioning, so given that the current version is ``X.Y.Z``, to find the next version number one needs to look at the ``CHANGELOG.rst`` file: - If there any new feature, then we must make a new **minor** release: next release will be ``X.Y+1.0``. - Otherwise it is just a **bug fix** release: ``X.Y.Z+1``. Steps ----- To publish a new release ``X.Y.Z``, the steps are as follows: #. Create a new branch named ``release-X.Y.Z`` from the latest ``master``. #. Update the ``CHANGELOG.rst`` file with the new release information. #. Commit and push the branch to ``upstream`` and open a PR. #. Once the PR is **green** and **approved**, start the ``deploy`` workflow manually from the branch ``release-VERSION``, passing ``VERSION`` as parameter. #. Merge the release PR to ``master``. execnet-2.1.1/Vagrantfile000066400000000000000000000057161460460142100153420ustar00rootroot00000000000000# -*- mode: ruby -*- # vi: set ft=ruby : # All Vagrant configuration is done below. The "2" in Vagrant.configure # configures the configuration version (we support older styles for # backwards compatibility). Please don't change it unless you know what # you're doing. Vagrant.configure("2") do |config| # The most common configuration options are documented and commented below. # For a complete reference, please see the online documentation at # https://docs.vagrantup.com. # Every Vagrant development environment requires a box. You can search for # boxes at https://vagrantcloud.com/search. config.vm.box = "generic/debian11" # Disable automatic box update checking. If you disable this, then # boxes will only be checked for updates when the user runs # `vagrant box outdated`. This is not recommended. # config.vm.box_check_update = false # Create a forwarded port mapping which allows access to a specific port # within the machine from a port on the host machine. In the example below, # accessing "localhost:8080" will access port 80 on the guest machine. # NOTE: This will enable public access to the opened port # config.vm.network "forwarded_port", guest: 80, host: 8080 # Create a forwarded port mapping which allows access to a specific port # within the machine from a port on the host machine and only allow access # via 127.0.0.1 to disable public access # config.vm.network "forwarded_port", guest: 80, host: 8080, host_ip: "127.0.0.1" # Create a private network, which allows host-only access to the machine # using a specific IP. # config.vm.network "private_network", ip: "192.168.33.10" # Create a public network, which generally matched to bridged network. # Bridged networks make the machine appear as another physical device on # your network. # config.vm.network "public_network" # Share an additional folder to the guest VM. The first argument is # the path on the host to the actual folder. The second argument is # the path on the guest to mount the folder. And the optional third # argument is a set of non-required options. # config.vm.synced_folder "../data", "/vagrant_data" # Provider-specific configuration so you can fine-tune various # backing providers for Vagrant. These expose provider-specific options. # Example for VirtualBox: # # config.vm.provider "virtualbox" do |vb| # # Display the VirtualBox GUI when booting the machine # vb.gui = true # # # Customize the amount of memory on the VM: # vb.memory = "1024" # end # # View the documentation for the provider you are using for more # information on available options. # Enable provisioning with a shell script. Additional provisioners such as # Ansible, Chef, Docker, Puppet and Salt are also available. Please see the # documentation for more information about their specific syntax and use. # config.vm.provision "shell", inline: <<-SHELL # apt-get update # apt-get install -y apache2 # SHELL end execnet-2.1.1/doc/000077500000000000000000000000001460460142100137115ustar00rootroot00000000000000execnet-2.1.1/doc/Makefile000066400000000000000000000061751460460142100153620ustar00rootroot00000000000000# Makefile for Sphinx documentation # # You can set these variables from the command line. SPHINXOPTS = SPHINXBUILD = sphinx-build PAPER = BUILDDIR = _build # Internal variables. PAPEROPT_a4 = -D latex_paper_size=a4 PAPEROPT_letter = -D latex_paper_size=letter ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . .PHONY: help clean html dirhtml pickle json htmlhelp qthelp latex changes linkcheck doctest help: @echo "Please use \`make ' where is one of" @echo " html to make standalone HTML files" @echo " dirhtml to make HTML files named index.html in directories" @echo " pickle to make pickle files" @echo " json to make JSON files" @echo " htmlhelp to make HTML files and a HTML help project" @echo " qthelp to make HTML files and a qthelp project" @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" @echo " changes to make an overview of all changed/added/deprecated items" @echo " linkcheck to check all external links for integrity" @echo " doctest to run all doctests embedded in the documentation (if enabled)" clean: -rm -rf $(BUILDDIR)/* install: clean html rsync -avz $(BUILDDIR)/html/ code:www-execnet/ html: $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html @echo @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." dirhtml: $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml @echo @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." pickle: $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle @echo @echo "Build finished; now you can process the pickle files." json: $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json @echo @echo "Build finished; now you can process the JSON files." htmlhelp: $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp @echo @echo "Build finished; now you can run HTML Help Workshop with the" \ ".hhp project file in $(BUILDDIR)/htmlhelp." qthelp: $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp @echo @echo "Build finished; now you can run "qcollectiongenerator" with the" \ ".qhcp project file in $(BUILDDIR)/qthelp, like this:" @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/execnet.qhcp" @echo "To view the help file:" @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/execnet.qhc" latex: $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex @echo @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." @echo "Run \`make all-pdf' or \`make all-ps' in that directory to" \ "run these through (pdf)latex." changes: $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes @echo @echo "The overview file is in $(BUILDDIR)/changes." linkcheck: $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck @echo @echo "Link check complete; look for any errors in the above output " \ "or in $(BUILDDIR)/linkcheck/output.txt." doctest: $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest @echo "Testing of doctests in the sources finished, look at the " \ "results in $(BUILDDIR)/doctest/output.txt." execnet-2.1.1/doc/__init__.py000066400000000000000000000000021460460142100160120ustar00rootroot00000000000000# execnet-2.1.1/doc/_static/000077500000000000000000000000001460460142100153375ustar00rootroot00000000000000execnet-2.1.1/doc/_static/basic1.png000066400000000000000000000570731460460142100172230ustar00rootroot00000000000000PNG  IHDR?iCCPICC ProfilexXwP;vzͥH^.{/*u)Rwi4Af""W)""z"( o7w<_NNIrNP$ ,<`jwsF ?EYA)KAzI#QA 'p бvȈHD~A>'@@ur A| b-DǾ[ GerXϟLt40OT @ ]/ e9>CL aU;h _8_ڼZ!zh*ʛ,F`76rXY^ j~QM^8DW{kο%P9(!NF,{X)+\<ܼgjTEwM;$ͤKȴi**v~TUWԈV9뭞~C#&ݦf$x9mmC;|tbv&8F˹O;'շy*$:^M0u-#|P'o`{>1}ui4|:1Ա#N>Yv*=3v_ U{Rj˫]&ym߱CCKn FGfGW~??!r駜{&e&- F.}yz`qx.e>duqڲU5s}'r fgre~jCsO?WA(Ix^, o. )%}E[C~ǎ IEsyequEHFE]z/ ІD#Rqvq s! eKS+OPۓvE9u8wvqp๺#+'/CVPTO 1 5 Fl"̢ccߖsoIÇԤI=撮L8'O:etZ<d-:gP--SsvQ`̹seb O+KզD.\lTt9Q _׫WD][  o77n ݂{ZW=(n`RG'9O9>S{.|kzoND^398fmCj/eO@0gI/ _Ŧئ־}781X~ޱu\|͕-,Ư+1@A6n0V:p:W`cI g* wq '@ sA< OB!D&:1Ej!Gא(R@P h>:]øb1ϱX3l: g;bb00221^f\cgČ`dndcfyêZ6ơQ)YǕw1O1o #~{1 NDHUK8H#R%j,:!m,N+!#9-U}4tL܈Q >”c2bNox)y9EퟻiLZ|%˘3;sM`> +h&H>Ar"w#o(yT0 ;ZN /iIJbV(.Q1 ˔9y%j->.&N.nN$]<뼏N  gDŌ"lHU3s2K< UE3%/b37U^. k4ô;uVtwy,3vwrWZL[[gL98y8ʺy`<@Eg[N9Nӌ'?t,taZ= 'OdU@=[^jx|:e+__ը[ٲ|TGm /Q۠'ξ~+A7jo cz0<4:Z>}KQ>3~UT7{άހ/ ;GN/.ϯTpZE^c[k^aÃ|i3`"QB)T &<%,4 dM%8 < 8JC-VLkC evtY=ₜ`|lSXƇCm LnU(g BexZ%cK#/Il\f) 3K(0b>-*>G@$EXH @=[QNT*0T[aڿsеSSa7 )]5Id8?ih5Zc s6Du`VT?ZI;&c u؜g 80xru VEGDɇ,HzD8H  ?d<I*D?qRДpv pHYs   IDATx] \TE?(EA M-\K+I\0LB& ZAX]+_ {r).iVX.%)a˲.g9s33T*C-r-V։rrrrx*6 6۴b PPP,(lҊQPPP2@9@9@9`oMK+F9@9@9@!rB6-x*6 6۴bd7Z̜&9`ggGWVZ]vZhO0"dR* 7r=M<uuRs&N9+NYE˒ogh>BhH9`(xի~l-<>Gk@0Iw#~ % L!~2Q#~ydnV&9)n< #r6+T  .FzA}ɯTVp|"A )}$' իWz8ʁFGG>}p`XNsFD/: ԋV0@65;\_G.(Q\.8k3mLM|CdГqNGk^}v(-N 'ύN*3n姾 $\Nt!Cs>k Mj)a$!: AyK{thТ0d^>7>8@+FLv9ZQS~N}\N5֙Aǁ hIrʿ<5 8/-Y|g3Jg ~EoxDQXiL_tJO:Pށ2L(y?zOW>bfgE2.آ%E/b'C\k\~/^$39 LYJs1j1]n #̫Ote0&c)s_ K ДG/Fn¸ c/ IӱYOxz> z՗w_ ?p4mOFJR@RrкdƇF$ʚ_(#Dq[D\.!i#EHDEu 03 cCSH`8)4>=O,^&5Jm.A_$TTRI ̤.#aJ^JQAFQaL+"7)4!8)#dFKJ%+IHg Д"HEq9eebR 0EaqH:QDRa t&df&2L&#Rz]r/ 뮻 d4s̵ [,ȟH  eCFB ksfK-æ$X!$_ F+]M4S\tqnsŤqbުO,/NTTf~]'#=C+$ggPajݧtP˜ ZƨǡdCՆ [Fǘ駟klԊt738_mT*I yU(l_($mhJ!!Ž2E+$6j"%,Dr *V}t*L2NEHbj Uw%UXWJ$dfRӣ. \bY6Aآ9HQ&%A&aÒ+E>Dr.ݜ۽{74*4½{jj/B1bV;'*BJb ĩE~Xd_-5D!ujIJ ;W  S7wLI:mGݧ (Qpٺi~P/PdA6A` 0AfY= %.;!t8 p+#ol`iSˌGpfSOC}==$ ?հAȄuC9bRFdp߮#`;B(3LcݾMxn$z`܅u9r&%  u:U(qr]eݩෛLVG=ݖ~An$U30\礄{qA޷U<=@+3gΒ%K֮]b/qb1z="ݺ6]?Kπ6}, nRK*ԽZ_Υ|NP~>>9݅WUZF-+0͍>I -5|Nzs}F-Q0tnda,h8n%t{gZ^}]tw+:Txϫ>Qyd%ѧSDZ{vqDC9 dSG~*խm((:pm}mLS95s*Ol(. Xɯl茒ArkK7/9|֤>L(#.5pֺQs }>ʎs v8X?p2X)xir9-<1񪻎:g=@n43Zho6w\M^bXHLhm%ji0`CJV>MFs{$+kZ:e-3I &7FGʇu7۫q5Wi)>hƶdz믴3DlX6Ǧ%F.\vM!,޹80cVe/ H2%,gXS1c^r{F՚RfOȡyU )aHi,d'XO-QTPZG3v<DlbQukv}| iX .Y*7tP2}7sOs91<N\FbCX!gS-1QbaC9hONX{~DHN3Wݧ4Jix]AQufU$A#|Dx?FOyAwJ4^R(=oBZR:/ ;d`!v`%dKtfEZxP?f'oh`]tȡ:WWɔn>~)[w׊y}9GBcϞСUu.]ۥ[^_}Nʸyz{8Ч7*kOOoW(-M|MsNʫ}kMϨliq-sݯzMqṁw)ζ+)²_Qyg S}AKZO"ՕU.4E؆"n(NuJd{F6TkdTDTY*㾩1a۳g'2پ}&l 5$Ly)kdnX0M&'Nq<|n |&T_R*U0قh6RmY]{o.J;㏇jN|?<83jX-#*e'{ѝ"ӿa6B Gnx)Z¤j%@䧽Hwbފ}Xڨ`8AoURugժv|@ 1-]eÌ9`t[nӦM^kxIR9gؘ/ɜ[Q%.ٵ12 𶐱x YRKVd#f[ ˯drjM!}9c`8nh~MK%=ݼ, ,">glF03+n,C}chaq!uۥtiz΍PwK21F<Γ9P{bPǢg 3%UjbjTkΦ լ2-Hj;8q~Aϔwjʶ@S,by,xwz@Z9ŁjV9O>ѳVe²f%A+},dvlxBrBu'ք|\`SBH`8os0?(aV>Tr9@7F!`8خB7nC ըY٣Gb^-9A:É'K/FYv`e7 BBrn ˹Dғ3g˅-K:G7pñ&?:(z [L,X`nr\gmϘY]k#gD;_4{ V;^˳qvn9)qqPwe1PBz{7ra8.yɮ#`57>[n cW8,W^ou-tU.8s " V:cǎ!}ڃź mUEʚ5x(1DZ@G|p9np]qNh (m :tCԩSa|@I$tM?#N"!Ϝ9c٬P~}[OY2egU sEʂd4{L4f+`tRo3o3Msdp+\OWSS駟/@*a*ɓ5zh =Gօ162;{,GN$48%+V㘯ՄRXN` KNpJP1^S.Ha_R'K]% igWՕ% YY:ΖT%Ó6]STXF)yH#ʔ)p~DZS(č%'PD&̒sU^ w52 rN:iERe5_nthex뭷5wIty OSѦ/2ӚѨB8ĢI"HN:f¡XdB: Q|aêj,i`=PWۦA 6 aرc(Sa5X++~G"zI kje&/a9eQN.$>)KK͈ݞa {ȈԖOcMSJ*g3̒3>3K?߉aBI1y$g(s'S ;dž|ymYf0r]wif6o<ŋS48lȑ /r$o+m89bǰsν⋰`K/aN|'2[\߄sYvɁuLiFa;. T6-KYs kq#G461GKܘIk@$ .8~&%wg YoٚssĄJ2QKq Dr<$90}˖-'N@[_kMPrS=F.Yua=j(̼tϟ`M A:D`+bnƯ+1Y]Pt(Y(H*Ņ Ux151e1 QTZ(`\@,6 K.g(LKXCMs|`l;0הB΂ G"́`8nP 0!C`=77נF X穹Cl˘N!&a0A1n:a1]ȹH -I)s~q`` 2<?PF;Lb>&** =5J7N&x:j2a|M )z >ڱcVSn/D_QQr߬?Xl,|Gm9>IDuuuXx"֒(&+Iq`}…nL^PPXhL<(bmfvblRB2eѫ/tх7Wsؠ݌6+v3vMl~j'KcE5àK ;=9O(BVAᤔkׂGuVN7Ҡ li裏{1>3f0Vpu5yΦwd0 {*T Z`[c_:֭[gk`=hbnݺ=Shm9_K [pX(6|/ IDATB0K' Xnoo]B8օl nɦ2o`x 2q }]]KHhq-LC`rk Y 8V7F|&yȧY(Xb>aMDDDwޘ{ PE|^y; LցlDZ/--СC8K*q ǂÉV4.BO ! qɓI&Ra0?:>//; oz!|ݱ>Q?(e8ױcx[maXzi- @0t7_[nG <6Bj@Nʂ_zMwB8Ж4aHwSX :%Y*Df ݻ7yx>:qGJvr,2i*|q!;c>vP݉nKJLL3XQ)؂ #p8 qN,f՟.~ ̙3;~ f0? UoEhuR7OA/;z(Q mMvcO ۀ *<c ,[ ;v4͛7CpuL{gavST\Y{럽hekNsoi[X0`/8`ӖW:C(M' @!xrRc;ۀxҎ 7111uXv9k,f\.-,1\2G.L!0 5$a4Yݝ<ņ7량e$*<4ɔ(Im p0B  :Nq 3g$!X6e8;[YBbxS64/ @XL>,! vaيPaVZB\]]j(ysT=: C+ Pl`hX;KHjLw`/G!("yqhC~ڕQ$ # MHLzq/9">gC¦ ^e

im#.Yl 7|a&(ěrX 形bW ߿?`7>v0C'<1 9kZN*@RAPh,JB€ $<ck.4$Y)Ą&e|->(_rh&7lEdɟY9UI"ISy(_a8 9V-W,u&tp!аɰR]]@-^E)'t2J9L^8_wk~Y$l dl,hpy? 5zvٳP(1h=Gݖ ̯IPS82ɭR9+ ䷓d5dC> _l?V=WH ]EրIٴ7N }S+G֌ZwOi5˅xpsW8ic@%Xl` o!"մzuܠ:vgfu \_Xů(e)3-F8yl) ;k >%VI.富 '􌚵Qݬ4rr\0dK}B6#;~Wt⭲(є4?CEgىZ'n$7Ҁ˔É!8al`|Vܷ9$$,5\Ր!2$l]){8gs;6$r3'F"Gjސ:lRRr_d`氻] x^[{dC$KX93ޔl[-?gsXEغ O64-4r8vfR^mDPxpJvLUM>P순$Kcޏ Dѧ- ~p=xĞ\p+Hvz8-zO[jp iin-D~Gyqx)Z)#1l왲q e@⤌e%~I+hRlmd4Gͤ駟 nf+,sn>`aa!Tnf{$ 6M}>/[[*ݵ#:1ҬQbF(d tuDV1v}urm?ssU^:y@DXQW[.ĺ>fmd5iҤW_}Offf9?9sW_}e)^er+W4_QN8pUٶL>L(#gkg57PIY0j!ÄJ/IGJ&' MK0i቉W]u\;3Hv,? 63e Ü:S) 0ϥW`nv=k d0a1 ;jb\Nfa] >jV12J =z#G(J\\\xx8nSoϟ3fLNN΄ u9–-['X=@-L'N8~81f0|i>ް~[ (̝XzQR/-޳bw/ `5tq"`Byə; g#$%ǭusgǦ%F.\*\,d̍&h291c˘zt,Ā4tjyڪU0ׯ(%K^z֬YFɭ3裏p;x≞/BJh?h9rm݆ٳgϟ?Μ9?/:{sRӛ{vբjأZ0n}beuU#vUks:O _\]tvpgbא}$gxl)'OD1bF_xᅛot.&$$ d"P24aPP>~@g(e7biVN/^d6sL:Ͷe:_i|=  .̝;j.Ќ&o5|Ưm3k["_~Mv= ZVhR|w_/z8  +RN>A -02y~m<^^^wssM 1.NΝWVV+NМ7*޽{wa zfaG//**^6+b7:iCO仺HZop{o_?]uUm{}8rsws%˪ZX{M8Q(%fF |G(\ 2P>|||y:B-iӴ­h 3B<m67Q"~X,d$''CcLMFu.Ж 3S̒o޼I^W~pG&s,ܸqWwf\t` YZ@=s `=;*&xӕƬX!!@b;bMXv{dBsL c.x6kGuH6YbNk$@(`hӰЛ  K,M8a:r[?c?$wZ8XI`[88ILU@o%T[Njgҵ%n>`:!3Tњ暙ɳ%iW[,ÁZs}pmYڳb+֫j>^\vW YyCŌXQ޶vcrJ[ J7T"a宐VGR#wq'3z%`' A; 8qα?0djijtlr৞z G > wL6bbLl'rcX{c+/FڏeYobÃ~-&q O"l\ɇO{(i-ZGB|&KLx}cg e 7Fv2I/۾h8)pJ6S;JJq\ S8O{$h$&oxo&#*#4C'YVr(8@ ^1 4o?Uֿ4f{T3D ^4 ǎb(XٜCQbZC1'CxJ8<ƺm#tU0'¹EsaA(u[Ȟò_d#/d"Dw;M *\/͡hA]P9P&xl3v4jJ $v)xi'R<]p5YGkXOTж䌡 i&GOcy # g;poۻ:WD('{G/۳qx8"oLNLBbV Y(ӭ,]c<~b(ě]+B1r.hVZ4!8ũ&-۱T]F@!,5N Pj -)623mXaiyja)ěz\p-<vua+/jdtyP;e?O cb3H`BOl p/5ě4Kʁ8@!#lrr`^2R:tyQ8iL+B,B)[t  Q,ΈÌv"ʁ[}~be-ôPnYY-=8+11Q+܊}F|j ӵJ$kR(W/KжQu˗L$'EFyիM/]jh_vZSc#5\_SBvY@6)qY5I}^DӑyGuS)ț,-*\\AT)A>G%;^ $!-s"rD]55WG;~~!4?)]ছnZ'|Ki"8PuUIKQG3+W+)9lʓ/Ʋۆ'zcMY=YIDAT{Q#4Q`mC邴|w0/}KmSjMqn4vR+T\bUX4z˾oF,aLzg?ohg@TGnyMDO[\/!o钀;w8_imCFN6Qh8phUO=G}/ZS,_ܯ03,t2/;3텨3^zhڜM/|,CJVEs?LW|B>.ƌhyT3)J6:8~|'9zkdpgԖrw }h{|y&+;FZvʲi^Nk>~.orUY;WGG䥾0n[e_F̝U=yÝ*m恷{9eϩrgfGY?.z`d?IC=~"<>?~rǻ=wwMdYW mq+>k|?{e}J:8@!^S,0/7'@Åetb+cqq16B{pmjRkC$52qw9}&Us^Uv=kׯ+z5$x(w z3q+EQE(8pkKӯ곷@j䰿ETq'6kv׮y/lܻBlwM~?==dGV2i"31=9r-_6jp3'E\7ȹS} lRU~?ևGͨ۫>RPX<씜m?5Bc_塿}S/5Y5\Cí2 >]j#sSW|‱tyo瀓Oy/J~rp捥~L]\짟|Ů[T-}us͛)y>+6yWfY~>rhg壦K 5m;jBC) psۇ(n 7>zrm%yzmpsww!Z7f>>n79fOL;h\OOkǀ7Dvgμ/s/}2^ 6rGWYҬ?2)ǓO֯^*Z݌7ť|9"6bjVpuӟz3l-InfO^8NJFl )دOr(:);;~b[-?. }U zٕ+We Mpouzܕk}ܽܜ{dQy{oyGtYre:kҒǯߪ@h&iU^<`Bg\`5~^νԥ3LS]v}ܼtCy[H>N4}Ba\WOxkmuJ7@a^ vkޱ_֙_;' e+rWmmlN+@hԯ'((rr8v`3sNhxL9@9@9`fP7s)((L m]3#|rRVb i[/TF}iΆ _ N9K#Dx94EfC5I)ě9,~}$ 2~E=~D.Әt!!! uDe,0J9@PH>~Ea >?hLpRG )7 ].}ef˺LIK`#Ƀpa?6rl>X솵Ҋ34d Y9s#nT(t18t1Ű2k-!{8=/D/B<ɔ C>†ZHZb$svvF+'bi00R#:ꋊCsB/[!uTC){!heH>Qn pxda;b$;xP|'!<"TzT.l0 I|(ODxAGdބ_HI/#:7I/Ce6mևl_w?o³ bGxBA1Z+NC-4r@LCCGyG/1IuСHŹd@- ^F' MOP燰$Q0yCwN!^1ur@<v@y ;z9oC2u`{QAt"T~wEQf59dK>i/#*rQhx$͂(K8_zawxD[=,*ڡB"Fy/ﴗYZ47xN,@<Nx䈲!U(BX*zA/w*|zrFkU C!p^&V$ij5 =2xK{յ>V# U@ ||"Nj ԧlr=B $;=IGM]*Fюu=몏譭x刈Jr᝝!"6/4xǤv|a3?.T~XZ]S\t<>u ||~q&IDATxbP E 4쀀GH:(.=P Ulm^Hp PҖOVVKV tlmŁZC @Rb f V>;!;;y!Yq;);^;;.^bIK1ىphZYP̎A@NM^ٞYΞ ԙfs1333+12"&;& ϐD^ vb`cQ@1HCLiKH(0v"B@10h q)l ;k{[c[nn}[[` -P- =+@mylem5mYmYEANd`԰(ĀMAVJ٩8HNԈޞQ8f ГĠ `! f0Y^ 였 `o%` ;.;Q{Q[KMf{y{.{A~bZ MH@1 ĵׇ[@ӃƲXCa @`uvv"V@= $"gh-%."jggL g)!m-TodhǢk`hkϢ+hed)$b @@ mmV:mmոmM ];^ +QV( %Lx:XŎ @3Ƙ=i@[L+ Ƨ/@YsH#NGNV;I\vvRvT'I,vlv@2,*lv2\\vB,l6ZTVejʥAĠ" `5q{,@D; 0a Rɬ*ʬleh%$aL#̆WD61S<33=3>- q!Q{{Q.!!!6fPjcfjojk@ \&B@Ӏ9H^+ABB]dI vM0KI @0@ IR$+,U$#%8X"-d ME#jbTa2n on.oiig.#..Ǣdj'$%f'.)% l@W[2 !{.9lPhC ;V^DXDDuEaw%H@VSu\QijL02R|,"|||Ġ̍ڢ<ܶ`m*u!"֌&+r|*o wŌ 0Mi@p3*vЊ[\i!9v - bp3ؙc @?>5=hhH6`[(/+ 3be i0Lp;B&``5&N> E`7"8mN! vVR\F6R6 AS >`cѳ Tpҟ 0"$#, ҤLV[h,BEƐQ iU^Κ`[UvheWn3h j qjZ 򵝀H*lWa iz.#"&$#kn8B D9Y`M!Cv&!]`ړ ih2F 9CK!p;[S R<̚lIH'b ‚m,`_eؠMvq;u`vf}`/d'*ťoe @@ӄDcC2T0(@29}DV\ZƈJ {>.#jAeu ԎhI0(H{l3 XfiMhI-7cCac iv,"Rg1Wg0 XZ}fR0IJV6FB ӔS f69ge L'X%#@!LN2"*:`} dH@CHR إc[BMQUX)-.'@ RRFZZʌ<:݊Q( -aJ,Y-ml8 5Y8@`aj iЃpG^0@S!` SXpl#njZ4 Fbc il<VjmQ`^s5<(ԁ p։lR հRh@?pQ1I RlR67v,\(#@f t>rAL|\ SQVpK h4A-`ȩlkk@MPgD$X(Sf1jjYl( i^J$=GP1224v9laŐۧϫi PEњ+0Yԁ- 1`w[RKZ @ t9 :54ÛLDd%m 6nԀ%!3b8 d@<}\ZDĠ&&s0݁{ƍ JefX,@1 =QP( 䁅 4@M5PMOښh*XZfeZ^v@@_UjqR-7q@ 3kU.2$:aF:3XC\r[X9<>=?$Ǭ/ @ jR8Au9@sSl+jMv JlĠ)är/'i*4fM`2}cVK rJn3u(fѵSD-dfԲ ]J6; 58rxgfRfp&". Ƹ@a f*״5@6P2Ů=@AZjZWHXz1-̀:8BclDS@䀽}L@ fJ bdRmvBliNi^ .6lRDtmm5ؠ"l\@{ym ]}+[^` i: <>B 8CĂ4cD5ظx RC mE:@L3dAX1&o 602Z4s&qVD MSBDv4 Q7x0093 4@>x8Dv\#Vę@ hKnIi &**T' tT*bQ11$lف->1==}s,*yeddL@ڰ3(t%! @!9 04 mhUSDWG [LLYrA,$69V,!AN ,N唂6< 򦜜j|\BHQPdBT0ZsPVFرcaR6Yy-}! 9 L0fg #SJJh0yjk&g)Sf:@'19 T N`.`+i8+& r@16 MC DX@^+{\ 7=`8ٙԤX <4vH.[爠M9,ik cbե^$z,,zyPdLWTEL[T3:a&~KKV;PSW1A @39!lSJ]u5uq^Fb`X`dM:L؇68l@PWu4̣iVU,rc]!@\ ZR0=hURn PӬep9 #l^Y_B1 絑 iK# no-,eL@P[SCv`[W4:4ɍdKVf:da@s%X9`͠MƈV#)5D Y0k@@ l ]K,} y$Rz4` 3VKr1#,,|0gسI L[2סq 攵(G ˈ+2ƿA -]!y B7l)n[2RJD 6sCPZѤ#  onΊLm@,}sP][,`Rqk 5P_҅]R ؂5%)#w.-]d@@4o[ 7&A- =N=h#%@&^ jk\Yjs@"R|V@Ӥ X[^RdKA]ApG@@&,Y%A6>+-a ,k!cOv@^Eqk"PjcEt 嵡]Z`IA@N HRPd7["w ހ({nZ XJSM2` .A{*2b͢5PD\ SXw]H>Of" LRvhc2PcgcS 7jou^EM+5=N 6x/V ؄\x$6kaU-ڃt؀)Y,^P>ek`@s%n ib 8'?yC&>..Q}<:VwZVKƐWsа*œ- Xm*p9(f N-b)}oen55 ֈ @-&1)yH%P@Oܽ ,8Y_-d8p 4% NC@wqSafA,lĠ [IX #G ؁Cyl #0@1Ȩ@(#U9`MNTB`%"QCS`E]] Y 4WM$,aJ *?p \sCҕ0[YۤY$',*glY&^7%7{1+Lfi>!;ffy+k*g Cvpy: ,0FDy]$,yPfd@zsl]71Hե^Ys.* P2Ej5 pm%/'FMG4*j(Ql 99tAFJwZq0iiAFB+#-𜭖'@@ur A| b-DǾ[ GerXϟLt40OT @ ]/ e9>CL aU;h _8_ڼZ!zh*ʛ,F`76rXY^ j~QM^8DW{kο%P9(!NF,{X)+\<ܼgjTEwM;$ͤKȴi**v~TUWԈV9뭞~C#&ݦf$x9mmC;|tbv&8F˹O;'շy*$:^M0u-#|P'o`{>1}ui4|:1Ա#N>Yv*=3v_ U{Rj˫]&ym߱CCKn FGfGW~??!r駜{&e&- F.}yz`qx.e>duqڲU5s}'r fgre~jCsO?WA(Ix^, o. )%}E[C~ǎ IEsyequEHFE]z/ ІD#Rqvq s! eKS+OPۓvE9u8wvqp๺#+'/CVPTO 1 5 Fl"̢ccߖsoIÇԤI=撮L8'O:etZ<d-:gP--SsvQ`̹seb O+KզD.\lTt9Q _׫WD][  o77n ݂{ZW=(n`RG'9O9>S{.|kzoND^398fmCj/eO@0gI/ _Ŧئ־}781X~ޱu\|͕-,Ư+1@A6n0V:p:W`cI g* wq '@ sA< OB!D&:1Ej!Gא(R@P h>:]øb1ϱX3l: g;bb00221^f\cgČ`dndcfyêZ6ơQ)YǕw1O1o #~{1 NDHUK8H#R%j,:!m,N+!#9-U}4tL܈Q >”c2bNox)y9EퟻiLZ|%˘3;sM`> +h&H>Ar"w#o(yT0 ;ZN /iIJbV(.Q1 ˔9y%j->.&N.nN$]<뼏N  gDŌ"lHU3s2K< UE3%/b37U^. k4ô;uVtwy,3vwrWZL[[gL98y8ʺy`<@Eg[N9Nӌ'?t,taZ= 'OdU@=[^jx|:e+__ը[ٲ|TGm /Q۠'ξ~+A7jo cz0<4:Z>}KQ>3~UT7{άހ/ ;GN/.ϯTpZE^c[k^aÃ|i3`"QB)T &<%,4 dM%8 < 8JC-VLkC evtY=ₜ`|lSXƇCm LnU(g BexZ%cK#/Il\f) 3K(0b>-*>G@$EXH @=[QNT*0T[aڿsеSSa7 )]5Id8?ih5Zc s6Du`VT?ZI;&c u؜g 80xru VEGDɇ,HzD8H  ?d<I*D?qRДpv pHYs   IDATx}|յJZ.YU,˽76@$$^x!$%/F $ ƸWYrQl}|gv$ƴ[˳3wΜs=b qppppL#OpspppppppppBS pppppppppBS pppppppppBS pppppppppBS ppppA|xfh=1' @La4P<<2\w$jSzY$N= CLsu$$!*ƧR R*$H:D2d,J"2#FH25K]/#@N11K qxSrt88mXc|>jG qu ^זVMjtrn{ǣqgoYi#| x 14AFNtf&kc}Y [\)fMa碈tURx_= (;NeG7]U\ѱ k!MS pCCC}@*_fD>qlYe"b]9RZ]}߁K#po4둩RWuUj>Z !>B n kG6 :Ŵ(_,t61KvЈ'^1.fZkuޞa\+Iʜ*qp\bfj"wwv æ7M)0c`:#G+6~@4;;&Jbb}D}72Ͻ_-g>fvď%b"jt6 K1*(=Bwײ6R&Xiֺz6H݊Cw~+=lt N/Afspp\bH b:7[⟊w EE3֒ ksZT*$G|L,xV3;D.+a]) p@I%5۷4j l/@d5u&!YR٧IH}i8x6g#cE;9o`K=?oE__AKpg888= *R^P]ܩ00a!Z?FPLdb jcHM>Rca.g%HIM+0V64ҘGx2!hψJn׌]9)!*ea^nzZL5Nj{-S_^Gp+T"B,M_Kl9iwIh΋#8-9W4C+$AH=;x_bAձG6$z"BUbQk[rFW_ r`]/CX/A) #HSPFI1,&9tә|wV]jRC48_xYhZEzwImg ( _g gǘ!m ڂ.&I`)2T(K[Է\W5il|q9a'P\Y1)]YfbP$j"hhA_'eN&gߤZ|'j?"_ZaGxD()  :}:ZNkhD'qP\&+,*^ҝ[L^oEH'>:פ9$.ct(̋% gey`moϾ>t:;P#a0P80m -y& .5;%gg>53ޖ;ٹ)Q)%+5;>X4M0e)l5$HW I (w DSWP֔hu#|)DH/@ngx ,##hi#O@uL_5b eYcOrđ"ܧ']x,#F_2%IXcP2G?O ,\4* ~ />[L(G\$A' Hcҏb16 ;ДYD4W2QX-b {UU؇c@:]Qe-nxQ>$ڊnùq#ڭFLN#Zjr vyh@7瘲.+A $A@$cjPH4TnM(fT{+}J#$Jc^=!- {JvDb/O:J҆~mIW4.\xb~Ց5ԫQԍPW Z1GbvҀ e<h".l>h/bhp<jbpYCq 'af / >B_X>6DLr1dGmCYw1԰1+Eyvp( s `y2H>jITu۾B0l (%Xd 0X|Y<]B/ eZ],z "p"NExAb4a,AZ"['d $FË888nYETx5`/ވ ֠xL0zB8\[\&0JbV# 2k*Xz,5:zTn!'oB?7 ʼ="H!1Ġ)+(h}2`x=ǐc٧_;|TaU&Djra??GiT !_%> t-fNzM-Xq{ظVVkXF"K"ңEE-]$Jw}iEz^.op4l#c:d 9 0Lt> Wnk3ߵAsܖoVT礥ZH,G`4q.gYk;.iӑAS a8Ձ# qǵ+X;Q7wɻ?{ݿ!p/uzB}0t5 (D5=܉j5-_eV+<^BvJMf 9oᆺrÿzakb 'E 5jP(!jS=ƴ2_R2/Ȟޕ.hh}kwҜu׬X^R\1 5FC8;+ | O  1/nhnqtXaIEY`D2Me̓F "kŖ#;ڒSG[^v=to!O"&7%V LFC\>Ǔ .ZX&^TuMzϢ4S´Ji5Ri2 xsgg6U6NaEWoߴ$?ɵLZE2!$A*Dք\Mv#tu+R0C/O DoǤbV$Xxl ?@v)GoR9֯. >aY%OhnޱjRĤ8|E qGPmd(Vh.,ɕ+o BRVnX (CwJ1mՐ84 "]m)%]}ּį_OKy{Ds6w/{&vC'vΈ}.ڴ5+*45/N?s vt8btܿfjɊ4x]U q<ԈD멩c5Wi|Q+xIB[zT"֯XNldKr;l]_tȅ7-*T!U,(c p h@ <:yV&ӱkf)a'{O?wt(lz He dR,qat(x١gu5r [oyT &fzFǃN&7zae|Q+jn3mn;,f^9?~ʕ$%T*Um;Bgbke a!pftgZ&PYϡɱWC?oͨfI4؇dR+ ߦ1Z Ǟ,Roi}b >~g݈>v{>3͑0̝ɧ[ݢϡ!$WDzITf}G60LNPE\d#I"u<Goy;x/Wbw8$"mܵp&,;ӟwy&iM/teӒ!> e~>MZZZALWadb }O9us J`'5iEXP,/%o>uZN<=WU攕L rf̏[Ze-_≐i&q<#zJ0dV_#oGw<덧Ͻh+(!R:aw7aSM:_M^&I@/uL z*b7ѺZtwFgd^TLzyeA}s;tw`ʜf Idk_7q) 4P?JuR )}"Pu{.'y-z̧~Y.mo{-%(מ)c7\S/SZEĝxObh$ `r%"xH۟S3C҅~ѳ1oi _\ؼ[糃~\WWKDJvN{Գ)d+W:(āC'ՋjkI6 +OkKBKVgFP}#9{}|nZ;%aŐ~S`)LìY&٤Rv́a!}6IkiH uN t<,H@;O!XlI0BmB?i]8]o{l$I v_P^}>O/՛;x62#nTZ-<hw9A >3)K7~gh8*0 )\^ b<3FDz{(Xȏ aZ!+SV\P6gZ0d h4xs,`G=):MsLNO٭QɩO dc]]ܓBsY$F0$F lie8R ~d3 5Ea `I8Шx1,'8pЖ 5yV$|P}3f;}|(|`BXO$;5ϓm_kbb#}"n:ݮa[Niߊ u}>tyZ cLLͷ "G6wzd5]6nEo4,a/C% Uaex[cqD&k/g!bn!~7v HЄ,a֓0A}q|o"Ha$ga+$ sKA ˗-Jl̪Dr&pB<0 D@.U6tfŚWHG<'] )h0:a:KoS'^|6M)`U8C7| 8|ІZ2T^aZFLQIw }izN :>L.=~1:RW)0>&O_!Рɖ9uAkdA1$V1>.OK#Kl6xtvvOaA\9k׽7 Tb]/b^Zxs{c!ߦ`D$HH@ mk{sqLya{sbAEѪpfکIԽ㧠6Խ{P^T$a}~SJ SPlcj AteD{aDQAnB<> "ţD4 HL-]ݾ~dwKr{Nz ۭY$g ےKQ ptC8YOMz}G+@daz 9HxE 5~ R$e|GBH=!~;/o;5$y~'c )u(Ray7$ų:FV~[糖gȐTAHI*+ Tսew8)4 V␝ I &IGQԂc|,^.͌' 5 Q#u;3Rh0DݔhMIyl=X ෰;o kQ gLWS`LcLLSiB Ar}ށ[7=!<5ApIz?8]SWTMϾS'=O/yݰ=uZm>NNM ,ҸS3 Md7rّKo|r<.9\[]#p Fq{( PHQ]Ã"P&~7˯&WK(Q+0/1f.8e+eA+(֔^}МYǼ}/צWqq8c`<W)WuI\]t ?GݐiZrq=C`A0^Ev:DH0e`NWm UO\P9 ]PPs(bD2{ ~`kۼ-2r2%0LMCQC dz|"f;32QD7r٭b?yw?a8͓8FBUSZU "LʈY2.;>6` 8 1N(4P3Lb/*d@Cy;ޭs%_ؼ;צןBVw(v$)Xo5&NoroTUްr5A ^'E#4ӅA#]2YW e Cŧ?i_',8ߩ0.XPV&b3A_y&h %/<ֵ?z;[MRC|MB%!IZrsUY2U Ȓ@*-Э!ڟ _7kui!6ꊛD`1C җ#9zTcδ}-+M~6tI9d  G?e:X.X݊ԕfʥW¼φH=ϝ pmx 铖4wZ_]wGW$7eeYVW,oJ¬\T椉+^Y´ 6o.dq{"z66'FMV.[HKWs hS֘,9Zd|F1dS&bqGK34P-He骵+Vdf$ ] LցE|>>Nsl̲J$v6N Ð|dΓe8 ڊSzR1`N>Cͨu׍ipuf,lXB39a_ :2 S$,:g}=0ۙ#I;xofSErl ‚4vD[RU㓔@277jA9R02p-^71mAEf~r{lE ^r藺Na($ Ƌ2Tl A88f(ʦ~vY&oM[RpERYliy`Huy,ɰ5wCo LMS6{ʾhq8ISJb*T `8ɐa{o(m=xwC7ojw};u`PQ%|{^ѝ8@,>JPM'~NAyZ~y 2gtz ^@C|T]LHO[ˎHg{gYcڛpѰJlqu]u2eCҋ 7$6tZ^)ep9eY1$?LI;İz6CnX$;+?Y r**! & 7ᙗ7X:71lYlK+> c&Tdk+ JXn|.HQ99ݑD}D :xpNixnD\L ЇҚŋS:q~>3F`qD=WT8 ­7 FB(̓4Ƚz7 > \߇>{UiP)680eΦ k ("Cb;Ft `}uį+5q՚xąʌΞA98QRHu;gJ8 ~^ xR2cma `SL}bFRK]x lǬg9yApLJX}eL&ee"hbR I< 8[=}M^Q#]T'xٻ2@ NӔ IJuT3j¥ OG=DdRX\>E%,H"CD2k2@m|xm! H$m03x4%QtIqqZOOX5p4z.j:.)CZ퓼oҡ232yڥs?j8_Y[X &| $_Vo@oaex" IhMK͐XI,X𹿢#o̺x*]K\RI- || iޗծ@%3YUr ƝGܦ۷KʘL$!N@SB21HP$vwwEtC驵Bjn 4!?~Rf]a'@G')5;DtFsyW\NJ~vܙV._&R)jp(nns[u;/(!TGDtR2` $hG;{̷*L`6 EŐyN[gٿ=a EԦ?Vmt"*3ЕJ;ʇ[oEnD?[F|Z[>9[rQ/~M ]JX5<>Ij& !)U%٪¼PzzT*̀abu/þ|-;}t($2s\@0/PP F@H1X8,8tNOS(0 OmP h%`JP31Uulu^h &[ۃPn[ Ԓ+ɛ`G슌/F_K۾ml뚲+THd YN괲R.0Vg{rCHq@H^"w!0/W5=E$¾rPzb!4e(8|o8{]4 E/zk[V01!XRdA4r7 L!8کX T@$y]-xW ?;_vfdGhe֐~ŗG{î$$YXҲƲ*᫢tqs%a4PVZ|2®SZ^#رa`t|NoX4Vbh h95.RSdkpCCCBL_c`_?YGL@x{Bl;~U/ ۺ݋BzA4=PmGV-ʈ:xi=Z>I.Jx IɉH"$GaA_,{)zUsk,al+y*-^Y׷syEVSUĹXK__}5d)mCA1TdQI:}x*\^17*j` "U?FŅ$ΰpKɃ@<^K*4P1ذFv F5e?\eґdf\rJ>NRμg<f ovy]YLIp;c7R<~L؜xpV IjQ<^d+{PE#@pCCC}Aj)!r !cxhۇWCf{ mϋFOem>Ĥ>Tr iQB$".&IR ^n s365Zvh eǚ`/bB~2`@т[`Pfe5kP F,@1Hz΁_9K"WĶuc=78A Q?>E&F E#0ƱdbJ ,eB*l7Xh`wD7Wweԓ-eŶ7-NÔLJ:$K9ERM! Őg%nzmǍOd&SѢ?xk0AAK =F?S(8Bq' Xc) ~+MPppp\bKD7A`)$`hfL/>yw3f6 ԰09)ƪF?œZ1>k# Hc\`v4+,n&%,B[OK׷=~++m}}{n(mak2Р.m,VeSCD%@-z^X} Vq<',Гp|O;K<|ዢܷ۽Kù^9fq0U.{UP5Rt`qOmh aA%$S}xB~ 8 (n&*%.NL-~u Hu " IDAT83r8]ͮ=Q Ycϗy&q|V.bd_HS0܎gc!pId檴hعI]~簙B;xwZp~r76*d#gQq4y^9q˾ Mߚ9]knx6kKWTs< HFj-'T!y߅ېoV5H@NE5x1Bn ^= ۫^r20(b|% gV nsw`&ʯD&h=r _[\+κvCv2NM4}z61vtw8~BTSL*;*ܗY{9qSG (dG:F!ڹ%A,\tclka'1ޜLUA]s/!N?c?}gM<(9Iem\j:v+8 ;@ݰN0I2Up ~NVFmB ~\f%bO( 6y_ec9fBW\Ϯ^DyNztĝ>K bu J> yUb ϐq- Ї-?dy<>!V}yP$`GJ!ƌ LAk4ru(ZrRHFGaR?iu8ݪ6Oņ$D=!ͬD} b,Td2@.itcAAQUUeeUEa~.T%sPbNG46)9po98A1_yhXR #>V0%L:1H>λ^(;N4.za=L;cc?8nmx 3)Y􊏟~Q3$(7, s 'LJRmsjUFzZ0:|XOo烪xu|BPRZk,,V)2D$ua3Z w NCa( Ecp"s UJYnڠ3dg5jjɹ,c{wkP PdI_aBX.pޅkmk>6] h6"D9`C\g>B2C릯Ɨ&2[QWx1y 7%Ao ٬jA!r988#M1\к@M~ r@g Q,v29^`_];څ?,LXlϿ!'k >. {(UǨ`\ t:S@δ0gi9 \l%h(yH$N^iAm| J\7P[Krpp\bFajm FmHk Xrb{ή:;v;|mIM»`[LՎXu8>yCX\)!kO"QӠ8pJW-^hŲEbԶ\# :IRc^+hRoC!!!pe@x+7@ƌ"ل΀ lBٸR)c y% m҄ KulAӶKڵbi} ;eg\c?-={!}{4[#pe=,oGW˃V&ur$ZHz'<^h=x$io!?x`AZ.kH 9 }tJvaMUmuK d }| x~&]wCCC/B7BoF󮼞s& #Gщ˯``b68y>/Bg]ұ,v/hqtؗqÅ#u˗/"5to>6:>6F.IϜ!!EʊI ! ke Ea?r$MWwp !m_sl=*lmmNCg"g}nnqߛ6"r54)2 ;HC$RCCrO 9IvχLyvޒ<'NP:](d!~(<\<4'X9G<(QqӦ +.ћ<"\ȂHgM ƍCC_++ӏ IU D|ІFNz2mR㺌?ơȔU͆! ek,} aE1rVXjcP(TitKNppp\Y1|@PDqs5rAU!׼*!9᭷Muq=8Cb8e2{R9@iQCX-Jk-YzZȧ º@k|{ieH hV N&k +FG 1Lpo陙zCfYW J_` c]Y{b8ۆv^8-fw<d.*Hci`Nb*L3mmT4ʺ!~i:e_,"TL4D0VCS ߲nKd8N7!)FM>bklfS6^ B( B @"Dy(TOhX(THoAIH!Mv-[SE?cO;w{f93̨=ݵuݗYvOO+?ګ"+Q; YG99aJ*BAjVz`xDZhh&[Vv̋,ECoro7cP1y Wpu[Pt̯1w2M5֨oyhrJ?6W9$ ot0e{mi!SBv,dQ 0TCx38@>f4c$nѡ8 |}pEk{[BtB >> vhY=vDc i qg@K `ތ#*:.挥A|SQys5e>AJV45E|1i. ]hT8nQ`X! _3NZǣZ]W͌hmEpvJe0p |{z,Tʯh^\}^@O"`f2 sQкk/iVM=2YL`Xy<_5UQk4|7;oBFx(fbHVEQ)Y|-"8AZLww 4J R1Ο2Ġ?cc`'f '݀;}SɃ WC0w1ya¢6.CQAL̂DPwͯY}z 3+v5l^ބ(&#t^%.*8@"L,1po?`(]q!0#iC9OgD884*A uWv4Zw Aj[Ё 9x4w xgX7|߆]&rIO7=D X0H1`D*+^j@=yRZZRiH1o@*A2 S#b|Zlr}{yJ]]jS0 (_wwWj1 /]V=,_ݏ1{1p Ag{3Z"$JkF &n~B Gŋ s[5Q14"e=>aȳNKII٩ 4]XQʠ{18u|G. rbYHE%[2aJgY36u[ ^w,|HC]4n~ CȉLQp06AEOMpi%b;<6딽:!ɮ,ͤCH$pp*ff50!A8T eʿ, @/82&|8FQ*/CNT@`xRCKٟ?{ӀY7b+.{*Y9@AJ|QHlv A(#}D:F6[t߈Ȉ0T,MzvcH#~xDq*EX~{m j\IMBN""]T_%[l w 2\#Æ\8/XU/Ԫq̱kXLO9t@9U[J\ȰPgroR޾ǚx6AaϜ8~K(4ݠp3d^'PrPK&+\WQ#`!Π7/!l *JJ2vlְU d`۝w NU!D1XފU{`|"/`JP  8D("96 c 8 te J/PN$O ):h]P 'AAg; W(B(ŀ40C#CҫtRQl_epC"p2 W<3RɨIMpP73$"h!-bQvmAFQc{߃f e5ypfaB{@{c'@2) TB |L 096.CL2aȨiZI*Qrv]3'D1vRaPB5Lˤ8Ra ȁ`?>Q}\"96..R /`hT%{^A#userj +_s5}Iuv <lV]KH\= clÎ7}-ԍv6kؿo,`FIi[@P <{Ҋ+۲^+mM650(=X.Z"4܎|?|nSau#n]K]QZС Lv:l>EÜNbw$;G;~Ks 6xz Z8t@K𓦐džy'X y@”&ellPIDAT,!m_ë [ \:~|Hs\`qIfy,7޼n]`[(-ܘ 3qG`ik W_ڧ|7 nup5BJ90bB*>6S`&Xk:È]_JOŒцq/#ؼITg7|nx{QiЫ sW5q|()YQiU!I'<J$_o"ӥ!a{kݟh7A{|Ư )i{7k+OX\{`z<ϼv/kS:d`h`5'+wl 4UjDuol^ZˊK.^^wmaFlȞ]h-!ۨ'g8cGԖ;-T*a0~b2XU*[ ڄ.lʌd1"lmlx\gg ?gg.;Fr觶[  rwρ#G2p谍&xVxLldtpx,658Ŵ`*Z7d_iG!D WdFwA% % or-CLP'/PLC, 58̤HO,V +#h( :%0؇x Ӓ }Xx={_"2uk_%`3YB"82&맽2b;=Uڒ(=+H.iE a/_Eܝ2'>ec@8G3Ozcb,zL_5i}S8h}x頊~#bs =E$=v\ bx]{y+ohKY|:~Ç;d[{YA1Q,~ĦIzHvl8ɺ`arߜ3xL&+/8~G۶)Oy.b60Orrwɱɶƶ8Dx :b+K;.ΌTY lC'9Jo> ͫ1]ƥNصkwqQJ.W%_jAk5.9mըBԍE2^wd?. 4rћe2,йf&!ap|O>i z&#B&6b0cl6c .c >CMx!*spo'P/D;k~z7cVD?0ub(']Jxh`9Gp$h/: .7 5O+q-_/ZՇ#)H#IlT\RbZ-wٿܭ࣐[B ;ϼkTK_s&֏R(l̶"&\0'cT;&SuFMCRm`lb혲50iH3ͭ>߽uxPa7sr/ɀ|Z=/?o6Դ%X2ܤk_Xظj xQ60D.m'$9zHa|XIN5Bn+O цd8}x=d&~8n"]u9 ;`}cW{u 2 /u'*oXBg7K,r8Z^:-9OӔ%Wd- jʸT,ˤLkc Ov6?o߇KZ-\[Ljfq:RXO*ho?| 57o!~5rpUİ;aut?^8aC^8yJ"DE-ms5rN2i\skP5LO#RB;0;p5$ -0 75 ` I\rb]e0S骊%(QV&0]':+׵aiTwS*iH&'Cs’=e2G*QJ:Ӳ~ F$(,cyE,$!e-ZT _".z4 F*Se$ #_f [EsyR{%5PflqHKdW٧Q^5q (*zq;6̿j+_q&笺k1U!6X笉m4<qϦJۥ!EPݶvLk+-K$ԊD ٷ "^(k5M8 !@!`pX|d_ U #%٬CW W*jm|k5F,n*9njYaaSgҶW_6U^QO?̐!5uvt͌~ȴIM Yi*MkY}]vKP7{>HE4`:2 jM 3S3V o][)ɬ8œ Of@]̨s-a VrDA\Pk봈~. mVtv%:u;S;wAɅ;$܃gU(+ml乹6.y榎WmEō xlR *g8l;r8+!R{~Տ 2UocƝۗ ?FmBj$k⁵:ѧ5O$Δ+26(= aܕJ6[DGx~?dR 6M0=M {j*Z3d37j&3@IF7tFqArIa^n`$0ةNo>4QUmTNE".Ig(c9!ia%SSpɰΣr9=@//2!_dQY<{3?q ]b%Kz٣-#cԼ{$+$6F\\졀gSsPI`%*Raո$dwtq'!$4Ӈjŵ[JjZ47۶|I#lGY;'2 XF{,y"Akck jttwue0BBB/*Noy;[$-}FFH/ 뱳jQ{z?nظ~pޅHurDݽtU7LQx f0f|JoE0-J ٓ*&J 92GRDpaSȀs1-xCH-NWD )_M7i jE k)+?BmP8Z`kmP lR1 7y"L[$t:]JR v9Ecf.óUR) ʺw۫뛚Z*+ʛpUd9jԠ>ήC>hjgPɀ; ruN&/͂Dݤo?=Gx&|  xMV1f,˄POvH]D)UI!,Zm[vb-աSSt8x;dI<ɀ4y۾jYW $4}XHLjLbp# !mM4Ws|t@cxy8V:#N:<4)mc@io!߮ä hUZ"N_H6qNdƘ D1y-NE-7ɢ{bDơ8=hE'Ot3?fXkOs`D!).i Jlɫ/e9\ 1Q36?^yؒ\ƥ :7I1KFcק 7e)+(X>s4xa>LJp#XG,y klɆNI}0|a)[OR%M&wa9*1a \ɄPDb+Džj@9^Z3^8 68- u$\< ƛ4) 1,D+ߊe_|f0l{{H &YV̾B+*˂W (A8%uުpÈzPMuVp,m˛6qauRwwmtFՓsЄ^v54*g!,=h!M+RiC}gI+GsMEs;oG ʉ,`QEg- f129WZ0"29",rD*Źg Jƍ+ڄ@ N->lnf%wWV+shy߉{kY ˱oQD*--)e)+nDHzru DrڍaH` 8= U 8 ,`ˋ[0`O&6O[ڂ ,``˽ X0c`y} ,``˽ X0c`y} ,``˽ X0c`y} ,``˽ X0c`y} ,`ՂllIENDB`execnet-2.1.1/doc/_static/pythonring.png000066400000000000000000000426151460460142100202560ustar00rootroot00000000000000PNG  IHDR J7?iCCPICC ProfilexXwP;vzͥH^.{/*u)Rwi4Af""W)""z"( o7w<_NNIrNP$ ,<`jwsF ?EYA)KAzI#QA 'p бvȈHD~A>'@@ur A| b-DǾ[ GerXϟLt40OT @ ]/ e9>CL aU;h _8_ڼZ!zh*ʛ,F`76rXY^ j~QM^8DW{kο%P9(!NF,{X)+\<ܼgjTEwM;$ͤKȴi**v~TUWԈV9뭞~C#&ݦf$x9mmC;|tbv&8F˹O;'շy*$:^M0u-#|P'o`{>1}ui4|:1Ա#N>Yv*=3v_ U{Rj˫]&ym߱CCKn FGfGW~??!r駜{&e&- F.}yz`qx.e>duqڲU5s}'r fgre~jCsO?WA(Ix^, o. )%}E[C~ǎ IEsyequEHFE]z/ ІD#Rqvq s! eKS+OPۓvE9u8wvqp๺#+'/CVPTO 1 5 Fl"̢ccߖsoIÇԤI=撮L8'O:etZ<d-:gP--SsvQ`̹seb O+KզD.\lTt9Q _׫WD][  o77n ݂{ZW=(n`RG'9O9>S{.|kzoND^398fmCj/eO@0gI/ _Ŧئ־}781X~ޱu\|͕-,Ư+1@A6n0V:p:W`cI g* wq '@ sA< OB!D&:1Ej!Gא(R@P h>:]øb1ϱX3l: g;bb00221^f\cgČ`dndcfyêZ6ơQ)YǕw1O1o #~{1 NDHUK8H#R%j,:!m,N+!#9-U}4tL܈Q >”c2bNox)y9EퟻiLZ|%˘3;sM`> +h&H>Ar"w#o(yT0 ;ZN /iIJbV(.Q1 ˔9y%j->.&N.nN$]<뼏N  gDŌ"lHU3s2K< UE3%/b37U^. k4ô;uVtwy,3vwrWZL[[gL98y8ʺy`<@Eg[N9Nӌ'?t,taZ= 'OdU@=[^jx|:e+__ը[ٲ|TGm /Q۠'ξ~+A7jo cz0<4:Z>}KQ>3~UT7{άހ/ ;GN/.ϯTpZE^c[k^aÃ|i3`"QB)T &<%,4 dM%8 < 8JC-VLkC evtY=ₜ`|lSXƇCm LnU(g BexZ%cK#/Il\f) 3K(0b>-*>G@$EXH @=[QNT*0T[aڿsеSSa7 )]5Id8?ih5Zc s6Du`VT?ZI;&c u؜g 80xru VEGDɇ,HzD8H  ?d<I*D?qRДpv pHYs   IDATx]|S?M.tPVٛ2 ES|* OPDx  ˆRh{6onii9s=|;(JBy(5OVJJ`=0#(H\Z5}(H 03VM)@FJ3ŘĥUS PwR3#qiՔ`0#fViii2 ^^^vvvJд6L 03>a> /^3888`ֳgOIK\[W׮]s4h^s7`b{UWW^ڑW`M5[˗/۷/'N}٬3rnܸ3`(?W54ti=}_-][nX zktb?3F+MHa~eB ׺{fmb"ӧO@h3O=TNN棧&%i\LV}O?%&kh ?-*&K ?#ۯ6zjs0L͵x^zzzj{v[… ?0շ`rJKRj/J˰0C~޽6624]0G$&M=e!+b;4udZ-}E~饗-ZVF{nu*xk>J8})`ePJmFZ'e^.!!?<9,¶kn^ ql@\zuaa!2#-0WS{{<f BnjH]uL9`րXl3󈣣#l)w؁/˒%K|MњZ5m߾w޹wχ+֭[١boi)˄l B/۩S' 4i&08TiD' 2::l>F,u͢i.BkVffDNzgM47A୷:vW $L ͭI})̾s<UY꧈eb OCDi`*A3ֵf Tԙ{3$úuwpG<27,x1C۞*cl޽Cgsn#]ƾ0pnf5aaP:=lU9̙35q\-TyPE`> $8a1 !`?`~ȉ4F('ٕh4"AJ q+"użNI,übhG E8ʉS;~VQQTRT.Ύ{7Ydߩ.v=՞KK>Oޞs}3 KW?%&Scx]o^(%/쬥OV˽f/sp$ղyr€4KP &{6؟mX`^)|Q]͍ 6s~늎$-Caݱ^X`RF2”c\0Bn<lofa̖vRB+dC/+RH(4)*/hu-@BgVuBHkstelݙ޻Φ]u':v[kE5ֻTCMuN GB[R.EoG\ǿZLqǞugշ[ČUB̏U-6 0i1QԪ:NĎnmwU_ˎC\L3\:eם Y_f|ZsmUm>wzzVqQBJֆ0c7XO&%^9p#3M23 *SJ2ӋE|"ri*}61.4jk2 3뾺RBݤ&oL{yr䛆V]nb^wYrx՜YHcfq'.xAF $ʂFWF%M/F,-_fé!YGY9$7UR-zÜ^=7n~he wUFVy^G SV8ȪK]| iEwzGF_yz8^6dWdoVu|{NK?r陴+EU*Y#OYu'bC~O0!O S9nMT3q{77EqWue( WJKӋݼ'﬒Wv.׬`5PA^ G P 8T(TP() R$pPߘ# H["XKjZX-f[ߡ*r,<ұVMqSL0 k_GE<"I!Bf- @T(JI!r $ˠ*#/% )Rڄ1͂_.'1AT0Tlqǐ*I"rO4LYQYqBi#ۇז2迈o־W/|yBXvYlDr+ j7T w~tmg`63'% w b8?N}zv@(::{l'rth _p6*P׷۪oPW?b2{:ƥ hMLrDYXK1pjWٻv( yd][ԧۣ10->{~CWc߃qzPx4V +CjYJg/G>$K %3$SQ)ku57Yt qz뾎.rp)-d; Gb4u_S ב UB:0rflRQ:iݣl{~dH#\[k xKQ}7O]yu=:![FiBF%pp9elyKO}0S >]YG앮JN~ TԸRg R6E|I2_gYyd*Y0 `a>Ρb2P)%3MդBY/ɤR(_i/Jܓ47WrS3q?,2eńjL:~e;CZTU@{VF{%&:1'GZ6pB˶خ[+mKxpYq;ʤ  )e.|wx|n5x-|[z">[$ T%RX\ּg?Vl0g3c(?Dzzjh-A'ne]fnāj8$qdʸd ؁w*eiU Ҟ<<xM uYP->!MWLVynHG9VWzNv_&&bJ"?>p;uItH_5MUSL?T.U`!|%U6,˲D53DrÆ`.E].@p+&s!}=] +vΆIQ3,G>/3d3jqgY~9v2!#f[i-P0b/1lM/X}8bfğ.gb Y]TKUX;U{^`WL}52xȇa*&z 54E4oԕ`ʮ+1 7n:2 (7u@nm-UqB3Sþ :5^p>- 2.tX{-6?Z 8y&ָƄ6GLLs2VՊfŭ`u * há +󢄂dȍP߭nnlzfi;l6ksDf;$s#QJ5iWP Lquߙy\&3c^jܛ}}=24_~(ſV.Mhը[aRcԸ 3qɸ ?\ܰFh.ST{F:OP_"?LA|X@<_YdU+ZR$<7"+h"0iء1M9oֲn_[m"U@g]IQHHa6K 0a ECۧ@S}W?ǁ]\gx9/k_EaRxF(Kҙ&Z4 ~MsHy  bo$w+1dk3yVH7af–Wj X@u+d"/lh<#(֦L[ ai _UX?uO:K,voJve=#Xm\6]s!N݉(Ȅ?|NrKF˫(  u􀇜a{'@.+"2PS),gYaœ'9uUw>X6O)j w[ia}_<`{<a7\q:Mӣ2Zz4d/Qw\&{oNҫwS\D:* Uדbg89$bLJ+;KBؑ^K%j{ep7qBBKʺ9[rU\$ !1#7.)3V 4P N[='ݝwro;PG:1v`ʤ^\}).|9W/0ѥ߳ |^`toߚ5p/-ur u LAu %~xI'ƛ^)p+6Uo$4X G57zf厣1l ?~׃/J/GzZv߼7e}P^)_n~D+7?Dz~2ӛ 3^_ӈ#ѷ9t&gǖ;u̓Kxz<{I)ZFj*w>x9G\uG.zE C#S .JƂ%;39+?#kLӧ'x1k왝|TDd(HCRQ@ԁ37x$]U"87ݯ:s-~Coyv[ !vfѻS`^[uN 6)[s*X'IDAT_'$$wqaB$db:*91k&-ۏHHpܝ {1 rfXN 4`n7y;CS?Wʔ d5`/3dm۶جX#EDu#!-Wg),޸=i8/(@8_LdaƀnNG1kZw O)+?Z˖Qjz/^kYV]H=y~35pÜ? K%w4g`_OI?ٷ(=oҜgo&@nD;Y?9%b\hϳIe2~;ty";z~yE[u*N/[vvu+: 0v3`ynWV9;~:0A=4E Cz` f,)'M^B\~UtOދգt rpcccvA 1g {{!X7x{ǪX=[? %%:?jIkg?yYcv>vQ۫zjȊٳgʔ)K)"p,FO?tСhU$}'6. Hbσ6dc 6388ݝ9Ά1E$dVrih.ܹs'x7knsߘ9XwvRs0(?;f܊`Xyw5) J7mO[WMC+bӓ&M:v_ TC:v=Xj-[! P`ݺu?,А4a-B ͛g[hOKK.=#-ئ[ :u$;HSnjiγX~ܹsBM+">[nea0455Hlrt:Ե+c~p0P~Xh#VHzEeKo' OЊ8>/{+׋ISOs>*v$lu[qQR Yki-mui1޽{NNN'x= o|Hލvsa] m}݂ -Ț0[\II ^v͖ܱXK.E'q JrqQʉB^1ș8GrKt{oT-iXLzeɓ'۰SQCc]KJJ8q!n%72x#RFH{\O?qHhע""8$_%AAȨy)tܳZ:E) #r]_6 =9܊#rwI~6"_v]4 ~8av;v8w8k; X5=/ ,===7'M,w9uxJY̑$L-̻VF撧L,Vj\$B]BúPJ'b, X믿>sLHHHÎ`8`blٲ6T$ph@/6_cN7sz3{VbPWB7DɄ$S/l-͛7nIJ/em;!i4-ܩa5n_qyX!6ˁ ~q\~U ;I/iUA/;Ҕ~#ͽ K.}`2OѥzV0x&"E)=D FBIpr{%4iʐo% +bIڮzwu%x>yK.&&K#џ{n쎽p˗qucì7|ΥG FM,K壘#̯IW$KjQϷbGL@PFGGh,>X20_aa~T5<%\8'N)ToLlM$Նw,IQHĐg̘I\EEEQ89]c BJct- X/؎[ '@hܳgŋcccI{ a3qeee8}vXロEztԺ+%`uY#8$ o<> lOV[`p8P3?} ~qqQ53x(`c9{r7VH`eRSSYuv:l@ / qɦlCެ_%>x4hV pB@`c Pd(-TT'g{ `El>SW֝$muBoe-Yݺ.T0͑F{La0p6`&+@A(//%3j"=)DIc{qҌdR,N8qSQk\ݻ-xBV$~UNm$"7rI (|q}Q.=kɭ0:λ !so&da꥙dLadAqL)+pI"DNWLdZBתeE)71VE0MK*\)I$(0NXg t/h%7 xB̾ڳcІ1mD|`;. S}fbRd$c 7Cnp 0IF wQ]▮xqtRHd/U9,Vo &`ʈ7\|XוFb/u3QL'51wt!@ EURK&JO_(d$,%W*đʊIN-S ܴijnCuTsB44)6`* `_b(]{M[(=0ÚdNkΜ9():{M(zNr$"dLL̸xc i%J: Ѕ3OUxg_\g\,!pd`H벦B)*8'W\yOiT^ȸHJID{tRĄݠՍC@IWp[kד~Db0cؗD%RBmK js*+M!0U{ŝ5RsV^ݠ(0]|)OʈS>ZtAgMtNv]8XBv_U!cHf}3 _xm~:K9~Gj kXؗ>L/-鍋)H']d㐫RBF]ȑ"WVbyG/߶GZW+ Y".Sm,bg[Tk>9X]!<#׸JFC֖!d{ +/8MB4oT`KNvWsc!S'ŗ~]J&;R31}ّa`Գk 0CIA?釞9} /.ɑdg2n6Pޚ{EjduT8D`,{K$]An+ EXhCDtSS4Ήƺ>F"~NA:398>Ʋc_o2*H= !姈}@Ye sQ+IIy{cA,r <%[<B3l>QQx0Úi z\ke'7eN$=x@+IrjG!1R!lj*Yܵ/Wuե4ALJ)@é4(䀊<"#dx$LCܩZ`|Ũ"B'2{:Mœ"p".KURkVhv[QO9pu!8iH->fQmX  f-(O>>jԠ#01(LLPum/6)< % .TW |\ J=[)h%ueS1ܬw&n<4n PTganÕ+ɬ/x2; wJ=m?RгKUID^JRm-,I 0KRmq83e+QOmkϙ(@Jofd8 v} @.˜=jDb6fˤRk~qѧŚFZ f>eҕ\'|<ٹafMyiuj|X)tM3 YMQ%ںDGiƛÁbDWf޸|Y S*D ڷ(@f6Ҋ)S Q P -`(@FJ3ŘĥUS PwR3#qiՔ`0#(H\Z5}(H3k-IENDB`execnet-2.1.1/doc/_templates/000077500000000000000000000000001460460142100160465ustar00rootroot00000000000000execnet-2.1.1/doc/_templates/indexsidebar.html000066400000000000000000000014461460460142100214020ustar00rootroot00000000000000

Download

{% if version.endswith('(hg)') %}

This documentation is for version {{ version }}, which is not released yet.

You can use it from the Git repo or look for released versions in the Python Package Index.

{% else %}

Current: {{ version }} [Changes]

Get execnet from the Python Package Index, or install it with:

pip install -U execnet
{% endif %}

Questions? Suggestions?

Join execnet-dev mailing list

come to #pytest on Libera Chat

execnet-2.1.1/doc/_templates/layout.html000066400000000000000000000015131460460142100202510ustar00rootroot00000000000000{% extends "!layout.html" %} {% block rootrellink %} {% endblock %} {% block header %}

execnet: Distributed Python deployment and communication

home |  install |  examples |  basic API |  support 
{% endblock %} execnet-2.1.1/doc/basics.rst000066400000000000000000000175051460460142100157170ustar00rootroot00000000000000============================================================================== API in a nutshell ============================================================================== execnet ad-hoc instantiates local and remote Python interpreters. Each interpreter is accessible through a **Gateway** which manages code and data communication. **Channels** allow to exchange data between the local and the remote end. **Groups** help to manage creation and termination of sub-interpreters. .. image:: _static/basic1.png .. currentmodule:: execnet Gateways: bootstrapping Python interpreters =================================================== All Gateways are instantiated via a call to ``makegateway()`` passing it a gateway specification or URL. .. _xspec: .. autofunction:: execnet.makegateway(spec) Here is an example which instantiates a simple Python subprocess:: >>> gateway = execnet.makegateway() Gateways allow to `remote execute code`_ and `exchange data`_ bidirectionally. Examples for valid gateway specifications ------------------------------------------- * ``ssh=wyvern//python=python3.3//chdir=mycache`` specifies a Python3.3 interpreter on the host ``wyvern``. The remote process will have ``mycache`` as its current working directory. * ``ssh=-p 5000 myhost`` makes execnet pass "-p 5000 myhost" arguments to the underlying ssh client binary, effectively specifying a custom port. * ``vagrant_ssh=default`` makes execnet connect to a Vagrant VM named ``default`` via SSH through Vagrant's ``vagrant ssh`` command. It supports the same additional parameters as regular SSH connections. * ``popen//python=python2.7//nice=20`` specification of a python subprocess using the ``python2.7`` executable which must be discoverable through the system ``PATH``; running with the lowest CPU priority ("nice" level). By default current dir will be the current dir of the instantiator. * ``popen//dont_write_bytecode`` uses the same executable as the current Python, and also passes the ``-B`` flag on startup, which tells Python not write ``.pyc`` or ``.pyo`` files. * ``popen//env:NAME=value`` specifies a subprocess that uses the same interpreter as the one it is initiated from and additionally remotely sets an environment variable ``NAME`` to ``value``. * ``popen//execmodel=eventlet`` specifies a subprocess that uses the same interpreter as the one it is initiated from but will run the other side using eventlet for handling IO and dispatching threads. * ``socket=192.168.1.4:8888`` specifies a Python Socket server process that listens on ``192.168.1.4:8888`` .. versionadded:: 1.5 * ``vagarant_ssh`` opens a python interpreter via the vagarant ssh command .. _`remote execute code`: remote_exec: execute source code remotely =================================================== .. currentmodule:: execnet.gateway All gateways offer a simple method to execute source code in the instantiated subprocess-interpreter: .. automethod:: Gateway.remote_exec(source) It is allowed to pass a module object as source code in which case its source code will be obtained and get sent for remote execution. ``remote_exec`` returns a channel object whose symmetric counterpart channel is available to the remotely executing source. .. method:: Gateway.reconfigure([py2str_as_py3str=True, py3str_as_py2str=False]) Reconfigures the string-coercion behaviour of the gateway .. _`Channel`: .. _`channel-api`: .. _`exchange data`: Channels: exchanging data with remote code ======================================================= .. currentmodule:: execnet.gateway_base A channel object allows to send and receive data between two asynchronously running programs. .. automethod:: Channel.send(item) .. automethod:: Channel.receive(timeout) .. automethod:: Channel.setcallback(callback, endmarker=_NOENDMARKER) .. automethod:: Channel.makefile(mode, proxyclose=False) .. automethod:: Channel.close(error) .. automethod:: Channel.waitclose(timeout) .. autoattribute:: Channel.RemoteError .. autoattribute:: Channel.TimeoutError .. _Group: Grouped Gateways and robust termination =============================================== .. currentmodule:: execnet.multi All created gateway instances are part of a group. If you call ``execnet.makegateway`` it actually is forwarded to the ``execnet.default_group``. Group objects are container objects (see :doc:`group examples `) and manage the final termination procedure: .. automethod:: Group.terminate(timeout=None) This method is implicitly called for each gateway group at process-exit, using a small timeout. This is fine for interactive sessions or random scripts which you rather like to error out than hang. If you start many processes then you often want to call ``group.terminate()`` yourself and specify a larger or not timeout. threading models: gevent, eventlet, thread, main_thread_only ==================================================================== .. versionadded:: 1.2 (status: experimental!) execnet supports "main_thread_only", "thread", "eventlet" and "gevent" as thread models on each of the two sides. You need to decide which model to use before you create any gateways:: # content of threadmodel.py import execnet # locally use "eventlet", remotely use "thread" model execnet.set_execmodel("eventlet", "thread") gw = execnet.makegateway() print (gw) print (gw.remote_status()) print (gw.remote_exec("channel.send(1)").receive()) You need to have eventlet installed in your environment and then you can execute this little test file:: $ python threadmodel.py 1 How to execute in the main thread ------------------------------------------------ When the remote side of a gateway uses the "thread" model, execution will preferably run in the main thread. This allows GUI loops or other code to behave correctly. If you, however, start multiple executions concurrently, they will run in non-main threads. remote_status: get low-level execution info =================================================== .. currentmodule:: execnet.gateway All gateways offer a simple method to obtain some status information from the remote side. .. automethod:: Gateway.remote_status(source) Calling this method tells you e.g. how many execution tasks are queued, how many are executing and how many channels are active. rsync: synchronise filesystem with remote =============================================================== .. currentmodule:: execnet ``execnet`` implements a simple efficient rsyncing protocol. Here is a basic example for using RSync:: rsync = execnet.RSync('/tmp/source') gw = execnet.makegateway() rsync.add_target(gw, '/tmp/dest') rsync.send() And here is API info about the RSync class. .. autoclass:: RSync :members: add_target,send Debugging execnet =============================================================== By setting the environment variable ``EXECNET_DEBUG`` you can configure a tracing mechanism: :EXECNET_DEBUG=1: write per-process trace-files to ``execnet-debug-PID`` :EXECNET_DEBUG=2: perform tracing to stderr (popen-gateway workers will send this to their instantiator) .. _`dumps/loads`: .. _`dumps/loads API`: Cross-interpreter serialization of Python objects ======================================================= .. versionadded:: 1.1 Execnet exposes a function pair which you can safely use to store and load values from different Python interpreters (e.g. Python2 and Python3, PyPy and Jython). Here is a basic example:: >>> import execnet >>> dump = execnet.dumps([1,2,3]) >>> execnet.loads(dump) [1,2,3] For more examples see :ref:`dumps/loads examples`. .. autofunction:: execnet.dumps(spec) .. autofunction:: execnet.loads(spec) execnet-2.1.1/doc/changelog.rst000066400000000000000000000001421460460142100163670ustar00rootroot00000000000000:tocdepth: 2 .. _changes: execnet CHANGELOG ******************** .. include:: ../CHANGELOG.rst execnet-2.1.1/doc/conf.py000066400000000000000000000145161460460142100152170ustar00rootroot00000000000000# # execnet documentation build configuration file, created by # sphinx-quickstart on Wed Sep 30 21:16:59 2009. # # This file is execfile()d with the wd set to its containing dir. # # Note that not all possible configuration values are present in this # autogenerated file. # # All configuration values have a default; values that are commented out # serve to show the default. import os import sys # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. sys.path.append(os.path.dirname(os.path.dirname(__file__))) from execnet._version import version release = ".".join(version.split(".")[:2]) # -- General configuration ---------------------------------------------------- # Add any Sphinx extension module names here, as strings. # They can be extensions # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. extensions = [ "sphinx.ext.autodoc", "sphinx.ext.doctest", "sphinx.ext.intersphinx", ] # Add any paths that contain templates here, relative to this directory. templates_path = ["_templates"] # The suffix of source filenames. source_suffix = ".rst" # The encoding of source files. # source_encoding = 'utf-8' # The master toctree document. master_doc = "index" # General information about the project. project = "execnet" copyright = "2012, holger krekel and others" # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. # language = None # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: # today = '' # Else, today_fmt is used as the format for a strftime call. # today_fmt = '%B %d, %Y' # List of documents that shouldn't be included in the build. # unused_docs = [] # List of directories, relative to source directory, that shouldn't be searched # for source files. exclude_trees = ["_build"] # The reST default role (used for this markup: `text`) to use for all documents # dfault_role = None # If true, '()' will be appended to :func: etc. cross-reference text. # add_function_parentheses = True # If true, the current module name will be prepended to all description # unit titles (such as .. function::). add_module_names = True linkcheck_timeout = 20 # If true, sectionauthor and moduleauthor directives will be shown in the # output. They are ignored by default. # show_authors = False # The name of the Pygments (syntax highlighting) style to use. pygments_style = "sphinx" # A list of ignored prefixes for module index sorting. # modindex_common_prefix = [] intersphinx_mapping = { "python": ("https://docs.python.org/3", None), } nitpicky = True nitpick_ignore = [ ("py:class", "execnet.gateway_base.ChannelFileRead"), ("py:class", "execnet.gateway_base.ChannelFileWrite"), ("py:class", "execnet.gateway.Gateway"), ] # -- Options for HTML output -------------------------------------------------- # The theme to use for HTML and HTML Help pages. Major themes that come with # Sphinx are currently 'default' and 'sphinxdoc'. html_theme = "sphinxdoc" # html_index = 'index.html' html_sidebars = {"index": ["indexsidebar.html"]} # html_additional_pages = {'index': 'index.html'} # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. # html_theme_options = {} # Add any paths that contain custom themes here, relative to this directory. # html_theme_path = [] # The name for this set of Sphinx documents. If None, it defaults to # " v documentation". # html_title = None # A shorter title for the navigation bar. Default is the same as html_title. # html_short_title = None # The name of an image file (relative to this directory) to place at the top # of the sidebar. # html_logo = "codespeak.png" # The name of an image file (within the static path) to use as favicon of the # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 # pixels large. # html_favicon = None # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". html_static_path = ["_static"] # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. # html_last_updated_fmt = '%b %d, %Y' # If true, SmartyPants will be used to convert quotes and dashes to # typographically correct entities. # html_use_smartypants = True # If false, no module index is generated. html_use_modindex = False # If false, no index is generated. # html_use_index = True # If true, the index is split into individual pages for each letter. # html_split_index = False # If true, links to the reST sources are added to the pages. html_show_sourcelink = False # If true, an OpenSearch description file will be output, and all pages will # contain a tag referring to it. The value of this option must be the # base URL from which the finished HTML is served. # html_use_opensearch = '' # If nonempty, this is the file name suffix for HTML files (e.g. ".xhtml"). # html_file_suffix = '' # Output file base name for HTML help builder. htmlhelp_basename = "execnetdoc" # -- Options for LaTeX output ------------------------------------------------- # The paper size ('letter' or 'a4'). # latex_paper_size = 'letter' # The font size ('10pt', '11pt' or '12pt'). # latex_font_size = '10pt' # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, documentclass [howto/manual]) latex_documents = [ ( "index", "execnet.tex", "execnet Documentation", "holger krekel and others", "manual", ) ] # The name of an image file (relative to this directory) to place at the top of # the title page. # latex_logo = None # For "manual" documents, if this is true, then toplevel headings are parts, # not chapters. # latex_use_parts = False # Additional stuff for the LaTeX preamble. # latex_preamble = '' # Documents to append as an appendix to all manuals. # latex_appendices = [] # If false, no module index is generated. # latex_use_modindex = True execnet-2.1.1/doc/example/000077500000000000000000000000001460460142100153445ustar00rootroot00000000000000execnet-2.1.1/doc/example/conftest.py000066400000000000000000000005731460460142100175500ustar00rootroot00000000000000import pathlib import sys # Make execnet and example code importable. cand = pathlib.Path(__file__).parent.parent.parent if cand.joinpath("execnet", "__init__.py").exists(): if str(cand) not in sys.path: sys.path.insert(0, str(cand)) cand = pathlib.Path(__file__).parent if str(cand) not in sys.path: sys.path.insert(0, str(cand)) pytest_plugins = ["doctest"] execnet-2.1.1/doc/example/funcmultiplier.py000066400000000000000000000005261460460142100207630ustar00rootroot00000000000000import execnet def multiplier(channel, factor): while not channel.isclosed(): param = channel.receive() channel.send(param * factor) gw = execnet.makegateway() channel = gw.remote_exec(multiplier, factor=10) for i in range(5): channel.send(i) result = channel.receive() assert result == i * 10 gw.exit() execnet-2.1.1/doc/example/hybridpython.rst000066400000000000000000000113541460460142100206250ustar00rootroot00000000000000Connecting different Python interpreters ========================================== .. _`dumps/loads examples`: Dumping and loading values across interpreter versions ---------------------------------------------------------- .. versionadded:: 1.1 Execnet offers a new safe and fast :ref:`dumps/loads API` which you can use to dump builtin python data structures and load them later with the same or a different python interpreter (including between Python2 and Python3). The standard library offers the pickle and marshal modules but they do not work safely between different interpreter versions. Using xml/json requires a mapping of Python objects and is not easy to get right. Moreover, execnet allows to control handling of bytecode/strings/unicode types. Here is an example:: # using python2 import execnet with open("data.py23", "wb") as f: f.write(execnet.dumps(["hello", "world"])) # using Python3 import execnet with open("data.py23", "rb") as f: val = execnet.loads(f.read(), py2str_as_py3str=True) assert val == ["hello", "world"] See the :ref:`dumps/loads API` for more details on string conversion options. Please note, that you can not dump user-level instances, only builtin python types. Connect to Python2/Numpy from Python3 ---------------------------------------- Here we run a Python3 interpreter to connect to a Python2.7 interpreter that has numpy installed. We send items to be added to an array and receive back the remote "repr" of the array:: import execnet gw = execnet.makegateway("popen//python=python2.7") channel = gw.remote_exec(""" import numpy array = numpy.array([1,2,3]) while 1: x = channel.receive() if x is None: break array = numpy.append(array, x) channel.send(repr(array)) """) for x in range(10): channel.send(x) channel.send(None) print (channel.receive()) will print on the CPython3.1 side:: array([1, 2, 3, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9]) A more refined real-life example of python3/python2 interaction is the anyvc_ project which uses version-control bindings in a Python2 subprocess in order to offer Python3-based library functionality. .. _anyvc: http://bitbucket.org/RonnyPfannschmidt/anyvc/overview/ Reconfiguring the string coercion between python2 and python3 ------------------------------------------------------------- Sometimes the default configuration of string coercion (2str to 3str, 3str to 2unicode) is inconvient, thus it can be reconfigured via `gw.reconfigure` and `channel.reconfigure`. Here is an example session on a Python2 interpreter:: >>> import execnet >>> execnet.makegateway("popen//python=python3.2") >>> gw=execnet.makegateway("popen//python=python3.2") >>> gw.remote_exec("channel.send('hello')").receive() u'hello' >>> gw.reconfigure(py3str_as_py2str=True) >>> gw.remote_exec("channel.send('hello')").receive() 'hello' >>> ch = gw.remote_exec('channel.send(type(channel.receive()).__name__)') >>> ch.send('a') >>> ch.receive() 'str' >>> ch = gw.remote_exec('channel.send(type(channel.receive()).__name__)') >>> ch.reconfigure(py2str_as_py3str=False) >>> ch.send('a') >>> ch.receive() u'bytes' Work with Java objects from CPython ---------------------------------------- Use your CPython interpreter to connect to a `Jython 2.5.1`_ interpreter and work with Java types:: import execnet gw = execnet.makegateway("popen//python=jython") channel = gw.remote_exec(""" from java.util import Vector v = Vector() v.add('aaa') v.add('bbb') for val in v: channel.send(val) """) for item in channel: print (item) will print on the CPython side:: aaa bbb .. _`Jython 2.5.1`: http://www.jython.org Work with C# objects from CPython ---------------------------------------- (Experimental) use your CPython interpreter to connect to a IronPython_ interpreter which can work with C# classes. Here is an example for instantiating a CLR Array instance and sending back its representation:: import execnet gw = execnet.makegateway("popen//python=ipy") channel = gw.remote_exec(""" import clr clr.AddReference("System") from System import Array array = Array[float]([1,2]) channel.send(str(array)) """) print (channel.receive()) using Mono 2.0 and IronPython-1.1 this will print on the CPython side:: System.Double[](1.0, 2.0) .. note:: Using IronPython needs more testing, likely newer versions will work better. please feedback if you have information. .. _IronPython: http://ironpython.net execnet-2.1.1/doc/example/popen_read_multiple.py000066400000000000000000000016721460460142100217530ustar00rootroot00000000000000""" example reading results from possibly blocking code running in sub processes. """ import execnet NUM_PROCESSES = 5 channels = [] for i in range(NUM_PROCESSES): gw = execnet.makegateway() # or use SSH or socket gateways channel = gw.remote_exec( """ import time secs = channel.receive() time.sleep(secs) channel.send("waited %d secs" % secs) """ ) channels.append(channel) print("*** instantiated subprocess", gw) mc = execnet.MultiChannel(channels) queue = mc.make_receive_queue() print("*** verifying that timeout on receiving results from blocked subprocesses works") try: queue.get(timeout=1.0) except Exception: pass print("*** sending subprocesses some data to have them unblock") mc.send_each(1) print("*** receiving results asynchronously") for i in range(NUM_PROCESSES): channel, result = queue.get(timeout=2.0) print("result", channel.gateway, result) execnet-2.1.1/doc/example/py3topy2.py000066400000000000000000000006101460460142100174240ustar00rootroot00000000000000import execnet gw = execnet.makegateway("popen//python=python2") channel = gw.remote_exec( """ import numpy array = numpy.array([1,2,3]) while 1: x = channel.receive() if x is None: break array = numpy.append(array, x) channel.send(repr(array)) """ ) for x in range(10): channel.send(x) channel.send(None) print(channel.receive()) execnet-2.1.1/doc/example/redirect_remote_output.py000066400000000000000000000012761460460142100225200ustar00rootroot00000000000000""" redirect output from remote to a local function showcasing features of the channel object: - sending a channel over a channel - adapting a channel to a file object - setting a callback for receiving channel data """ import execnet gw = execnet.makegateway() outchan = gw.remote_exec( """ import sys outchan = channel.gateway.newchannel() sys.stdout = outchan.makefile("w") channel.send(outchan) """ ).receive() # note: callbacks execute in receiver thread! def write(data): print("received:", repr(data)) outchan.setcallback(write) # type: ignore[attr-defined] gw.remote_exec( """ print('hello world') print('remote execution ends') """ ).waitclose() execnet-2.1.1/doc/example/remote1.py000066400000000000000000000002161460460142100172710ustar00rootroot00000000000000# content of a module remote1.py if __name__ == "__channelexec__": channel.send("initialization complete") # type: ignore[name-defined] execnet-2.1.1/doc/example/remotecmd.py000066400000000000000000000004271460460142100177000ustar00rootroot00000000000000import os # contents of: remotecmd.py def simple(arg): return arg + 1 def listdir(path): return os.listdir(path) if __name__ == "__channelexec__": for item in channel: # type: ignore[name-defined] channel.send(eval(item)) # type: ignore[name-defined] execnet-2.1.1/doc/example/servefiles.py000066400000000000000000000003631460460142100200670ustar00rootroot00000000000000# content of servefiles.py def servefiles(channel): for fn in channel: f = open(fn, "rb") channel.send(f.read()) f.close() if __name__ == "__channelexec__": servefiles(channel) # type: ignore[name-defined] execnet-2.1.1/doc/example/svn-sync-repo.py000066400000000000000000000074571460460142100204560ustar00rootroot00000000000000#!/usr/bin/env python """ small utility for hot-syncing a svn repository through ssh. uses execnet. """ import os import pathlib import subprocess import sys import execnet def usage(): arg0 = sys.argv[0] print(arg0, "[user@]remote-host:/repo/location localrepo [ssh-config-file]") def main(args): remote = args[0] localrepo = pathlib.Path(args[1]) if not localrepo.is_dir(): raise SystemExit(f"localrepo {localrepo} does not exist") if len(args) == 3: configfile = args[2] else: configfile = None remote_host, path = remote.split(":", 1) print("ssh-connecting to", remote_host) gw = getgateway(remote_host, configfile) local_rev = get_svn_youngest(localrepo) # local protocol # 1. client sends rev/repo -> server # 2. server checks for newer revisions and sends dumps # 3. client receives dumps, updates local repo # 4. client goes back to step 1 c = gw.remote_exec( """ import os import subprocess import time remote_rev, repopath = channel.receive() while True: rev = subprocess.run( ["svnlook", "youngest", repopath], check=True, capture_output=True, text=True, ).stdout rev = int(rev) if rev > remote_rev: revrange = (remote_rev+1, rev) dumpchannel = channel.gateway.newchannel() channel.send(revrange) channel.send(dumpchannel) f = os.popen( "svnadmin dump -q --incremental -r %s:%s %s" % (revrange[0], revrange[1], repopath), 'r') try: maxcount = dumpchannel.receive() count = maxcount while 1: s = f.read(8192) if not s: raise EOFError dumpchannel.send(s) count = count - 1 if count <= 0: ack = dumpchannel.receive() count = maxcount except EOFError: dumpchannel.close() remote_rev = rev else: # using svn-hook instead would be nice here time.sleep(30) """ ) c.send((local_rev, path)) print("checking revisions from %d in %s" % (local_rev, remote)) while 1: revstart, revend = c.receive() dumpchannel = c.receive() print("receiving revisions", revstart, "-", revend, "replaying...") svn_load(localrepo, dumpchannel) print("current revision", revend) def svn_load(repo, dumpchannel, maxcount=100): # every maxcount we will send an ACK to the other # side in order to synchronise and avoid our side # growing buffers (execnet does not control # RAM usage or receive queue sizes) dumpchannel.send(maxcount) f = os.popen(f"svnadmin load -q {repo}", "w") count = maxcount for x in dumpchannel: sys.stdout.write(".") sys.stdout.flush() f.write(x) count = count - 1 if count <= 0: dumpchannel.send(maxcount) count = maxcount print() f.close() def get_svn_youngest(repo): rev = subprocess.run( ["svnlook", "youngest", repo], check=True, capture_output=True, text=True, ).stdout return int(rev) def getgateway(host, configfile=None): xspec = "ssh=%s" % host if configfile is not None: xspec += "//ssh_config=%s" % configfile return execnet.makegateway(xspec) if __name__ == "__main__": if len(sys.argv) < 3: usage() raise SystemExit(1) main(sys.argv[1:]) execnet-2.1.1/doc/example/sysinfo.py000066400000000000000000000113721460460142100174140ustar00rootroot00000000000000""" sysinfo.py [host1] [host2] [options] obtain system info from remote machine. (c) Holger Krekel, MIT license """ import optparse import re import sys import execnet parser = optparse.OptionParser(usage=__doc__) parser.add_option( "-f", "--sshconfig", action="store", dest="ssh_config", default=None, help="use given ssh config file," " and add info all contained hosts for getting info", ) parser.add_option( "-i", "--ignore", action="store", dest="ignores", default=None, help="ignore hosts " "(useful if the list of hostnames come from a file list)", ) def parsehosts(path): host_regex = re.compile(r"Host\s*(\S+)") l = [] with open(path) as fp: for line in fp: m = host_regex.match(line) if m is not None: (sshname,) = m.groups() l.append(sshname) return l class RemoteInfo: def __init__(self, gateway): self.gw = gateway self._cache = {} def exreceive(self, execstring): if execstring not in self._cache: channel = self.gw.remote_exec(execstring) self._cache[execstring] = channel.receive() return self._cache[execstring] def getmodattr(self, modpath): module = modpath.split(".")[0] return self.exreceive( """ import %s channel.send(%s) """ % (module, modpath) ) def islinux(self): return self.getmodattr("sys.platform").find("linux") != -1 def getfqdn(self): return self.exreceive( """ import socket channel.send(socket.getfqdn()) """ ) def getmemswap(self): if self.islinux(): return self.exreceive( r""" import commands, re out = commands.getoutput("free") mem = re.search(r"Mem:\s+(\S*)", out).group(1) swap = re.search(r"Swap:\s+(\S*)", out).group(1) channel.send((mem, swap)) """ ) def getcpuinfo(self): if self.islinux(): return self.exreceive( """ # a hyperthreaded cpu core only counts as 1, although it # is present as 2 in /proc/cpuinfo. Counting it as 2 is # misleading because it is *by far* not as efficient as # two independent cores. cpus = {} cpuinfo = {} f = open("/proc/cpuinfo") lines = f.readlines() f.close() for line in lines + ['']: if line.strip(): key, value = line.split(":", 1) cpuinfo[key.strip()] = value.strip() else: corekey = (cpuinfo.get("physical id"), cpuinfo.get("core id")) cpus[corekey] = 1 numcpus = len(cpus) model = cpuinfo.get("model name") channel.send((numcpus, model)) """ ) def debug(*args): print(" ".join(map(str, args)), file=sys.stderr) def error(*args): debug("ERROR", args[0] + ":", *args[1:]) def getinfo(sshname, ssh_config=None, loginfo=sys.stdout): if ssh_config: spec = f"ssh=-F {ssh_config} {sshname}" else: spec = "ssh=%s" % sshname debug("connecting to", repr(spec)) try: gw = execnet.makegateway(spec) except OSError: error("could not get sshgatway", sshname) else: ri = RemoteInfo(gw) # print "%s info:" % sshname prefix = sshname.upper() + " " print(prefix, "fqdn:", ri.getfqdn(), file=loginfo) for attr in ("sys.platform", "sys.version_info"): loginfo.write(f"{prefix} {attr}: ") loginfo.flush() value = ri.getmodattr(attr) loginfo.write(str(value)) loginfo.write("\n") loginfo.flush() memswap = ri.getmemswap() if memswap: mem, swap = memswap print(prefix, "Memory:", mem, "Swap:", swap, file=loginfo) cpuinfo = ri.getcpuinfo() if cpuinfo: numcpu, model = cpuinfo print(prefix, "number of cpus:", numcpu, file=loginfo) print(prefix, "cpu model", model, file=loginfo) return ri if __name__ == "__main__": options, args = parser.parse_args() hosts = list(args) ssh_config = options.ssh_config if ssh_config: hosts.extend(parsehosts(ssh_config)) ignores = options.ignores or () if ignores: ignores = ignores.split(",") for host in hosts: if host not in ignores: getinfo(host, ssh_config=ssh_config) execnet-2.1.1/doc/example/taskserver.py000066400000000000000000000025041460460142100201100ustar00rootroot00000000000000from __future__ import annotations import execnet group = execnet.Group() for i in range(4): # 4 CPUs group.makegateway() def process_item(channel): # task processor, sits on each CPU import time import random channel.send("ready") for x in channel: if x is None: # we can shutdown break # sleep random time, send result time.sleep(random.randrange(3)) channel.send(x * 10) # execute taskprocessor everywhere mch = group.remote_exec(process_item) # get a queue that gives us results q = mch.make_receive_queue(endmarker=-1) tasks: list[int] | None = list(range(10)) # a list of tasks, here just integers terminated = 0 while 1: channel, item = q.get() if item == -1: terminated += 1 print("terminated %s" % channel.gateway.id) if terminated == len(mch): print("got all results, terminating") break continue if item != "ready": print(f"other side {channel.gateway.id} returned {item!r}") if not tasks and tasks is not None: print("no tasks remain, sending termination request to all") mch.send_each(None) tasks = None if tasks: task = tasks.pop() channel.send(task) print(f"sent task {task!r} to {channel.gateway.id}") group.terminate() execnet-2.1.1/doc/example/test_debug.rst000066400000000000000000000037231460460142100202300ustar00rootroot00000000000000 Debugging execnet / wire messages =============================================================== By setting the environment variable ``EXECNET_DEBUG`` you can configure the execnet tracing mechanism: :EXECNET_DEBUG=1: write per-process trace-files to ``${TEMPROOT}/execnet-debug-PID`` :EXECNET_DEBUG=2: perform tracing to stderr (popen-gateway workers will send this to their instantiator) Here is a simple example to see what goes on with a simple execution:: EXECNET_DEBUG=2 # or "set EXECNET_DEBUG=2" on windows python -c 'import execnet ; execnet.makegateway().remote_exec("42")' which will show PID-prefixed trace entries:: [2326] gw0 starting to receive [2326] gw0 sent [2327] creating workergateway on [2327] gw0-worker starting to receive [2327] gw0-worker received [2327] gw0-worker execution starts[1]: '42' [2327] gw0-worker execution finished [2327] gw0-worker sent [2327] gw0-worker 1 sent channel close message [2326] gw0 received [2326] gw0 1 channel.__del__ [2326] === atexit cleanup === [2326] gw0 gateway.exit() called [2326] gw0 --> sending GATEWAY_TERMINATE [2326] gw0 sent [2326] gw0 joining receiver thread [2327] gw0-worker received [2327] gw0-worker putting None to execqueue [2327] gw0-worker io.close_read() [2327] gw0-worker leaving [2327] gw0-worker 1 channel.__del__ [2327] gw0-worker io.close_write() [2327] gw0-worker workergateway.serve finished [2327] gw0-worker gateway.join() called while receiverthread already finished [2326] gw0 leaving execnet-2.1.1/doc/example/test_funcmultiplier.py000066400000000000000000000000571460460142100220210ustar00rootroot00000000000000def test_function(): import funcmultiplier execnet-2.1.1/doc/example/test_group.rst000066400000000000000000000066551460460142100203050ustar00rootroot00000000000000Managing multiple gateways and clusters ================================================== Usings Groups for managing multiple gateways ------------------------------------------------------ Use ``execnet.Group`` to manage membership and lifetime of multiple gateways:: >>> import execnet >>> group = execnet.Group(['popen'] * 2) >>> len(group) 2 >>> group >>> list(group) [, ] >>> 'gw0' in group and 'gw1' in group True >>> group['gw0'] == group[0] True >>> group['gw1'] == group[1] True >>> group.terminate() # exit all member gateways >>> group Assigning gateway IDs ------------------------------------------------------ All gateways are created as part of a group and receive a per-group unique ``id`` after successful initialization. Pass an ``id=MYNAME`` part to ``group.makegateway``. Example:: >>> import execnet >>> group = execnet.Group() >>> gw = group.makegateway("popen//id=sub1") >>> assert gw.id == "sub1" >>> group['sub1'] Getting (auto) IDs before instantiation ------------------------------------------------------ Sometimes it's useful to know the gateway ID ahead of instantiating it:: >>> import execnet >>> group = execnet.Group() >>> spec = execnet.XSpec("popen") >>> group.allocate_id(spec) >>> allocated_id = spec.id >>> gw = group.makegateway(spec) >>> assert gw.id == allocated_id execnet.makegateway uses execnet.default_group ------------------------------------------------------ Each time you create a gateway with ``execnet.makegateway()`` you actually use the ``execnet.default_group``:: >>> import execnet >>> gw = execnet.makegateway() >>> gw in execnet.default_group True >>> execnet.default_group.defaultspec # used for empty makegateway() calls 'popen' Robust termination of SSH/popen processes ----------------------------------------------- Use ``group.terminate(timeout)`` if you want to terminate member gateways and ensure that no local subprocesses remain. You can specify a ``timeout`` after which an attempt at killing the related process is made:: >>> import execnet >>> group = execnet.Group() >>> gw = group.makegateway("popen//id=sleeper") >>> ch = gw.remote_exec("import time ; time.sleep(2.0)") >>> group >>> group.terminate(timeout=1.0) >>> group execnet aims to provide totally robust termination so if you have left-over processes or other termination issues please :doc:`report them <../support>`. Thanks! Using Groups to manage a certain type of gateway ------------------------------------------------------ Set ``group.defaultspec`` to determine the default gateway specification used by ``group.makegateway()``: >>> import execnet >>> group = execnet.Group() >>> group.defaultspec = "ssh=localhost//chdir=mytmp//nice=20" >>> gw = group.makegateway() >>> ch = gw.remote_exec(""" ... import os.path ... basename = os.path.basename(os.getcwd()) ... channel.send(basename) ... """) >>> ch.receive() 'mytmp' This way a Group object becomes kind of a Gateway factory where the factory-caller does not need to know the setup. execnet-2.1.1/doc/example/test_info.rst000066400000000000000000000141771460460142100201020ustar00rootroot00000000000000Basic local and remote communication ==================================== Execute source code in subprocess, communicate through a channel ------------------------------------------------------------------- You can instantiate a subprocess gateway, execute code in it and bidirectionally send messages:: >>> import execnet >>> gw = execnet.makegateway() >>> channel = gw.remote_exec("channel.send(channel.receive()+1)") >>> channel.send(1) >>> channel.receive() 2 The initiating and the remote execution happen concurrently. ``channel.receive()`` operations return when input is available. ``channel.send(data)`` operations return when the message could be delivered to the IO system. The initiating and the "other" process work use a `share-nothing model`_ and ``channel.send|receive`` are means to pass basic data messages between two processes. .. _`share-nothing model`: http://en.wikipedia.org/wiki/Shared_nothing_architecture Remote-exec a function (avoiding inlined source part I) ------------------------------------------------------- You can send and remote execute parametrized pure functions like this: .. include:: funcmultiplier.py :literal: The ``multiplier`` function executes remotely and establishes a loop multipliying incoming data with a constant factor passed in via keyword arguments to ``remote_exec``. Notes: * unfortunately, you can not type this example interactively because ``inspect.getsource(func)`` fails for interactively defined functions. * You will get an explicit error if you try to execute non-pure functions, i.e. functions that access any global state (which will not be available remotely as we have a share-nothing model between the nodes). Remote-exec a module (avoiding inlined source part II) ------------------------------------------------------ You can pass a module object to ``remote_exec`` in which case its source code will be sent. No dependencies will be transferred so the module must be self-contained or only use modules that are installed on the "other" side. Module code can detect if it is running in a remote_exec situation by checking for the special ``__name__`` attribute. .. include:: remote1.py :literal: You can now remote-execute the module like this:: >>> import execnet, remote1 >>> gw = execnet.makegateway() >>> ch = gw.remote_exec(remote1) >>> print (ch.receive()) initialization complete which will print the 'initialization complete' string. Compare current working directories ---------------------------------------- A local subprocess gateway has the same working directory as the instantiatior:: >>> import execnet, os >>> gw = execnet.makegateway() >>> ch = gw.remote_exec("import os; channel.send(os.getcwd())") >>> res = ch.receive() >>> assert res == os.getcwd() "ssh" gateways default to the login home directory. Get information from remote SSH account --------------------------------------- Use simple execution to obtain information from remote environments:: >>> import execnet, os >>> gw = execnet.makegateway("ssh=codespeak.net") >>> channel = gw.remote_exec(""" ... import sys, os ... channel.send((sys.platform, tuple(sys.version_info), os.getpid())) ... """) >>> platform, version_info, remote_pid = channel.receive() >>> platform 'linux2' >>> version_info (2, 6, 6, 'final', 0) Use a callback instead of receive() and wait for completion ------------------------------------------------------------- Set a channel callback to immediately react on incoming data:: >>> import execnet >>> gw = execnet.makegateway() >>> channel = gw.remote_exec("for i in range(10): channel.send(i)") >>> l = [] >>> channel.setcallback(l.append, endmarker=None) >>> channel.waitclose() # waits for closing, i.e. remote exec finish >>> l [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, None] Note that the callback function will execute in the receiver thread so it should not block on IO or long to execute. Sending channels over channels ------------------------------------------------------ You can create and transfer a channel over an existing channel and use it to transfer information:: >>> import execnet >>> gw = execnet.makegateway() >>> channel = gw.remote_exec(""" ... ch1, ch2 = channel.receive() ... ch2.send("world") ... ch1.send("hello") ... """) >>> c1 = gw.newchannel() # create new channel >>> c2 = gw.newchannel() # create another channel >>> channel.send((c1, c2)) # send them over >>> c1.receive() 'hello' >>> c2.receive() 'world' A simple command loop pattern -------------------------------------------------------------- If you want the remote side to serve a number of synchronous function calls into your module you can setup a serving loop and write a local protocol. .. include:: remotecmd.py :literal: Then on the local side you can do:: >>> import execnet, remotecmd >>> gw = execnet.makegateway() >>> ch = gw.remote_exec(remotecmd) >>> ch.send('simple(10)') # execute func-call remotely >>> ch.receive() 11 Our remotecmd module starts up remote serving through the ``for item in channel`` loop which will terminate when the channel closes. It evaluates all incoming requests in the global name space and sends back the results. Instantiate gateways through sockets ----------------------------------------------------- .. _`socketserver.py`: https://raw.githubusercontent.com/pytest-dev/execnet/master/execnet/script/socketserver.py In cases where you do not have SSH-access to a machine you need to download a small version-independent standalone `socketserver.py`_ script to provide a remote bootstrapping-point. You do not need to install the execnet package remotely. Simply run the script like this:: python socketserver.py :8888 # bind to all IPs, port 8888 You can then instruct execnet on your local machine to bootstrap itself into the remote socket endpoint:: import execnet gw = execnet.makegateway("socket=TARGET-IP:8888") That's it, you can now use the gateway object just like a popen- or SSH-based one. .. include:: test_ssh_fileserver.rst execnet-2.1.1/doc/example/test_multi.rst000066400000000000000000000056561460460142100203030ustar00rootroot00000000000000Advanced (multi) channel communication ===================================================== MultiChannel: container for multiple channels ------------------------------------------------------ Use ``execnet.MultiChannel`` to work with multiple channels:: >>> import execnet >>> ch1 = execnet.makegateway().remote_exec("channel.send(1)") >>> ch2 = execnet.makegateway().remote_exec("channel.send(2)") >>> mch = execnet.MultiChannel([ch1, ch2]) >>> len(mch) 2 >>> mch[0] is ch1 and mch[1] is ch2 True >>> ch1 in mch and ch2 in mch True >>> sum(mch.receive_each()) 3 Receive results from sub processes with a Queue ----------------------------------------------------- Use ``MultiChannel.make_receive_queue()`` to get a queue from which to obtain results:: >>> ch1 = execnet.makegateway().remote_exec("channel.send(1)") >>> ch2 = execnet.makegateway().remote_exec("channel.send(2)") >>> mch = execnet.MultiChannel([ch1, ch2]) >>> queue = mch.make_receive_queue() >>> chan1, res1 = queue.get() >>> chan2, res2 = queue.get(timeout=3) >>> res1 + res2 3 Working asynchronously/event-based with channels --------------------------------------------------- Use channel callbacks if you want to process incoming data immediately and without blocking execution:: >>> import execnet >>> gw = execnet.makegateway() >>> ch = gw.remote_exec("channel.receive() ; channel.send(42)") >>> l = [] >>> ch.setcallback(l.append) >>> ch.send(1) >>> ch.waitclose() >>> assert l == [42] Note that the callback function will be executed in the receiver thread and should not block or run for too long. Robustly receive results and termination notification ----------------------------------------------------- Use ``MultiChannel.make_receive_queue(endmarker)`` to specify an object to be put to the queue when the remote side of a channel is closed. The endmarker will also be put to the Queue if the gateway is blocked in execution and is terminated/killed:: >>> group = execnet.Group(['popen'] * 3) # create three gateways >>> mch = group.remote_exec("channel.send(channel.receive()+1)") >>> queue = mch.make_receive_queue(endmarker=42) >>> mch[0].send(1) >>> chan1, res1 = queue.get() >>> res1 2 >>> group.terminate(timeout=1) # kill processes waiting on receive >>> for i in range(3): ... chan1, res1 = queue.get() ... assert res1 == 42 >>> group Saturate multiple Hosts and CPUs with tasks to process -------------------------------------------------------- If you have multiple CPUs or hosts you can create as many gateways and then have a process sit on each CPU and wait for a task to proceed. One complication is that we want to ensure clean termination of all processes and loose no result. Here is an example that just uses local subprocesses and does the task: .. include:: taskserver.py :literal: execnet-2.1.1/doc/example/test_proxy.rst000066400000000000000000000013521460460142100203170ustar00rootroot00000000000000Managing proxied gateways ========================== Simple proxying ---------------- Using the ``via`` arg of specs we can create a gateway whose io is created on a remote gateway and proxied to the master. The simplest use case, is where one creates one master process and uses it to control new workers and their environment :: >>> import execnet >>> group = execnet.Group() >>> group.defaultspec = 'popen//via=master' >>> master = group.makegateway('popen//id=master') >>> master >>> worker = group.makegateway() >>> worker >>> group execnet-2.1.1/doc/example/test_ssh_fileserver.rst000066400000000000000000000010751460460142100221630ustar00rootroot00000000000000Receive file contents from remote SSH account ----------------------------------------------------- Here is some small server code that you can use to retrieve contents of remote files: .. include:: servefiles.py :literal: And here is some code to use it to retrieve remote contents:: import execnet import servefiles gw = execnet.makegateway("ssh=codespeak.net") channel = gw.remote_exec(servefiles) for fn in ('/etc/passwd', '/etc/group'): channel.send(fn) content = channel.receive() print(fn) print(content) execnet-2.1.1/doc/examples.rst000066400000000000000000000011331460460142100162570ustar00rootroot00000000000000============================================================================== examples ============================================================================== .. _`execnet-dev`: http://mail.python.org/mailman/listinfo/execnet-dev .. _`execnet-commit`: http://mail.python.org/mailman/listinfo/execnet-commit Note: all examples with `>>>` prompts are automatically tested. .. toctree:: :maxdepth: 2 example/test_info example/test_group example/test_proxy example/test_multi example/hybridpython example/test_debug .. toctree:: :hidden: example/test_ssh_fileserver execnet-2.1.1/doc/implnotes.rst000066400000000000000000000023271460460142100164610ustar00rootroot00000000000000gateway_base.py ---------------------- The code of this module is sent to the "other side" as a means of bootstrapping a Gateway object capable of receiving and executing code, and routing data through channels. Gateways operate on InputOutput objects offering a write and a read(n) method. Once bootstrapped a higher level protocol based on Messages is used. Messages are serialized to and from InputOutput objects. The details of this protocol are locally defined in this module. There is no need for standardizing or versioning the protocol. After bootstrapping the BaseGateway opens a receiver thread which accepts encoded messages and triggers actions to interpret them. Sending of channel data items happens directly through write operations to InputOutput objects so there is no separate thread. Code execution messages are put into an execqueue from which they will be taken for execution. gateway.serve() will take and execute such items, one by one. This means that by incoming default execution is single-threaded. The receiver thread terminates if the remote side sends a gateway termination message or if the IO-connection drops. It puts an end symbol into the execqueue so that serve() can cleanly finish as well. execnet-2.1.1/doc/index.rst000066400000000000000000000063541460460142100155620ustar00rootroot00000000000000.. image:: _static/pythonring.png :align: right .. warning:: execnet currently is in maintenance-only mode, mostly because it is still the backend of the pytest-xdist plugin. Do not use in new projects. Python_ is a mature dynamic language whose interpreters can interact with all major computing platforms today. **execnet** provides a `share-nothing model`_ with `channel-send/receive`_ communication for distributing execution across many Python interpreters across version, platform and network barriers. It has a minimal and fast API targeting the following uses: * Distribute tasks to (many) local or remote CPUs * Write and deploy hybrid multi-process applications * Write scripts to administer multiple environments .. _`channel-send/receive`: http://en.wikipedia.org/wiki/Channel_(programming) .. _`share-nothing model`: http://en.wikipedia.org/wiki/Shared_nothing_architecture .. _Python: http://www.python.org Features ------------------ * Automatic bootstrapping: no manual remote installation. * Safe and simple serialization of Python builtin types for sending/receiving structured data messages. (New in 1.1) execnet offers a new :ref:`dumps/loads ` API which allows cross-interpreter compatible serialization of Python builtin types. * Flexible communication: synchronous send/receive as well as callback/queue mechanisms supported * Easy creation, handling and termination of multiple processes * Well tested interactions between CPython 2.5-2.7, CPython-3.3, Jython 2.5.1 and PyPy interpreters. * Fully interoperable between Windows and Unix-ish systems. * Many tested :doc:`examples` Known uses ------------------- * `pytest`_ uses it for its `distributed testing`_ mechanism. * `quora`_ uses it for `connecting CPython and PyPy`_. * Jacob Perkins uses it for his `Distributed NTLK with execnet`_ project to launch computation processes through ssh. He also compares `disco and execnet`_ in a subsequent post. * Ronny Pfannschmidt uses it for his `anyvc`_ VCS-abstraction project to bridge the Python2/Python3 version gap. * Sysadmins and developers are using it for ad-hoc custom scripting .. _`quora`: http://quora.com .. _`connecting CPython and PyPy`: http://www.quora.com/Quora-Infrastructure/Did-Quoras-switch-to-PyPy-result-in-increased-memory-consumption .. _`pytest`: https://docs.pytest.org .. _`distributed testing`: https://pypi.python.org/pypi/pytest-xdist .. _`Distributed NTLK with execnet`: http://streamhacker.com/2009/11/29/distributed-nltk-execnet/ .. _`disco and execnet`: http://streamhacker.com/2009/12/14/execnet-disco-distributed-nltk/ .. _`anyvc`: http://bitbucket.org/RonnyPfannschmidt/anyvc/ Project status -------------------------- The project is currently in **maintenance-only mode**, with PRs fixing bugs being gracefully accepted. Currently there are no plans to improve the project further, being maintained mostly because it is used as backend of the popular `pytest-xdist `__ plugin. ``execnet`` was conceived originally by `Holger Krekel`_ and is licensed under the MIT license since version 1.2. .. _`basic API`: basics.html .. _`Holger Krekel`: http://twitter.com/hpk42 .. toctree:: :hidden: support implnotes install execnet-2.1.1/doc/install.rst000066400000000000000000000014361460460142100161150ustar00rootroot00000000000000Info in a nutshell ==================== **Pythons**: 3.8+, PyPy 3 **Operating systems**: Linux, Windows, OSX, Unix **Distribution names**: * PyPI name: ``execnet`` * Redhat Fedora: ``python-execnet`` * Debian: ``python-execnet`` * Gentoo: ``dev-python/execnet`` **git repository**: https://github.com/pytest-dev/execnet Installation ==================== Install via pip_:: pip install execnet Next checkout the basic api and examples: .. toctree:: :maxdepth: 1 examples basics changelog .. _pip: http://pypi.python.org/pypi/pip .. _`github repository`: https://github.com/pytest-dev/execnet .. _`execnet git repository`: https://github.com/pytest-dev/execnet .. _`pypi release`: http://pypi.python.org/pypi/execnet .. _distribute: http://pypi.python.org/pypi/distribute execnet-2.1.1/doc/support.rst000066400000000000000000000013261460460142100161610ustar00rootroot00000000000000Contact and Support channels ------------------------------ If you have interest, questions, issues or suggestions you are welcome to: * Join `execnet-dev`_ for general discussions * Join `execnet-commit`_ to be notified of changes * Clone the `github repository`_ and submit patches * Hang out on the #pytest channel on `irc.libera.chat `_ (using an IRC client, via `webchat `_, or `via Matrix `_). .. _`execnet-dev`: http://mail.python.org/mailman/listinfo/execnet-dev .. _`execnet-commit`: http://mail.python.org/mailman/listinfo/execnet-commit .. _`github repository`: https://github.com/pytest-dev/execnet execnet-2.1.1/pyproject.toml000066400000000000000000000055251460460142100160670ustar00rootroot00000000000000[build-system] requires = [ "hatchling", "hatch-vcs", ] build-backend = "hatchling.build" [project] name = "execnet" dynamic = ["version"] description = "execnet: rapid multi-Python deployment" readme = {"file" = "README.rst", "content-type" = "text/x-rst"} license = "MIT" requires-python = ">=3.8" authors = [ { name = "holger krekel and others" }, ] classifiers = [ "Development Status :: 5 - Production/Stable", "Intended Audience :: Developers", "License :: OSI Approved :: MIT License", "Operating System :: MacOS :: MacOS X", "Operating System :: Microsoft :: Windows", "Operating System :: POSIX", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", "Topic :: Software Development :: Libraries", "Topic :: System :: Distributed Computing", "Topic :: System :: Networking", ] [project.optional-dependencies] testing = [ "pre-commit", "pytest", "tox", "hatch", ] [project.urls] Homepage = "https://execnet.readthedocs.io/en/latest/" [tool.ruff.lint] extend-select = [ "B", # bugbear "E", # pycodestyle "F", # pyflakes "I", # isort "PYI", # flake8-pyi "UP", # pyupgrade "RUF", # ruff "W", # pycodestyle "PIE", # flake8-pie "PGH", # pygrep-hooks "PLE", # pylint error "PLW", # pylint warning ] ignore = [ # bugbear ignore "B007", # Loop control variable `i` not used within loop body "B011", # Do not `assert False` (`python -O` removes these calls) # pycodestyle ignore "E501", # Line too long "E741", # Ambiguous variable name # ruff ignore "RUF012", # Mutable class attributes should be annotated with `typing.ClassVar` # pylint ignore "PLW0603", # Using the global statement "PLW0120", # remove the else and dedent its contents "PLW2901", # for loop variable overwritten by assignment target "PLR5501", # Use `elif` instead of `else` then `if` ] [tool.ruff.lint.isort] force-single-line = true known-third-party = ["src"] [tool.hatch.version] source = "vcs" [tool.hatch.build.hooks.vcs] version-file = "src/execnet/_version.py" [tool.hatch.build.targets.sdist] include = [ "/doc", "/src", "/testing", "tox.ini", ] [tool.mypy] python_version = "3.8" mypy_path = ["src"] files = ["src", "testing"] strict = true warn_unreachable = true warn_unused_ignores = false disallow_untyped_calls = false disallow_untyped_defs = false disallow_incomplete_defs = false [[tool.mypy.overrides]] module = [ "eventlet.*", "gevent.thread.*", ] ignore_missing_imports = true execnet-2.1.1/src/000077500000000000000000000000001460460142100137335ustar00rootroot00000000000000execnet-2.1.1/src/execnet/000077500000000000000000000000001460460142100153665ustar00rootroot00000000000000execnet-2.1.1/src/execnet/__init__.py000066400000000000000000000022051460460142100174760ustar00rootroot00000000000000""" execnet ------- pure python lib for connecting to local and remote Python Interpreters. (c) 2012, Holger Krekel and others """ from ._version import version as __version__ from .gateway import Gateway from .gateway_base import Channel from .gateway_base import DataFormatError from .gateway_base import DumpError from .gateway_base import LoadError from .gateway_base import RemoteError from .gateway_base import TimeoutError from .gateway_base import dump from .gateway_base import dumps from .gateway_base import load from .gateway_base import loads from .gateway_bootstrap import HostNotFound from .multi import Group from .multi import MultiChannel from .multi import default_group from .multi import makegateway from .multi import set_execmodel from .rsync import RSync from .xspec import XSpec __all__ = [ "__version__", "makegateway", "set_execmodel", "HostNotFound", "RemoteError", "TimeoutError", "XSpec", "Gateway", "Group", "MultiChannel", "RSync", "default_group", "Channel", "dumps", "dump", "DumpError", "loads", "load", "LoadError", "DataFormatError", ] execnet-2.1.1/src/execnet/gateway.py000066400000000000000000000171471460460142100174130ustar00rootroot00000000000000"""Gateway code for initiating popen, socket and ssh connections. (c) 2004-2013, Holger Krekel and others """ from __future__ import annotations import inspect import linecache import textwrap import types from typing import TYPE_CHECKING from typing import Any from typing import Callable from . import gateway_base from .gateway_base import IO from .gateway_base import Channel from .gateway_base import Message from .multi import Group from .xspec import XSpec class Gateway(gateway_base.BaseGateway): """Gateway to a local or remote Python Interpreter.""" _group: Group def __init__(self, io: IO, spec: XSpec) -> None: """:private:""" super().__init__(io=io, id=spec.id, _startcount=1) self.spec = spec self._initreceive() @property def remoteaddress(self) -> str: # Only defined for remote IO types. return self._io.remoteaddress # type: ignore[attr-defined,no-any-return] def __repr__(self) -> str: """A string representing gateway type and status.""" try: r: str = self.hasreceiver() and "receive-live" or "not-receiving" i = str(len(self._channelfactory.channels())) except AttributeError: r = "uninitialized" i = "no" return f"<{self.__class__.__name__} id={self.id!r} {r}, {self.execmodel.backend} model, {i} active channels>" def exit(self) -> None: """Trigger gateway exit. Defer waiting for finishing of receiver-thread and subprocess activity to when group.terminate() is called. """ self._trace("gateway.exit() called") if self not in self._group: self._trace("gateway already unregistered with group") return self._group._unregister(self) try: self._trace("--> sending GATEWAY_TERMINATE") self._send(Message.GATEWAY_TERMINATE) self._trace("--> io.close_write") self._io.close_write() except (ValueError, EOFError, OSError) as exc: self._trace("io-error: could not send termination sequence") self._trace(" exception: %r" % exc) def reconfigure( self, py2str_as_py3str: bool = True, py3str_as_py2str: bool = False ) -> None: """Set the string coercion for this gateway. The default is to try to convert py2 str as py3 str, but not to try and convert py3 str to py2 str. """ self._strconfig = (py2str_as_py3str, py3str_as_py2str) data = gateway_base.dumps_internal(self._strconfig) self._send(Message.RECONFIGURE, data=data) def _rinfo(self, update: bool = False) -> RInfo: """Return some sys/env information from remote.""" if update or not hasattr(self, "_cache_rinfo"): ch = self.remote_exec(rinfo_source) try: self._cache_rinfo = RInfo(ch.receive()) finally: ch.waitclose() return self._cache_rinfo def hasreceiver(self) -> bool: """Whether gateway is able to receive data.""" return self._receivepool.active_count() > 0 def remote_status(self) -> RemoteStatus: """Obtain information about the remote execution status.""" channel = self.newchannel() self._send(Message.STATUS, channel.id) statusdict = channel.receive() # the other side didn't actually instantiate a channel # so we just delete the internal id/channel mapping self._channelfactory._local_close(channel.id) return RemoteStatus(statusdict) def remote_exec( self, source: str | types.FunctionType | Callable[..., object] | types.ModuleType, **kwargs: object, ) -> Channel: """Return channel object and connect it to a remote execution thread where the given ``source`` executes. * ``source`` is a string: execute source string remotely with a ``channel`` put into the global namespace. * ``source`` is a pure function: serialize source and call function with ``**kwargs``, adding a ``channel`` object to the keyword arguments. * ``source`` is a pure module: execute source of module with a ``channel`` in its global namespace. In all cases the binding ``__name__='__channelexec__'`` will be available in the global namespace of the remotely executing code. """ call_name = None file_name = None if isinstance(source, types.ModuleType): file_name = inspect.getsourcefile(source) linecache.updatecache(file_name) # type: ignore[arg-type] source = inspect.getsource(source) elif isinstance(source, types.FunctionType): call_name = source.__name__ file_name = inspect.getsourcefile(source) source = _source_of_function(source) else: source = textwrap.dedent(str(source)) if not call_name and kwargs: raise TypeError("can't pass kwargs to non-function remote_exec") channel = self.newchannel() self._send( Message.CHANNEL_EXEC, channel.id, gateway_base.dumps_internal((source, file_name, call_name, kwargs)), ) return channel def remote_init_threads(self, num: int | None = None) -> None: """DEPRECATED. Is currently a NO-OPERATION already.""" print("WARNING: remote_init_threads() is a no-operation in execnet-1.2") class RInfo: def __init__(self, kwargs) -> None: self.__dict__.update(kwargs) def __repr__(self) -> str: info = ", ".join(f"{k}={v}" for k, v in sorted(self.__dict__.items())) return "" % info if TYPE_CHECKING: def __getattr__(self, name: str) -> Any: ... RemoteStatus = RInfo def rinfo_source(channel) -> None: import os import sys channel.send( dict( executable=sys.executable, version_info=sys.version_info[:5], platform=sys.platform, cwd=os.getcwd(), pid=os.getpid(), ) ) def _find_non_builtin_globals(source: str, codeobj: types.CodeType) -> list[str]: import ast import builtins vars = dict.fromkeys(codeobj.co_varnames) return [ node.id for node in ast.walk(ast.parse(source)) if isinstance(node, ast.Name) and node.id not in vars and node.id not in builtins.__dict__ ] def _source_of_function(function: types.FunctionType | Callable[..., object]) -> str: if function.__name__ == "": raise ValueError("can't evaluate lambda functions'") # XXX: we dont check before remote instantiation # if arguments are used properly try: sig = inspect.getfullargspec(function) except AttributeError: args = inspect.getargspec(function)[0] else: args = sig.args if not args or args[0] != "channel": raise ValueError("expected first function argument to be `channel`") closure = function.__closure__ codeobj = function.__code__ if closure is not None: raise ValueError("functions with closures can't be passed") try: source = inspect.getsource(function) except OSError as e: raise ValueError("can't find source file for %s" % function) from e source = textwrap.dedent(source) # just for inner functions used_globals = _find_non_builtin_globals(source, codeobj) if used_globals: raise ValueError("the use of non-builtin globals isn't supported", used_globals) leading_ws = "\n" * (codeobj.co_firstlineno - 1) return leading_ws + source execnet-2.1.1/src/execnet/gateway_base.py000066400000000000000000001633211460460142100204010ustar00rootroot00000000000000"""Base execnet gateway code send to the other side for bootstrapping. :copyright: 2004-2015 :authors: - Holger Krekel - Armin Rigo - Benjamin Peterson - Ronny Pfannschmidt - many others """ from __future__ import annotations import abc import os import struct import sys import traceback import weakref from _thread import interrupt_main from io import BytesIO from typing import Any from typing import Callable from typing import Iterator from typing import Literal from typing import MutableSet from typing import Protocol from typing import cast from typing import overload class WriteIO(Protocol): def write(self, data: bytes, /) -> None: ... class ReadIO(Protocol): def read(self, numbytes: int, /) -> bytes: ... class IO(Protocol): execmodel: ExecModel def read(self, numbytes: int, /) -> bytes: ... def write(self, data: bytes, /) -> None: ... def close_read(self) -> None: ... def close_write(self) -> None: ... def wait(self) -> int | None: ... def kill(self) -> None: ... class Event(Protocol): """Protocol for types which look like threading.Event.""" def is_set(self) -> bool: ... def set(self) -> None: ... def clear(self) -> None: ... def wait(self, timeout: float | None = None) -> bool: ... class ExecModel(metaclass=abc.ABCMeta): @property @abc.abstractmethod def backend(self) -> str: raise NotImplementedError() def __repr__(self) -> str: return "" % self.backend @property @abc.abstractmethod def queue(self): raise NotImplementedError() @property @abc.abstractmethod def subprocess(self): raise NotImplementedError() @property @abc.abstractmethod def socket(self): raise NotImplementedError() @abc.abstractmethod def start(self, func, args=()) -> None: raise NotImplementedError() @abc.abstractmethod def get_ident(self) -> int: raise NotImplementedError() @abc.abstractmethod def sleep(self, delay: float) -> None: raise NotImplementedError() @abc.abstractmethod def fdopen(self, fd, mode, bufsize=1, closefd=True): raise NotImplementedError() @abc.abstractmethod def Lock(self): raise NotImplementedError() @abc.abstractmethod def RLock(self): raise NotImplementedError() @abc.abstractmethod def Event(self) -> Event: raise NotImplementedError() class ThreadExecModel(ExecModel): backend = "thread" @property def queue(self): import queue return queue @property def subprocess(self): import subprocess return subprocess @property def socket(self): import socket return socket def get_ident(self) -> int: import _thread return _thread.get_ident() def sleep(self, delay: float) -> None: import time time.sleep(delay) def start(self, func, args=()) -> None: import _thread _thread.start_new_thread(func, args) def fdopen(self, fd, mode, bufsize=1, closefd=True): import os return os.fdopen(fd, mode, bufsize, encoding="utf-8", closefd=closefd) def Lock(self): import threading return threading.RLock() def RLock(self): import threading return threading.RLock() def Event(self): import threading return threading.Event() class MainThreadOnlyExecModel(ThreadExecModel): backend = "main_thread_only" class EventletExecModel(ExecModel): backend = "eventlet" @property def queue(self): import eventlet return eventlet.queue @property def subprocess(self): import eventlet.green.subprocess return eventlet.green.subprocess @property def socket(self): import eventlet.green.socket return eventlet.green.socket def get_ident(self) -> int: import eventlet.green.thread return eventlet.green.thread.get_ident() # type: ignore[no-any-return] def sleep(self, delay: float) -> None: import eventlet eventlet.sleep(delay) def start(self, func, args=()) -> None: import eventlet eventlet.spawn_n(func, *args) def fdopen(self, fd, mode, bufsize=1, closefd=True): import eventlet.green.os return eventlet.green.os.fdopen(fd, mode, bufsize, closefd=closefd) def Lock(self): import eventlet.green.threading return eventlet.green.threading.RLock() def RLock(self): import eventlet.green.threading return eventlet.green.threading.RLock() def Event(self): import eventlet.green.threading return eventlet.green.threading.Event() class GeventExecModel(ExecModel): backend = "gevent" @property def queue(self): import gevent.queue return gevent.queue @property def subprocess(self): import gevent.subprocess return gevent.subprocess @property def socket(self): import gevent return gevent.socket def get_ident(self) -> int: import gevent.thread return gevent.thread.get_ident() # type: ignore[no-any-return] def sleep(self, delay: float) -> None: import gevent gevent.sleep(delay) def start(self, func, args=()) -> None: import gevent gevent.spawn(func, *args) def fdopen(self, fd, mode, bufsize=1, closefd=True): # XXX import gevent.fileobject return gevent.fileobject.FileObjectThread(fd, mode, bufsize, closefd=closefd) def Lock(self): import gevent.lock return gevent.lock.RLock() def RLock(self): import gevent.lock return gevent.lock.RLock() def Event(self): import gevent.event return gevent.event.Event() def get_execmodel(backend: str | ExecModel) -> ExecModel: if isinstance(backend, ExecModel): return backend if backend == "thread": return ThreadExecModel() elif backend == "main_thread_only": return MainThreadOnlyExecModel() elif backend == "eventlet": return EventletExecModel() elif backend == "gevent": return GeventExecModel() else: raise ValueError(f"unknown execmodel {backend!r}") class Reply: """Provide access to the result of a function execution that got dispatched through WorkerPool.spawn().""" def __init__(self, task, threadmodel: ExecModel) -> None: self.task = task self._result_ready = threadmodel.Event() self.running = True def get(self, timeout: float | None = None): """get the result object from an asynchronous function execution. if the function execution raised an exception, then calling get() will reraise that exception including its traceback. """ self.waitfinish(timeout) try: return self._result except AttributeError: raise self._exc from None def waitfinish(self, timeout: float | None = None) -> None: if not self._result_ready.wait(timeout): raise OSError(f"timeout waiting for {self.task!r}") def run(self) -> None: func, args, kwargs = self.task try: try: self._result = func(*args, **kwargs) except BaseException as exc: self._exc = exc finally: self._result_ready.set() self.running = False class WorkerPool: """A WorkerPool allows to spawn function executions to threads, returning a reply object on which you can ask for the result (and get exceptions reraised). This implementation allows the main thread to integrate itself into performing function execution through calling integrate_as_primary_thread() which will return when the pool received a trigger_shutdown(). By default allows unlimited number of spawns. """ _primary_thread_task: Reply | None def __init__(self, execmodel: ExecModel, hasprimary: bool = False) -> None: self.execmodel = execmodel self._running_lock = self.execmodel.Lock() self._running: MutableSet[Reply] = set() self._shuttingdown = False self._waitall_events: list[Event] = [] if hasprimary: if self.execmodel.backend not in ("thread", "main_thread_only"): raise ValueError("hasprimary=True requires thread model") self._primary_thread_task_ready: Event | None = self.execmodel.Event() else: self._primary_thread_task_ready = None def integrate_as_primary_thread(self) -> None: """Integrate the thread with which we are called as a primary thread for executing functions triggered with spawn().""" assert self.execmodel.backend in ("thread", "main_thread_only"), self.execmodel primary_thread_task_ready = self._primary_thread_task_ready assert primary_thread_task_ready is not None # interacts with code at REF1 while 1: primary_thread_task_ready.wait() reply = self._primary_thread_task if reply is None: # trigger_shutdown() woke us up break self._perform_spawn(reply) # we are concurrent with trigger_shutdown and spawn with self._running_lock: if self._shuttingdown: break # Only clear if _try_send_to_primary_thread has not # yet set the next self._primary_thread_task reply # after waiting for this one to complete. if reply is self._primary_thread_task: primary_thread_task_ready.clear() def trigger_shutdown(self) -> None: with self._running_lock: self._shuttingdown = True if self._primary_thread_task_ready is not None: self._primary_thread_task = None self._primary_thread_task_ready.set() def active_count(self) -> int: return len(self._running) def _perform_spawn(self, reply: Reply) -> None: reply.run() with self._running_lock: self._running.remove(reply) if not self._running: while self._waitall_events: waitall_event = self._waitall_events.pop() waitall_event.set() def _try_send_to_primary_thread(self, reply: Reply) -> bool: # REF1 in 'thread' model we give priority to running in main thread # note that we should be called with _running_lock hold primary_thread_task_ready = self._primary_thread_task_ready if primary_thread_task_ready is not None: if not primary_thread_task_ready.is_set(): self._primary_thread_task = reply # wake up primary thread primary_thread_task_ready.set() return True elif ( self.execmodel.backend == "main_thread_only" and self._primary_thread_task is not None ): self._primary_thread_task.waitfinish() self._primary_thread_task = reply # wake up primary thread (it's okay if this is already set # because we waited for the previous task to finish above # and integrate_as_primary_thread will not clear it when # it enters self._running_lock if it detects that a new # task is available) primary_thread_task_ready.set() return True return False def spawn(self, func, *args, **kwargs) -> Reply: """Asynchronously dispatch func(*args, **kwargs) and return a Reply.""" reply = Reply((func, args, kwargs), self.execmodel) with self._running_lock: if self._shuttingdown: raise ValueError("pool is shutting down") self._running.add(reply) if not self._try_send_to_primary_thread(reply): self.execmodel.start(self._perform_spawn, (reply,)) return reply def terminate(self, timeout: float | None = None) -> bool: """Trigger shutdown and wait for completion of all executions.""" self.trigger_shutdown() return self.waitall(timeout=timeout) def waitall(self, timeout: float | None = None) -> bool: """Wait until all active spawns have finished executing.""" with self._running_lock: if not self._running: return True # if a Reply still runs, we let run_and_release # signal us -- note that we are still holding the # _running_lock to avoid race conditions my_waitall_event = self.execmodel.Event() self._waitall_events.append(my_waitall_event) return my_waitall_event.wait(timeout=timeout) sysex = (KeyboardInterrupt, SystemExit) DEBUG = os.environ.get("EXECNET_DEBUG") pid = os.getpid() if DEBUG == "2": def trace(*msg: object) -> None: try: line = " ".join(map(str, msg)) sys.stderr.write(f"[{pid}] {line}\n") sys.stderr.flush() except Exception: pass # nothing we can do, likely interpreter-shutdown elif DEBUG: import os import tempfile fn = os.path.join(tempfile.gettempdir(), "execnet-debug-%d" % pid) # sys.stderr.write("execnet-debug at %r" % (fn,)) debugfile = open(fn, "w") def trace(*msg: object) -> None: try: line = " ".join(map(str, msg)) debugfile.write(line + "\n") debugfile.flush() except Exception as exc: try: sys.stderr.write(f"[{pid}] exception during tracing: {exc!r}\n") except Exception: pass # nothing we can do, likely interpreter-shutdown else: notrace = trace = lambda *msg: None class Popen2IO: error = (IOError, OSError, EOFError) def __init__(self, outfile, infile, execmodel: ExecModel) -> None: # we need raw byte streams self.outfile, self.infile = outfile, infile if sys.platform == "win32": import msvcrt try: msvcrt.setmode(infile.fileno(), os.O_BINARY) msvcrt.setmode(outfile.fileno(), os.O_BINARY) except (AttributeError, OSError): pass self._read = getattr(infile, "buffer", infile).read self._write = getattr(outfile, "buffer", outfile).write self.execmodel = execmodel def read(self, numbytes: int) -> bytes: """Read exactly 'numbytes' bytes from the pipe.""" # a file in non-blocking mode may return less bytes, so we loop buf = b"" while numbytes > len(buf): data = self._read(numbytes - len(buf)) if not data: raise EOFError("expected %d bytes, got %d" % (numbytes, len(buf))) buf += data return buf def write(self, data: bytes) -> None: """Write out all data bytes.""" assert isinstance(data, bytes) self._write(data) self.outfile.flush() def close_read(self) -> None: self.infile.close() def close_write(self) -> None: self.outfile.close() class Message: """Encapsulates Messages and their wire protocol.""" # message code -> name, handler _types: dict[int, tuple[str, Callable[[Message, BaseGateway], None]]] = {} def __init__(self, msgcode: int, channelid: int = 0, data: bytes = b"") -> None: self.msgcode = msgcode self.channelid = channelid self.data = data @staticmethod def from_io(io: ReadIO) -> Message: try: header = io.read(9) # type 1, channel 4, payload 4 if not header: raise EOFError("empty read") except EOFError as e: raise EOFError("couldn't load message header, " + e.args[0]) from None msgtype, channel, payload = struct.unpack("!bii", header) return Message(msgtype, channel, io.read(payload)) def to_io(self, io: WriteIO) -> None: header = struct.pack("!bii", self.msgcode, self.channelid, len(self.data)) io.write(header + self.data) def received(self, gateway: BaseGateway) -> None: handler = self._types[self.msgcode][1] handler(self, gateway) def __repr__(self) -> str: name = self._types[self.msgcode][0] return f"" def _status(message: Message, gateway: BaseGateway) -> None: # we use the channelid to send back information # but don't instantiate a channel object d = { "numchannels": len(gateway._channelfactory._channels), # TODO(typing): Attribute `_execpool` is only on WorkerGateway. "numexecuting": gateway._execpool.active_count(), # type: ignore[attr-defined] "execmodel": gateway.execmodel.backend, } gateway._send(Message.CHANNEL_DATA, message.channelid, dumps_internal(d)) gateway._send(Message.CHANNEL_CLOSE, message.channelid) STATUS = 0 _types[STATUS] = ("STATUS", _status) def _reconfigure(message: Message, gateway: BaseGateway) -> None: data = loads_internal(message.data, gateway) assert isinstance(data, tuple) strconfig: tuple[bool, bool] = data if message.channelid == 0: gateway._strconfig = strconfig else: gateway._channelfactory.new(message.channelid)._strconfig = strconfig RECONFIGURE = 1 _types[RECONFIGURE] = ("RECONFIGURE", _reconfigure) def _gateway_terminate(message: Message, gateway: BaseGateway) -> None: raise GatewayReceivedTerminate(gateway) GATEWAY_TERMINATE = 2 _types[GATEWAY_TERMINATE] = ("GATEWAY_TERMINATE", _gateway_terminate) def _channel_exec(message: Message, gateway: BaseGateway) -> None: channel = gateway._channelfactory.new(message.channelid) gateway._local_schedulexec(channel=channel, sourcetask=message.data) CHANNEL_EXEC = 3 _types[CHANNEL_EXEC] = ("CHANNEL_EXEC", _channel_exec) def _channel_data(message: Message, gateway: BaseGateway) -> None: gateway._channelfactory._local_receive(message.channelid, message.data) CHANNEL_DATA = 4 _types[CHANNEL_DATA] = ("CHANNEL_DATA", _channel_data) def _channel_close(message: Message, gateway: BaseGateway) -> None: gateway._channelfactory._local_close(message.channelid) CHANNEL_CLOSE = 5 _types[CHANNEL_CLOSE] = ("CHANNEL_CLOSE", _channel_close) def _channel_close_error(message: Message, gateway: BaseGateway) -> None: error_message = loads_internal(message.data) assert isinstance(error_message, str) remote_error = RemoteError(error_message) gateway._channelfactory._local_close(message.channelid, remote_error) CHANNEL_CLOSE_ERROR = 6 _types[CHANNEL_CLOSE_ERROR] = ("CHANNEL_CLOSE_ERROR", _channel_close_error) def _channel_last_message(message: Message, gateway: BaseGateway) -> None: gateway._channelfactory._local_close(message.channelid, sendonly=True) CHANNEL_LAST_MESSAGE = 7 _types[CHANNEL_LAST_MESSAGE] = ("CHANNEL_LAST_MESSAGE", _channel_last_message) class GatewayReceivedTerminate(Exception): """Receiverthread got termination message.""" def geterrortext( exc: BaseException, format_exception=traceback.format_exception, sysex: tuple[type[BaseException], ...] = sysex, ) -> str: try: # In py310, can change this to: # l = format_exception(exc) l = format_exception(type(exc), exc, exc.__traceback__) errortext = "".join(l) except sysex: raise except BaseException: errortext = f"{type(exc).__name__}: {exc}" return errortext class RemoteError(Exception): """Exception containing a stringified error from the other side.""" def __init__(self, formatted: str) -> None: super().__init__() self.formatted = formatted def __str__(self) -> str: return self.formatted def __repr__(self) -> str: return f"{self.__class__.__name__}: {self.formatted}" def warn(self) -> None: if self.formatted != INTERRUPT_TEXT: # XXX do this better sys.stderr.write(f"[{os.getpid()}] Warning: unhandled {self!r}\n") class TimeoutError(IOError): """Exception indicating that a timeout was reached.""" NO_ENDMARKER_WANTED = object() class Channel: """Communication channel between two Python Interpreter execution points.""" RemoteError = RemoteError TimeoutError = TimeoutError _INTERNALWAKEUP = 1000 _executing = False def __init__(self, gateway: BaseGateway, id: int) -> None: """:private:""" assert isinstance(id, int) assert not isinstance(gateway, type) self.gateway = gateway # XXX: defaults copied from Unserializer self._strconfig = getattr(gateway, "_strconfig", (True, False)) self.id = id self._items = self.gateway.execmodel.queue.Queue() self._closed = False self._receiveclosed = self.gateway.execmodel.Event() self._remoteerrors: list[RemoteError] = [] def _trace(self, *msg: object) -> None: self.gateway._trace(self.id, *msg) def setcallback( self, callback: Callable[[Any], Any], endmarker: object = NO_ENDMARKER_WANTED, ) -> None: """Set a callback function for receiving items. All already-queued items will immediately trigger the callback. Afterwards the callback will execute in the receiver thread for each received data item and calls to ``receive()`` will raise an error. If an endmarker is specified the callback will eventually be called with the endmarker when the channel closes. """ _callbacks = self.gateway._channelfactory._callbacks with self.gateway._receivelock: if self._items is None: raise OSError(f"{self!r} has callback already registered") items = self._items self._items = None while 1: try: olditem = items.get(block=False) except self.gateway.execmodel.queue.Empty: if not (self._closed or self._receiveclosed.is_set()): _callbacks[self.id] = (callback, endmarker, self._strconfig) break else: if olditem is ENDMARKER: items.put(olditem) # for other receivers if endmarker is not NO_ENDMARKER_WANTED: callback(endmarker) break else: callback(olditem) def __repr__(self) -> str: flag = self.isclosed() and "closed" or "open" return "" % (self.id, flag) def __del__(self) -> None: if self.gateway is None: # can be None in tests return # type: ignore[unreachable] self._trace("channel.__del__") # no multithreading issues here, because we have the last ref to 'self' if self._closed: # state transition "closed" --> "deleted" for error in self._remoteerrors: error.warn() elif self._receiveclosed.is_set(): # state transition "sendonly" --> "deleted" # the remote channel is already in "deleted" state, nothing to do pass else: # state transition "opened" --> "deleted" # check if we are in the middle of interpreter shutdown # in which case the process will go away and we probably # don't need to try to send a closing or last message # (and often it won't work anymore to send things out) if Message is not None: if self._items is None: # has_callback msgcode = Message.CHANNEL_LAST_MESSAGE else: msgcode = Message.CHANNEL_CLOSE try: self.gateway._send(msgcode, self.id) except (OSError, ValueError): # ignore problems with sending pass def _getremoteerror(self): try: return self._remoteerrors.pop(0) except IndexError: try: return self.gateway._error except AttributeError: pass return None # # public API for channel objects # def isclosed(self) -> bool: """Return True if the channel is closed. A closed channel may still hold items. """ return self._closed @overload def makefile(self, mode: Literal["r"], proxyclose: bool = ...) -> ChannelFileRead: pass @overload def makefile( self, mode: Literal["w"] = ..., proxyclose: bool = ..., ) -> ChannelFileWrite: pass def makefile( self, mode: Literal["r", "w"] = "w", proxyclose: bool = False, ) -> ChannelFileWrite | ChannelFileRead: """Return a file-like object. mode can be 'w' or 'r' for writeable/readable files. If proxyclose is true, file.close() will also close the channel. """ if mode == "w": return ChannelFileWrite(channel=self, proxyclose=proxyclose) elif mode == "r": return ChannelFileRead(channel=self, proxyclose=proxyclose) raise ValueError(f"mode {mode!r} not available") def close(self, error=None) -> None: """Close down this channel with an optional error message. Note that closing of a channel tied to remote_exec happens automatically at the end of execution and cannot be done explicitly. """ if self._executing: raise OSError("cannot explicitly close channel within remote_exec") if self._closed: self.gateway._trace(self, "ignoring redundant call to close()") if not self._closed: # state transition "opened/sendonly" --> "closed" # threads warning: the channel might be closed under our feet, # but it's never damaging to send too many CHANNEL_CLOSE messages # however, if the other side triggered a close already, we # do not send back a closed message. if not self._receiveclosed.is_set(): put = self.gateway._send if error is not None: put(Message.CHANNEL_CLOSE_ERROR, self.id, dumps_internal(error)) else: put(Message.CHANNEL_CLOSE, self.id) self._trace("sent channel close message") if isinstance(error, RemoteError): self._remoteerrors.append(error) self._closed = True # --> "closed" self._receiveclosed.set() queue = self._items if queue is not None: queue.put(ENDMARKER) self.gateway._channelfactory._no_longer_opened(self.id) def waitclose(self, timeout: float | None = None) -> None: """Wait until this channel is closed (or the remote side otherwise signalled that no more data was being sent). The channel may still hold receiveable items, but not receive any more after waitclose() has returned. Exceptions from executing code on the other side are reraised as local channel.RemoteErrors. EOFError is raised if the reading-connection was prematurely closed, which often indicates a dying process. self.TimeoutError is raised after the specified number of seconds (default is None, i.e. wait indefinitely). """ # wait for non-"opened" state self._receiveclosed.wait(timeout=timeout) if not self._receiveclosed.is_set(): raise self.TimeoutError("Timeout after %r seconds" % timeout) error = self._getremoteerror() if error: raise error def send(self, item: object) -> None: """Sends the given item to the other side of the channel, possibly blocking if the sender queue is full. The item must be a simple Python type and will be copied to the other side by value. OSError is raised if the write pipe was prematurely closed. """ if self.isclosed(): raise OSError(f"cannot send to {self!r}") self.gateway._send(Message.CHANNEL_DATA, self.id, dumps_internal(item)) def receive(self, timeout: float | None = None) -> Any: """Receive a data item that was sent from the other side. timeout: None [default] blocked waiting. A positive number indicates the number of seconds after which a channel.TimeoutError exception will be raised if no item was received. Note that exceptions from the remotely executing code will be reraised as channel.RemoteError exceptions containing a textual representation of the remote traceback. """ itemqueue = self._items if itemqueue is None: raise OSError("cannot receive(), channel has receiver callback") try: x = itemqueue.get(timeout=timeout) except self.gateway.execmodel.queue.Empty: raise self.TimeoutError("no item after %r seconds" % timeout) from None if x is ENDMARKER: itemqueue.put(x) # for other receivers raise self._getremoteerror() or EOFError() else: return x def __iter__(self) -> Iterator[Any]: return self def next(self) -> Any: try: return self.receive() except EOFError: raise StopIteration from None __next__ = next def reconfigure( self, py2str_as_py3str: bool = True, py3str_as_py2str: bool = False ) -> None: """Set the string coercion for this channel. The default is to try to convert py2 str as py3 str, but not to try and convert py3 str to py2 str """ self._strconfig = (py2str_as_py3str, py3str_as_py2str) data = dumps_internal(self._strconfig) self.gateway._send(Message.RECONFIGURE, self.id, data=data) ENDMARKER = object() INTERRUPT_TEXT = "keyboard-interrupted" MAIN_THREAD_ONLY_DEADLOCK_TEXT = ( "concurrent remote_exec would cause deadlock for main_thread_only execmodel" ) class ChannelFactory: def __init__(self, gateway: BaseGateway, startcount: int = 1) -> None: self._channels: weakref.WeakValueDictionary[int, Channel] = ( weakref.WeakValueDictionary() ) # Channel ID => (callback, end marker, strconfig) self._callbacks: dict[ int, tuple[Callable[[Any], Any], object, tuple[bool, bool]] ] = {} self._writelock = gateway.execmodel.Lock() self.gateway = gateway self.count = startcount self.finished = False self._list = list # needed during interp-shutdown def new(self, id: int | None = None) -> Channel: """Create a new Channel with 'id' (or create new id if None).""" with self._writelock: if self.finished: raise OSError(f"connection already closed: {self.gateway}") if id is None: id = self.count self.count += 2 try: channel = self._channels[id] except KeyError: channel = self._channels[id] = Channel(self.gateway, id) return channel def channels(self) -> list[Channel]: return self._list(self._channels.values()) # # internal methods, called from the receiver thread # def _no_longer_opened(self, id: int) -> None: try: del self._channels[id] except KeyError: pass try: callback, endmarker, strconfig = self._callbacks.pop(id) except KeyError: pass else: if endmarker is not NO_ENDMARKER_WANTED: callback(endmarker) def _local_close(self, id: int, remoteerror=None, sendonly: bool = False) -> None: channel = self._channels.get(id) if channel is None: # channel already in "deleted" state if remoteerror: remoteerror.warn() self._no_longer_opened(id) else: # state transition to "closed" state if remoteerror: channel._remoteerrors.append(remoteerror) queue = channel._items if queue is not None: queue.put(ENDMARKER) self._no_longer_opened(id) if not sendonly: # otherwise #--> "sendonly" channel._closed = True # --> "closed" channel._receiveclosed.set() def _local_receive(self, id: int, data) -> None: # executes in receiver thread channel = self._channels.get(id) try: callback, endmarker, strconfig = self._callbacks[id] except KeyError: queue = channel._items if channel is not None else None if queue is None: pass # drop data else: item = loads_internal(data, channel) queue.put(item) else: try: data = loads_internal(data, channel, strconfig) callback(data) # even if channel may be already closed except Exception as exc: self.gateway._trace("exception during callback: %s" % exc) errortext = self.gateway._geterrortext(exc) self.gateway._send( Message.CHANNEL_CLOSE_ERROR, id, dumps_internal(errortext) ) self._local_close(id, errortext) def _finished_receiving(self) -> None: with self._writelock: self.finished = True for id in self._list(self._channels): self._local_close(id, sendonly=True) for id in self._list(self._callbacks): self._no_longer_opened(id) class ChannelFile: def __init__(self, channel: Channel, proxyclose: bool = True) -> None: self.channel = channel self._proxyclose = proxyclose def isatty(self) -> bool: return False def close(self) -> None: if self._proxyclose: self.channel.close() def __repr__(self) -> str: state = self.channel.isclosed() and "closed" or "open" return "" % (self.channel.id, state) class ChannelFileWrite(ChannelFile): def write(self, out: bytes) -> None: self.channel.send(out) def flush(self) -> None: pass class ChannelFileRead(ChannelFile): def __init__(self, channel: Channel, proxyclose: bool = True) -> None: super().__init__(channel, proxyclose) self._buffer: str | None = None def read(self, n: int) -> str: try: if self._buffer is None: self._buffer = cast(str, self.channel.receive()) while len(self._buffer) < n: self._buffer += cast(str, self.channel.receive()) except EOFError: self.close() if self._buffer is None: ret = "" else: ret = self._buffer[:n] self._buffer = self._buffer[n:] return ret def readline(self) -> str: if self._buffer is not None: i = self._buffer.find("\n") if i != -1: return self.read(i + 1) line = self.read(len(self._buffer) + 1) else: line = self.read(1) while line and line[-1] != "\n": c = self.read(1) if not c: break line += c return line class BaseGateway: _sysex = sysex id = "" def __init__(self, io: IO, id, _startcount: int = 2) -> None: self.execmodel = io.execmodel self._io = io self.id = id self._strconfig = (Unserializer.py2str_as_py3str, Unserializer.py3str_as_py2str) self._channelfactory = ChannelFactory(self, _startcount) self._receivelock = self.execmodel.RLock() # globals may be NONE at process-termination self.__trace = trace self._geterrortext = geterrortext self._receivepool = WorkerPool(self.execmodel) def _trace(self, *msg: object) -> None: self.__trace(self.id, *msg) def _initreceive(self) -> None: self._receivepool.spawn(self._thread_receiver) def _thread_receiver(self) -> None: def log(*msg: object) -> None: self._trace("[receiver-thread]", *msg) log("RECEIVERTHREAD: starting to run") io = self._io try: while 1: msg = Message.from_io(io) log("received", msg) with self._receivelock: msg.received(self) del msg except (KeyboardInterrupt, GatewayReceivedTerminate): pass except EOFError as exc: log("EOF without prior gateway termination message") self._error = exc except Exception as exc: log(self._geterrortext(exc)) log("finishing receiving thread") # wake up and terminate any execution waiting to receive self._channelfactory._finished_receiving() log("terminating execution") self._terminate_execution() log("closing read") self._io.close_read() log("closing write") self._io.close_write() log("terminating our receive pseudo pool") self._receivepool.trigger_shutdown() def _terminate_execution(self) -> None: pass def _send(self, msgcode: int, channelid: int = 0, data: bytes = b"") -> None: message = Message(msgcode, channelid, data) try: message.to_io(self._io) self._trace("sent", message) except (OSError, ValueError) as e: self._trace("failed to send", message, e) # ValueError might be because the IO is already closed raise OSError("cannot send (already closed?)") from e def _local_schedulexec(self, channel: Channel, sourcetask: bytes) -> None: channel.close("execution disallowed") # _____________________________________________________________________ # # High Level Interface # _____________________________________________________________________ # def newchannel(self) -> Channel: """Return a new independent channel.""" return self._channelfactory.new() def join(self, timeout: float | None = None) -> None: """Wait for receiverthread to terminate.""" self._trace("waiting for receiver thread to finish") self._receivepool.waitall(timeout) class WorkerGateway(BaseGateway): def _local_schedulexec(self, channel: Channel, sourcetask: bytes) -> None: if self._execpool.execmodel.backend == "main_thread_only": assert self._executetask_complete is not None # It's necessary to wait for a short time in order to ensure # that we do not report a false-positive deadlock error, since # channel close does not elicit a response that would provide # a guarantee to remote_exec callers that the previous task # has released the main thread. If the timeout expires then it # should be practically impossible to report a false-positive. if not self._executetask_complete.wait(timeout=1): channel.close(MAIN_THREAD_ONLY_DEADLOCK_TEXT) return # It's only safe to clear here because the above wait proves # that there is not a previous task about to set it again. self._executetask_complete.clear() sourcetask_ = loads_internal(sourcetask) self._execpool.spawn(self.executetask, (channel, sourcetask_)) def _terminate_execution(self) -> None: # called from receiverthread self._trace("shutting down execution pool") self._execpool.trigger_shutdown() if not self._execpool.waitall(5.0): self._trace("execution ongoing after 5 secs," " trying interrupt_main") # We try hard to terminate execution based on the assumption # that there is only one gateway object running per-process. if sys.platform != "win32": self._trace("sending ourselves a SIGINT") os.kill(os.getpid(), 2) # send ourselves a SIGINT elif interrupt_main is not None: self._trace("calling interrupt_main()") interrupt_main() if not self._execpool.waitall(10.0): self._trace( "execution did not finish in another 10 secs, " "calling os._exit()" ) os._exit(1) def serve(self) -> None: def trace(msg: str) -> None: self._trace("[serve] " + msg) hasprimary = self.execmodel.backend in ("thread", "main_thread_only") self._execpool = WorkerPool(self.execmodel, hasprimary=hasprimary) self._executetask_complete = None if self.execmodel.backend == "main_thread_only": self._executetask_complete = self.execmodel.Event() # Initialize state to indicate that there is no previous task # executing so that we don't need a separate flag to track this. self._executetask_complete.set() trace("spawning receiver thread") self._initreceive() try: if hasprimary: # this will return when we are in shutdown trace("integrating as primary thread") self._execpool.integrate_as_primary_thread() trace("joining receiver thread") self.join() except KeyboardInterrupt: # in the worker we can't really do anything sensible trace("swallowing keyboardinterrupt, serve finished") def executetask( self, item: tuple[Channel, tuple[str, str | None, str | None, dict[str, object]]], ) -> None: try: channel, (source, file_name, call_name, kwargs) = item loc: dict[str, Any] = {"channel": channel, "__name__": "__channelexec__"} self._trace(f"execution starts[{channel.id}]: {repr(source)[:50]}") channel._executing = True try: co = compile(source + "\n", file_name or "", "exec") exec(co, loc) if call_name: self._trace("calling %s(**%60r)" % (call_name, kwargs)) function = loc[call_name] function(channel, **kwargs) finally: channel._executing = False self._trace("execution finished") except KeyboardInterrupt: channel.close(INTERRUPT_TEXT) raise except BaseException as exc: if not isinstance(exc, EOFError): if not channel.gateway._channelfactory.finished: self._trace(f"got exception: {exc!r}") errortext = self._geterrortext(exc) channel.close(errortext) return self._trace("ignoring EOFError because receiving finished") channel.close() if self._executetask_complete is not None: # Indicate that this task has finished executing, meaning # that there is no possibility of it triggering a deadlock # for the next spawn call. self._executetask_complete.set() # # Cross-Python pickling code, tested from test_serializer.py # class DataFormatError(Exception): pass class DumpError(DataFormatError): """Error while serializing an object.""" class LoadError(DataFormatError): """Error while unserializing an object.""" def bchr(n: int) -> bytes: return bytes([n]) DUMPFORMAT_VERSION = bchr(2) FOUR_BYTE_INT_MAX = 2147483647 FLOAT_FORMAT = "!d" FLOAT_FORMAT_SIZE = struct.calcsize(FLOAT_FORMAT) COMPLEX_FORMAT = "!dd" COMPLEX_FORMAT_SIZE = struct.calcsize(COMPLEX_FORMAT) class _Stop(Exception): pass class opcode: """Container for name -> num mappings.""" BUILDTUPLE = b"@" BYTES = b"A" CHANNEL = b"B" FALSE = b"C" FLOAT = b"D" FROZENSET = b"E" INT = b"F" LONG = b"G" LONGINT = b"H" LONGLONG = b"I" NEWDICT = b"J" NEWLIST = b"K" NONE = b"L" PY2STRING = b"M" PY3STRING = b"N" SET = b"O" SETITEM = b"P" STOP = b"Q" TRUE = b"R" UNICODE = b"S" COMPLEX = b"T" class Unserializer: num2func: dict[bytes, Callable[[Unserializer], None]] = {} py2str_as_py3str = True # True py3str_as_py2str = False # false means py2 will get unicode def __init__( self, stream: ReadIO, channel_or_gateway: Channel | BaseGateway | None = None, strconfig: tuple[bool, bool] | None = None, ) -> None: if isinstance(channel_or_gateway, Channel): gw: BaseGateway | None = channel_or_gateway.gateway else: gw = channel_or_gateway if channel_or_gateway is not None: strconfig = channel_or_gateway._strconfig if strconfig: self.py2str_as_py3str, self.py3str_as_py2str = strconfig self.stream = stream if gw is None: self.channelfactory = None else: self.channelfactory = gw._channelfactory def load(self, versioned: bool = False) -> Any: if versioned: ver = self.stream.read(1) if ver != DUMPFORMAT_VERSION: raise LoadError("wrong dumpformat version %r" % ver) self.stack: list[object] = [] try: while True: opcode = self.stream.read(1) if not opcode: raise EOFError try: loader = self.num2func[opcode] except KeyError: raise LoadError( f"unknown opcode {opcode!r} - wire protocol corruption?" ) from None loader(self) except _Stop: if len(self.stack) != 1: raise LoadError("internal unserialization error") from None return self.stack.pop(0) else: raise LoadError("didn't get STOP") def load_none(self) -> None: self.stack.append(None) num2func[opcode.NONE] = load_none def load_true(self) -> None: self.stack.append(True) num2func[opcode.TRUE] = load_true def load_false(self) -> None: self.stack.append(False) num2func[opcode.FALSE] = load_false def load_int(self) -> None: i = self._read_int4() self.stack.append(i) num2func[opcode.INT] = load_int def load_longint(self) -> None: s = self._read_byte_string() self.stack.append(int(s)) num2func[opcode.LONGINT] = load_longint load_long = load_int num2func[opcode.LONG] = load_long load_longlong = load_longint num2func[opcode.LONGLONG] = load_longlong def load_float(self) -> None: binary = self.stream.read(FLOAT_FORMAT_SIZE) self.stack.append(struct.unpack(FLOAT_FORMAT, binary)[0]) num2func[opcode.FLOAT] = load_float def load_complex(self) -> None: binary = self.stream.read(COMPLEX_FORMAT_SIZE) self.stack.append(complex(*struct.unpack(COMPLEX_FORMAT, binary))) num2func[opcode.COMPLEX] = load_complex def _read_int4(self) -> int: value: int = struct.unpack("!i", self.stream.read(4))[0] return value def _read_byte_string(self) -> bytes: length = self._read_int4() as_bytes = self.stream.read(length) return as_bytes def load_py3string(self) -> None: as_bytes = self._read_byte_string() if self.py3str_as_py2str: # XXX Should we try to decode into latin-1? self.stack.append(as_bytes) else: self.stack.append(as_bytes.decode("utf-8")) num2func[opcode.PY3STRING] = load_py3string def load_py2string(self) -> None: as_bytes = self._read_byte_string() if self.py2str_as_py3str: s: bytes | str = as_bytes.decode("latin-1") else: s = as_bytes self.stack.append(s) num2func[opcode.PY2STRING] = load_py2string def load_bytes(self) -> None: s = self._read_byte_string() self.stack.append(s) num2func[opcode.BYTES] = load_bytes def load_unicode(self) -> None: self.stack.append(self._read_byte_string().decode("utf-8")) num2func[opcode.UNICODE] = load_unicode def load_newlist(self) -> None: length = self._read_int4() self.stack.append([None] * length) num2func[opcode.NEWLIST] = load_newlist def load_setitem(self) -> None: if len(self.stack) < 3: raise LoadError("not enough items for setitem") value = self.stack.pop() key = self.stack.pop() self.stack[-1][key] = value # type: ignore[index] num2func[opcode.SETITEM] = load_setitem def load_newdict(self) -> None: self.stack.append({}) num2func[opcode.NEWDICT] = load_newdict def _load_collection(self, type_: type) -> None: length = self._read_int4() if length: res = type_(self.stack[-length:]) del self.stack[-length:] self.stack.append(res) else: self.stack.append(type_()) def load_buildtuple(self) -> None: self._load_collection(tuple) num2func[opcode.BUILDTUPLE] = load_buildtuple def load_set(self) -> None: self._load_collection(set) num2func[opcode.SET] = load_set def load_frozenset(self) -> None: self._load_collection(frozenset) num2func[opcode.FROZENSET] = load_frozenset def load_stop(self) -> None: raise _Stop num2func[opcode.STOP] = load_stop def load_channel(self) -> None: id = self._read_int4() assert self.channelfactory is not None newchannel = self.channelfactory.new(id) self.stack.append(newchannel) num2func[opcode.CHANNEL] = load_channel def dumps(obj: object) -> bytes: """Serialize the given obj to a bytestring. The obj and all contained objects must be of a builtin Python type (so nested dicts, sets, etc. are all OK but not user-level instances). """ return _Serializer().save(obj, versioned=True) # type: ignore[return-value] def dump(byteio, obj: object) -> None: """write a serialized bytestring of the given obj to the given stream.""" _Serializer(write=byteio.write).save(obj, versioned=True) def loads( bytestring: bytes, py2str_as_py3str: bool = False, py3str_as_py2str: bool = False ) -> Any: """Deserialize the given bytestring to an object. py2str_as_py3str: If true then string (str) objects previously dumped on Python2 will be loaded as Python3 strings which really are text objects. py3str_as_py2str: If true then string (str) objects previously dumped on Python3 will be loaded as Python2 strings instead of unicode objects. If the bytestring was dumped with an incompatible protocol version or if the bytestring is corrupted, the ``execnet.DataFormatError`` will be raised. """ io = BytesIO(bytestring) return load( io, py2str_as_py3str=py2str_as_py3str, py3str_as_py2str=py3str_as_py2str ) def load( io: ReadIO, py2str_as_py3str: bool = False, py3str_as_py2str: bool = False ) -> Any: """Derserialize an object form the specified stream. Behaviour and parameters are otherwise the same as with ``loads`` """ strconfig = (py2str_as_py3str, py3str_as_py2str) return Unserializer(io, strconfig=strconfig).load(versioned=True) def loads_internal( bytestring: bytes, channelfactory=None, strconfig: tuple[bool, bool] | None = None, ) -> Any: io = BytesIO(bytestring) return Unserializer(io, channelfactory, strconfig).load() def dumps_internal(obj: object) -> bytes: return _Serializer().save(obj) # type: ignore[return-value] class _Serializer: _dispatch: dict[type, Callable[[_Serializer, object], None]] = {} def __init__(self, write: Callable[[bytes], None] | None = None) -> None: if write is None: self._streamlist: list[bytes] = [] write = self._streamlist.append self._write = write def save(self, obj: object, versioned: bool = False) -> bytes | None: # calling here is not re-entrant but multiple instances # may write to the same stream because of the common platform # atomic-write guarantee (concurrent writes each happen atomically) if versioned: self._write(DUMPFORMAT_VERSION) self._save(obj) self._write(opcode.STOP) try: streamlist = self._streamlist except AttributeError: return None return b"".join(streamlist) def _save(self, obj: object) -> None: tp = type(obj) try: dispatch = self._dispatch[tp] except KeyError: methodname = "save_" + tp.__name__ meth: Callable[[_Serializer, object], None] | None = getattr( self.__class__, methodname, None ) if meth is None: raise DumpError(f"can't serialize {tp}") from None dispatch = self._dispatch[tp] = meth dispatch(self, obj) def save_NoneType(self, non: None) -> None: self._write(opcode.NONE) def save_bool(self, boolean: bool) -> None: if boolean: self._write(opcode.TRUE) else: self._write(opcode.FALSE) def save_bytes(self, bytes_: bytes) -> None: self._write(opcode.BYTES) self._write_byte_sequence(bytes_) def save_str(self, s: str) -> None: self._write(opcode.PY3STRING) self._write_unicode_string(s) def _write_unicode_string(self, s: str) -> None: try: as_bytes = s.encode("utf-8") except UnicodeEncodeError as e: raise DumpError("strings must be utf-8 encodable") from e self._write_byte_sequence(as_bytes) def _write_byte_sequence(self, bytes_: bytes) -> None: self._write_int4(len(bytes_), "string is too long") self._write(bytes_) def _save_integral(self, i: int, short_op: bytes, long_op: bytes) -> None: if i <= FOUR_BYTE_INT_MAX: self._write(short_op) self._write_int4(i) else: self._write(long_op) self._write_byte_sequence(str(i).rstrip("L").encode("ascii")) def save_int(self, i: int) -> None: self._save_integral(i, opcode.INT, opcode.LONGINT) def save_long(self, l: int) -> None: self._save_integral(l, opcode.LONG, opcode.LONGLONG) def save_float(self, flt: float) -> None: self._write(opcode.FLOAT) self._write(struct.pack(FLOAT_FORMAT, flt)) def save_complex(self, cpx: complex) -> None: self._write(opcode.COMPLEX) self._write(struct.pack(COMPLEX_FORMAT, cpx.real, cpx.imag)) def _write_int4( self, i: int, error: str = "int must be less than %i" % (FOUR_BYTE_INT_MAX,) ) -> None: if i > FOUR_BYTE_INT_MAX: raise DumpError(error) self._write(struct.pack("!i", i)) def save_list(self, L: list[object]) -> None: self._write(opcode.NEWLIST) self._write_int4(len(L), "list is too long") for i, item in enumerate(L): self._write_setitem(i, item) def _write_setitem(self, key: object, value: object) -> None: self._save(key) self._save(value) self._write(opcode.SETITEM) def save_dict(self, d: dict[object, object]) -> None: self._write(opcode.NEWDICT) for key, value in d.items(): self._write_setitem(key, value) def save_tuple(self, tup: tuple[object, ...]) -> None: for item in tup: self._save(item) self._write(opcode.BUILDTUPLE) self._write_int4(len(tup), "tuple is too long") def _write_set(self, s: set[object] | frozenset[object], op: bytes) -> None: for item in s: self._save(item) self._write(op) self._write_int4(len(s), "set is too long") def save_set(self, s: set[object]) -> None: self._write_set(s, opcode.SET) def save_frozenset(self, s: frozenset[object]) -> None: self._write_set(s, opcode.FROZENSET) def save_Channel(self, channel: Channel) -> None: self._write(opcode.CHANNEL) self._write_int4(channel.id) def init_popen_io(execmodel: ExecModel) -> Popen2IO: if not hasattr(os, "dup"): # jython io = Popen2IO(sys.stdout, sys.stdin, execmodel) import tempfile sys.stdin = tempfile.TemporaryFile("r") sys.stdout = tempfile.TemporaryFile("w") else: try: devnull = os.devnull except AttributeError: if os.name == "nt": devnull = "NUL" else: devnull = "/dev/null" # stdin stdin = execmodel.fdopen(os.dup(0), "r", 1) fd = os.open(devnull, os.O_RDONLY) os.dup2(fd, 0) os.close(fd) # stdout stdout = execmodel.fdopen(os.dup(1), "w", 1) fd = os.open(devnull, os.O_WRONLY) os.dup2(fd, 1) # stderr for win32 if os.name == "nt": sys.stderr = execmodel.fdopen(os.dup(2), "w", 1) os.dup2(fd, 2) os.close(fd) io = Popen2IO(stdout, stdin, execmodel) # Use closefd=False since 0 and 1 are shared with # sys.__stdin__ and sys.__stdout__. sys.stdin = execmodel.fdopen(0, "r", 1, closefd=False) sys.stdout = execmodel.fdopen(1, "w", 1, closefd=False) return io def serve(io: IO, id) -> None: trace(f"creating workergateway on {io!r}") WorkerGateway(io=io, id=id, _startcount=2).serve() execnet-2.1.1/src/execnet/gateway_bootstrap.py000066400000000000000000000053221460460142100215000ustar00rootroot00000000000000"""Code to initialize the remote side of a gateway once the IO is created.""" from __future__ import annotations import inspect import os import execnet from . import gateway_base from .gateway_base import IO from .xspec import XSpec importdir = os.path.dirname(os.path.dirname(execnet.__file__)) class HostNotFound(Exception): pass def bootstrap_import(io: IO, spec: XSpec) -> None: # Only insert the importdir into the path if we must. This prevents # bugs where backports expect to be shadowed by the standard library on # newer versions of python but would instead shadow the standard library. sendexec( io, "import sys", "if %r not in sys.path:" % importdir, " sys.path.insert(0, %r)" % importdir, "from execnet.gateway_base import serve, init_popen_io, get_execmodel", "sys.stdout.write('1')", "sys.stdout.flush()", "execmodel = get_execmodel(%r)" % spec.execmodel, "serve(init_popen_io(execmodel), id='%s-worker')" % spec.id, ) s = io.read(1) assert s == b"1", repr(s) def bootstrap_exec(io: IO, spec: XSpec) -> None: try: sendexec( io, inspect.getsource(gateway_base), "execmodel = get_execmodel(%r)" % spec.execmodel, "io = init_popen_io(execmodel)", "io.write('1'.encode('ascii'))", "serve(io, id='%s-worker')" % spec.id, ) s = io.read(1) assert s == b"1" except EOFError: ret = io.wait() if ret == 255 and hasattr(io, "remoteaddress"): raise HostNotFound(io.remoteaddress) from None def bootstrap_socket(io: IO, id) -> None: # XXX: switch to spec from execnet.gateway_socket import SocketIO sendexec( io, inspect.getsource(gateway_base), "import socket", inspect.getsource(SocketIO), "try: execmodel", "except NameError:", " execmodel = get_execmodel('thread')", "io = SocketIO(clientsock, execmodel)", "io.write('1'.encode('ascii'))", "serve(io, id='%s-worker')" % id, ) s = io.read(1) assert s == b"1" def sendexec(io: IO, *sources: str) -> None: source = "\n".join(sources) io.write((repr(source) + "\n").encode("utf-8")) def bootstrap(io: IO, spec: XSpec) -> execnet.Gateway: if spec.popen: if spec.via or spec.python: bootstrap_exec(io, spec) else: bootstrap_import(io, spec) elif spec.ssh or spec.vagrant_ssh: bootstrap_exec(io, spec) elif spec.socket: bootstrap_socket(io, spec) else: raise ValueError("unknown gateway type, can't bootstrap") gw = execnet.Gateway(io, spec) return gw execnet-2.1.1/src/execnet/gateway_io.py000066400000000000000000000175641460460142100201050ustar00rootroot00000000000000"""execnet IO initialization code. Creates IO instances used for gateway IO. """ from __future__ import annotations import shlex import sys from typing import TYPE_CHECKING from typing import cast if TYPE_CHECKING: from execnet.gateway_base import Channel from execnet.gateway_base import ExecModel from execnet.xspec import XSpec try: from execnet.gateway_base import Message from execnet.gateway_base import Popen2IO except ImportError: from __main__ import Message # type: ignore[no-redef] from __main__ import Popen2IO # type: ignore[no-redef] from functools import partial class Popen2IOMaster(Popen2IO): # Set externally, for some specs only. remoteaddress: str def __init__(self, args, execmodel: ExecModel) -> None: PIPE = execmodel.subprocess.PIPE self.popen = p = execmodel.subprocess.Popen(args, stdout=PIPE, stdin=PIPE) super().__init__(p.stdin, p.stdout, execmodel=execmodel) def wait(self) -> int | None: try: return self.popen.wait() # type: ignore[no-any-return] except OSError: return None def kill(self) -> None: try: self.popen.kill() except OSError as e: sys.stderr.write("ERROR killing: %s\n" % e) sys.stderr.flush() popen_bootstrapline = "import sys;exec(eval(sys.stdin.readline()))" def shell_split_path(path: str) -> list[str]: """ Use shell lexer to split the given path into a list of components, taking care to handle Windows' '\' correctly. """ if sys.platform.startswith("win"): # replace \\ by / otherwise shlex will strip them out path = path.replace("\\", "/") return shlex.split(path) def popen_args(spec: XSpec) -> list[str]: args = shell_split_path(spec.python) if spec.python else [sys.executable] args.append("-u") if spec.dont_write_bytecode: args.append("-B") args.extend(["-c", popen_bootstrapline]) return args def ssh_args(spec: XSpec) -> list[str]: # NOTE: If changing this, you need to sync those changes to vagrant_args # as well, or, take some time to further refactor the commonalities of # ssh_args and vagrant_args. remotepython = spec.python or "python" args = ["ssh", "-C"] if spec.ssh_config is not None: args.extend(["-F", str(spec.ssh_config)]) assert spec.ssh is not None args.extend(spec.ssh.split()) remotecmd = f'{remotepython} -c "{popen_bootstrapline}"' args.append(remotecmd) return args def vagrant_ssh_args(spec: XSpec) -> list[str]: # This is the vagrant-wrapped version of SSH. Unfortunately the # command lines are incompatible to just channel through ssh_args # due to ordering/templating issues. # NOTE: This should be kept in sync with the ssh_args behaviour. # spec.vagrant is identical to spec.ssh in that they both carry # the remote host "address". assert spec.vagrant_ssh is not None remotepython = spec.python or "python" args = ["vagrant", "ssh", spec.vagrant_ssh, "--", "-C"] if spec.ssh_config is not None: args.extend(["-F", str(spec.ssh_config)]) remotecmd = f'{remotepython} -c "{popen_bootstrapline}"' args.extend([remotecmd]) return args def create_io(spec: XSpec, execmodel: ExecModel) -> Popen2IOMaster: if spec.popen: args = popen_args(spec) return Popen2IOMaster(args, execmodel) if spec.ssh: args = ssh_args(spec) io = Popen2IOMaster(args, execmodel) io.remoteaddress = spec.ssh return io if spec.vagrant_ssh: args = vagrant_ssh_args(spec) io = Popen2IOMaster(args, execmodel) io.remoteaddress = spec.vagrant_ssh return io assert False # # Proxy Gateway handling code # # master: proxy initiator # forwarder: forwards between master and sub # sub: sub process that is proxied to the initiator RIO_KILL = 1 RIO_WAIT = 2 RIO_REMOTEADDRESS = 3 RIO_CLOSE_WRITE = 4 class ProxyIO: """A Proxy IO object allows to instantiate a Gateway through another "via" gateway. A master:ProxyIO object provides an IO object effectively connected to the sub via the forwarder. To achieve this, master:ProxyIO interacts with forwarder:serve_proxy_io() which itself instantiates and interacts with the sub. """ def __init__(self, proxy_channel: Channel, execmodel: ExecModel) -> None: # after exchanging the control channel we use proxy_channel # for messaging IO self.controlchan = proxy_channel.gateway.newchannel() proxy_channel.send(self.controlchan) self.iochan = proxy_channel self.iochan_file = self.iochan.makefile("r") self.execmodel = execmodel def read(self, nbytes: int) -> bytes: # TODO(typing): The IO protocol requires bytes here but ChannelFileRead # returns str. return self.iochan_file.read(nbytes) # type: ignore[return-value] def write(self, data: bytes) -> None: self.iochan.send(data) def _controll(self, event: int) -> object: self.controlchan.send(event) return self.controlchan.receive() def close_write(self) -> None: self._controll(RIO_CLOSE_WRITE) def close_read(self) -> None: raise NotImplementedError() def kill(self) -> None: self._controll(RIO_KILL) def wait(self) -> int | None: response = self._controll(RIO_WAIT) assert response is None or isinstance(response, int) return response @property def remoteaddress(self) -> str: response = self._controll(RIO_REMOTEADDRESS) assert isinstance(response, str) return response def __repr__(self) -> str: return f"" class PseudoSpec: def __init__(self, vars) -> None: self.__dict__.update(vars) def __getattr__(self, name: str) -> None: return None def serve_proxy_io(proxy_channelX: Channel) -> None: execmodel = proxy_channelX.gateway.execmodel log = partial( proxy_channelX.gateway._trace, "serve_proxy_io:%s" % proxy_channelX.id ) spec = cast("XSpec", PseudoSpec(proxy_channelX.receive())) # create sub IO object which we will proxy back to our proxy initiator sub_io = create_io(spec, execmodel) control_chan = cast("Channel", proxy_channelX.receive()) log("got control chan", control_chan) # read data from master, forward it to the sub # XXX writing might block, thus blocking the receiver thread def forward_to_sub(data: bytes) -> None: log("forward data to sub, size %s" % len(data)) sub_io.write(data) proxy_channelX.setcallback(forward_to_sub) def control(data: int) -> None: if data == RIO_WAIT: control_chan.send(sub_io.wait()) elif data == RIO_KILL: sub_io.kill() control_chan.send(None) elif data == RIO_REMOTEADDRESS: control_chan.send(sub_io.remoteaddress) elif data == RIO_CLOSE_WRITE: sub_io.close_write() control_chan.send(None) control_chan.setcallback(control) # write data to the master coming from the sub forward_to_master_file = proxy_channelX.makefile("w") # read bootstrap byte from sub, send it on to master log("reading bootstrap byte from sub", spec.id) initial = sub_io.read(1) assert initial == b"1", initial log("forwarding bootstrap byte from sub", spec.id) forward_to_master_file.write(initial) # enter message forwarding loop while True: try: message = Message.from_io(sub_io) except EOFError: log("EOF from sub, terminating proxying loop", spec.id) break message.to_io(forward_to_master_file) # proxy_channelX will be closed from remote_exec's finalization code if __name__ == "__channelexec__": serve_proxy_io(channel) # type: ignore[name-defined] # noqa:F821 execnet-2.1.1/src/execnet/gateway_socket.py000066400000000000000000000056141460460142100207570ustar00rootroot00000000000000from __future__ import annotations import sys from typing import cast from execnet.gateway import Gateway from execnet.gateway_base import ExecModel from execnet.gateway_bootstrap import HostNotFound from execnet.multi import Group from execnet.xspec import XSpec class SocketIO: remoteaddress: str def __init__(self, sock, execmodel: ExecModel) -> None: self.sock = sock self.execmodel = execmodel socket = execmodel.socket try: # IPTOS_LOWDELAY sock.setsockopt(socket.SOL_IP, socket.IP_TOS, 0x10) sock.setsockopt(socket.SOL_TCP, socket.TCP_NODELAY, 1) except (AttributeError, OSError): sys.stderr.write("WARNING: cannot set socketoption") def read(self, numbytes: int) -> bytes: "Read exactly 'bytes' bytes from the socket." buf = b"" while len(buf) < numbytes: t = self.sock.recv(numbytes - len(buf)) if not t: raise EOFError buf += t return buf def write(self, data: bytes) -> None: self.sock.sendall(data) def close_read(self) -> None: try: self.sock.shutdown(0) except self.execmodel.socket.error: pass def close_write(self) -> None: try: self.sock.shutdown(1) except self.execmodel.socket.error: pass def wait(self) -> None: pass def kill(self) -> None: pass def start_via( gateway: Gateway, hostport: tuple[str, int] | None = None ) -> tuple[str, int]: """Instantiate a socketserver on the given gateway. Returns a host, port tuple. """ if hostport is None: host, port = ("localhost", 0) else: host, port = hostport from execnet.script import socketserver # execute the above socketserverbootstrap on the other side channel = gateway.remote_exec(socketserver) channel.send((host, port)) realhost, realport = cast("tuple[str, int]", channel.receive()) # self._trace("new_remote received" # "port=%r, hostname = %r" %(realport, hostname)) if not realhost or realhost == "0.0.0.0": realhost = "localhost" return realhost, realport def create_io(spec: XSpec, group: Group, execmodel: ExecModel) -> SocketIO: assert spec.socket is not None assert not spec.python, "socket: specifying python executables not yet supported" gateway_id = spec.installvia if gateway_id: host, port = start_via(group[gateway_id]) else: host, port_str = spec.socket.split(":") port = int(port_str) socket = execmodel.socket sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) io = SocketIO(sock, execmodel) io.remoteaddress = "%s:%d" % (host, port) try: sock.connect((host, port)) except execmodel.socket.gaierror as e: raise HostNotFound() from e return io execnet-2.1.1/src/execnet/multi.py000066400000000000000000000267021460460142100171010ustar00rootroot00000000000000""" Managing Gateway Groups and interactions with multiple channels. (c) 2008-2014, Holger Krekel and others """ from __future__ import annotations import atexit import types from functools import partial from threading import Lock from typing import TYPE_CHECKING from typing import Any from typing import Callable from typing import Iterable from typing import Iterator from typing import Literal from typing import Sequence from typing import overload from . import gateway_bootstrap from . import gateway_io from .gateway_base import Channel from .gateway_base import ExecModel from .gateway_base import WorkerPool from .gateway_base import get_execmodel from .gateway_base import trace from .xspec import XSpec if TYPE_CHECKING: from .gateway import Gateway NO_ENDMARKER_WANTED = object() class Group: """Gateway Group.""" defaultspec = "popen" def __init__( self, xspecs: Iterable[XSpec | str | None] = (), execmodel: str = "thread" ) -> None: """Initialize a group and make gateways as specified. execmodel can be one of the supported execution models. """ self._gateways: list[Gateway] = [] self._autoidcounter = 0 self._autoidlock = Lock() self._gateways_to_join: list[Gateway] = [] # we use the same execmodel for all of the Gateway objects # we spawn on our side. Probably we should not allow different # execmodels between different groups but not clear. # Note that "other side" execmodels may differ and is typically # specified by the spec passed to makegateway. self.set_execmodel(execmodel) for xspec in xspecs: self.makegateway(xspec) atexit.register(self._cleanup_atexit) @property def execmodel(self) -> ExecModel: return self._execmodel @property def remote_execmodel(self) -> ExecModel: return self._remote_execmodel def set_execmodel( self, execmodel: str, remote_execmodel: str | None = None ) -> None: """Set the execution model for local and remote site. execmodel can be one of the supported execution models. It determines the execution model for any newly created gateway. If remote_execmodel is not specified it takes on the value of execmodel. NOTE: Execution models can only be set before any gateway is created. """ if self._gateways: raise ValueError( "can not set execution models if " "gateways have been created already" ) if remote_execmodel is None: remote_execmodel = execmodel self._execmodel = get_execmodel(execmodel) self._remote_execmodel = get_execmodel(remote_execmodel) def __repr__(self) -> str: idgateways = [gw.id for gw in self] return "" % idgateways def __getitem__(self, key: int | str | Gateway) -> Gateway: if isinstance(key, int): return self._gateways[key] for gw in self._gateways: if gw == key or gw.id == key: return gw raise KeyError(key) def __contains__(self, key: str) -> bool: try: self[key] return True except KeyError: return False def __len__(self) -> int: return len(self._gateways) def __iter__(self) -> Iterator[Gateway]: return iter(list(self._gateways)) def makegateway(self, spec: XSpec | str | None = None) -> Gateway: """Create and configure a gateway to a Python interpreter. The ``spec`` string encodes the target gateway type and configuration information. The general format is:: key1=value1//key2=value2//... If you leave out the ``=value`` part a True value is assumed. Valid types: ``popen``, ``ssh=hostname``, ``socket=host:port``. Valid configuration:: id= specifies the gateway id python= specifies which python interpreter to execute execmodel=model 'thread', 'main_thread_only', 'eventlet', 'gevent' execution model chdir= specifies to which directory to change nice= specifies process priority of new process env:NAME=value specifies a remote environment variable setting. If no spec is given, self.defaultspec is used. """ if not spec: spec = self.defaultspec if not isinstance(spec, XSpec): spec = XSpec(spec) self.allocate_id(spec) if spec.execmodel is None: spec.execmodel = self.remote_execmodel.backend if spec.via: assert not spec.socket master = self[spec.via] proxy_channel = master.remote_exec(gateway_io) proxy_channel.send(vars(spec)) proxy_io_master = gateway_io.ProxyIO(proxy_channel, self.execmodel) gw = gateway_bootstrap.bootstrap(proxy_io_master, spec) elif spec.popen or spec.ssh or spec.vagrant_ssh: io = gateway_io.create_io(spec, execmodel=self.execmodel) gw = gateway_bootstrap.bootstrap(io, spec) elif spec.socket: from . import gateway_socket sio = gateway_socket.create_io(spec, self, execmodel=self.execmodel) gw = gateway_bootstrap.bootstrap(sio, spec) else: raise ValueError(f"no gateway type found for {spec._spec!r}") gw.spec = spec self._register(gw) if spec.chdir or spec.nice or spec.env: channel = gw.remote_exec( """ import os path, nice, env = channel.receive() if path: if not os.path.exists(path): os.mkdir(path) os.chdir(path) if nice and hasattr(os, 'nice'): os.nice(nice) if env: for name, value in env.items(): os.environ[name] = value """ ) nice = spec.nice and int(spec.nice) or 0 channel.send((spec.chdir, nice, spec.env)) channel.waitclose() return gw def allocate_id(self, spec: XSpec) -> None: """(re-entrant) allocate id for the given xspec object.""" if spec.id is None: with self._autoidlock: id = "gw" + str(self._autoidcounter) self._autoidcounter += 1 if id in self: raise ValueError(f"already have gateway with id {id!r}") spec.id = id def _register(self, gateway: Gateway) -> None: assert not hasattr(gateway, "_group") assert gateway.id assert gateway.id not in self self._gateways.append(gateway) gateway._group = self def _unregister(self, gateway: Gateway) -> None: self._gateways.remove(gateway) self._gateways_to_join.append(gateway) def _cleanup_atexit(self) -> None: trace(f"=== atexit cleanup {self!r} ===") self.terminate(timeout=1.0) def terminate(self, timeout: float | None = None) -> None: """Trigger exit of member gateways and wait for termination of member gateways and associated subprocesses. After waiting timeout seconds try to to kill local sub processes of popen- and ssh-gateways. Timeout defaults to None meaning open-ended waiting and no kill attempts. """ while self: vias: set[str] = set() for gw in self: if gw.spec.via: vias.add(gw.spec.via) for gw in self: if gw.id not in vias: gw.exit() def join_wait(gw: Gateway) -> None: gw.join() gw._io.wait() def kill(gw: Gateway) -> None: trace("Gateways did not come down after timeout: %r" % gw) gw._io.kill() safe_terminate( self.execmodel, timeout, [ (partial(join_wait, gw), partial(kill, gw)) for gw in self._gateways_to_join ], ) self._gateways_to_join[:] = [] def remote_exec( self, source: str | types.FunctionType | Callable[..., object] | types.ModuleType, **kwargs, ) -> MultiChannel: """remote_exec source on all member gateways and return a MultiChannel connecting to all sub processes.""" channels = [] for gw in self: channels.append(gw.remote_exec(source, **kwargs)) return MultiChannel(channels) class MultiChannel: def __init__(self, channels: Sequence[Channel]) -> None: self._channels = channels def __len__(self) -> int: return len(self._channels) def __iter__(self) -> Iterator[Channel]: return iter(self._channels) def __getitem__(self, key: int) -> Channel: return self._channels[key] def __contains__(self, chan: Channel) -> bool: return chan in self._channels def send_each(self, item: object) -> None: for ch in self._channels: ch.send(item) @overload def receive_each(self, withchannel: Literal[False] = ...) -> list[Any]: pass @overload def receive_each(self, withchannel: Literal[True]) -> list[tuple[Channel, Any]]: pass def receive_each( self, withchannel: bool = False ) -> list[tuple[Channel, Any]] | list[Any]: assert not hasattr(self, "_queue") l: list[object] = [] for ch in self._channels: obj = ch.receive() if withchannel: l.append((ch, obj)) else: l.append(obj) return l def make_receive_queue(self, endmarker: object = NO_ENDMARKER_WANTED): try: return self._queue # type: ignore[has-type] except AttributeError: self._queue = None for ch in self._channels: if self._queue is None: self._queue = ch.gateway.execmodel.queue.Queue() def putreceived(obj, channel: Channel = ch) -> None: self._queue.put((channel, obj)) # type: ignore[union-attr] if endmarker is NO_ENDMARKER_WANTED: ch.setcallback(putreceived) else: ch.setcallback(putreceived, endmarker=endmarker) return self._queue def waitclose(self) -> None: first = None for ch in self._channels: try: ch.waitclose() except ch.RemoteError as exc: if first is None: first = exc if first: raise first def safe_terminate( execmodel: ExecModel, timeout: float | None, list_of_paired_functions ) -> None: workerpool = WorkerPool(execmodel) def termkill(termfunc, killfunc) -> None: termreply = workerpool.spawn(termfunc) try: termreply.get(timeout=timeout) except OSError: killfunc() replylist = [] for termfunc, killfunc in list_of_paired_functions: reply = workerpool.spawn(termkill, termfunc, killfunc) replylist.append(reply) for reply in replylist: reply.get() workerpool.waitall(timeout=timeout) default_group = Group() makegateway = default_group.makegateway set_execmodel = default_group.set_execmodel execnet-2.1.1/src/execnet/py.typed000066400000000000000000000000001460460142100170530ustar00rootroot00000000000000execnet-2.1.1/src/execnet/rsync.py000066400000000000000000000212521460460142100171000ustar00rootroot00000000000000""" 1:N rsync implementation on top of execnet. (c) 2006-2009, Armin Rigo, Holger Krekel, Maciej Fijalkowski """ from __future__ import annotations import os import stat from hashlib import md5 from queue import Queue from typing import Callable from typing import Literal import execnet.rsync_remote from execnet.gateway import Gateway from execnet.gateway_base import BaseGateway from execnet.gateway_base import Channel class RSync: """This class allows to send a directory structure (recursively) to one or multiple remote filesystems. There is limited support for symlinks, which means that symlinks pointing to the sourcetree will be send "as is" while external symlinks will be just copied (regardless of existence of such a path on remote side). """ def __init__(self, sourcedir, callback=None, verbose: bool = True) -> None: self._sourcedir = str(sourcedir) self._verbose = verbose assert callback is None or callable(callback) self._callback = callback self._channels: dict[Channel, Callable[[], None] | None] = {} self._receivequeue: Queue[ tuple[ Channel, ( None | tuple[Literal["send"], tuple[list[str], bytes]] | tuple[Literal["list_done"], None] | tuple[Literal["ack"], str] | tuple[Literal["links"], None] | tuple[Literal["done"], None] ), ] ] = Queue() self._links: list[tuple[Literal["linkbase", "link"], str, str]] = [] def filter(self, path: str) -> bool: return True def _end_of_channel(self, channel: Channel) -> None: if channel in self._channels: # too early! we must have got an error channel.waitclose() # or else we raise one raise OSError(f"connection unexpectedly closed: {channel.gateway} ") def _process_link(self, channel: Channel) -> None: for link in self._links: channel.send(link) # completion marker, this host is done channel.send(42) def _done(self, channel: Channel) -> None: """Call all callbacks.""" finishedcallback = self._channels.pop(channel) if finishedcallback: finishedcallback() channel.waitclose() def _list_done(self, channel: Channel) -> None: # sum up all to send if self._callback: s = sum([self._paths[i] for i in self._to_send[channel]]) self._callback("list", s, channel) def _send_item( self, channel: Channel, modified_rel_path_components: list[str], checksum: bytes, ) -> None: """Send one item.""" modifiedpath = os.path.join(self._sourcedir, *modified_rel_path_components) try: f = open(modifiedpath, "rb") data = f.read() except OSError: data = None # provide info to progress callback function modified_rel_path = "/".join(modified_rel_path_components) if data is not None: self._paths[modified_rel_path] = len(data) else: self._paths[modified_rel_path] = 0 if channel not in self._to_send: self._to_send[channel] = [] self._to_send[channel].append(modified_rel_path) # print "sending", modified_rel_path, data and len(data) or 0, checksum if data is not None: f.close() if checksum is not None and checksum == md5(data).digest(): data = None # not really modified else: self._report_send_file(channel.gateway, modified_rel_path) channel.send(data) def _report_send_file(self, gateway: BaseGateway, modified_rel_path: str) -> None: if self._verbose: print(f"{gateway} <= {modified_rel_path}") def send(self, raises: bool = True) -> None: """Sends a sourcedir to all added targets. raises indicates whether to raise an error or return in case of lack of targets. """ if not self._channels: if raises: raise OSError( "no targets available, maybe you " "are trying call send() twice?" ) return # normalize a trailing '/' away self._sourcedir = os.path.dirname(os.path.join(self._sourcedir, "x")) # send directory structure and file timestamps/sizes self._send_directory_structure(self._sourcedir) # paths and to_send are only used for doing # progress-related callbacks self._paths: dict[str, int] = {} self._to_send: dict[Channel, list[str]] = {} # send modified file to clients while self._channels: channel, req = self._receivequeue.get() if req is None: self._end_of_channel(channel) else: if req[0] == "links": self._process_link(channel) elif req[0] == "done": self._done(channel) elif req[0] == "ack": if self._callback: self._callback("ack", self._paths[req[1]], channel) elif req[0] == "list_done": self._list_done(channel) elif req[0] == "send": self._send_item(channel, req[1][0], req[1][1]) else: assert "Unknown command %s" % req[0] # type: ignore[unreachable] def add_target( self, gateway: Gateway, destdir: str | os.PathLike[str], finishedcallback: Callable[[], None] | None = None, **options, ) -> None: """Add a remote target specified via a gateway and a remote destination directory.""" for name in options: assert name in ("delete",) def itemcallback(req) -> None: self._receivequeue.put((channel, req)) channel = gateway.remote_exec(execnet.rsync_remote) channel.reconfigure(py2str_as_py3str=False, py3str_as_py2str=False) channel.setcallback(itemcallback, endmarker=None) channel.send((str(destdir), options)) self._channels[channel] = finishedcallback def _broadcast(self, msg: object) -> None: for channel in self._channels: channel.send(msg) def _send_link( self, linktype: Literal["linkbase", "link"], basename: str, linkpoint: str, ) -> None: self._links.append((linktype, basename, linkpoint)) def _send_directory(self, path: str) -> None: # dir: send a list of entries names = [] subpaths = [] for name in os.listdir(path): p = os.path.join(path, name) if self.filter(p): names.append(name) subpaths.append(p) mode = os.lstat(path).st_mode self._broadcast([mode, *names]) for p in subpaths: self._send_directory_structure(p) def _send_link_structure(self, path: str) -> None: sourcedir = self._sourcedir basename = path[len(self._sourcedir) + 1 :] linkpoint = os.readlink(path) # On Windows, readlink returns an extended path (//?/) for # absolute links, but relpath doesn't like mixing extended # and non-extended paths. So fix it up ourselves. if ( os.path.__name__ == "ntpath" and linkpoint.startswith("\\\\?\\") and not self._sourcedir.startswith("\\\\?\\") ): sourcedir = "\\\\?\\" + self._sourcedir try: relpath = os.path.relpath(linkpoint, sourcedir) except ValueError: relpath = None if ( relpath is not None and relpath not in (os.curdir, os.pardir) and not relpath.startswith(os.pardir + os.sep) ): self._send_link("linkbase", basename, relpath) else: # relative or absolute link, just send it self._send_link("link", basename, linkpoint) self._broadcast(None) def _send_directory_structure(self, path: str) -> None: try: st = os.lstat(path) except OSError: self._broadcast((None, 0, 0)) return if stat.S_ISREG(st.st_mode): # regular file: send a mode/timestamp/size pair self._broadcast((st.st_mode, st.st_mtime, st.st_size)) elif stat.S_ISDIR(st.st_mode): self._send_directory(path) elif stat.S_ISLNK(st.st_mode): self._send_link_structure(path) else: raise ValueError(f"cannot sync {path!r}") execnet-2.1.1/src/execnet/rsync_remote.py000066400000000000000000000101601460460142100204470ustar00rootroot00000000000000""" (c) 2006-2013, Armin Rigo, Holger Krekel, Maciej Fijalkowski """ from __future__ import annotations from typing import TYPE_CHECKING from typing import Literal from typing import cast if TYPE_CHECKING: from execnet.gateway_base import Channel def serve_rsync(channel: Channel) -> None: import os import shutil import stat from hashlib import md5 destdir, options = cast("tuple[str, dict[str, object]]", channel.receive()) modifiedfiles = [] def remove(path: str) -> None: assert path.startswith(destdir) try: os.unlink(path) except OSError: # assume it's a dir shutil.rmtree(path, True) def receive_directory_structure(path: str, relcomponents: list[str]) -> None: try: st = os.lstat(path) except OSError: st = None msg = channel.receive() if isinstance(msg, list): if st and not stat.S_ISDIR(st.st_mode): os.unlink(path) st = None if not st: os.makedirs(path) mode = msg.pop(0) if mode: # Ensure directories are writable, otherwise a # permission denied error (EACCES) would be raised # when attempting to receive read-only directory # structures. os.chmod(path, mode | 0o700) entrynames = {} for entryname in msg: destpath = os.path.join(path, entryname) receive_directory_structure(destpath, [*relcomponents, entryname]) entrynames[entryname] = True if options.get("delete"): for othername in os.listdir(path): if othername not in entrynames: otherpath = os.path.join(path, othername) remove(otherpath) elif msg is not None: assert isinstance(msg, tuple) checksum = None if st: if stat.S_ISREG(st.st_mode): msg_mode, msg_mtime, msg_size = msg if msg_size != st.st_size: pass elif msg_mtime != st.st_mtime: f = open(path, "rb") checksum = md5(f.read()).digest() f.close() elif msg_mode and msg_mode != st.st_mode: os.chmod(path, msg_mode | 0o700) return else: return # already fine else: remove(path) channel.send(("send", (relcomponents, checksum))) modifiedfiles.append((path, msg)) receive_directory_structure(destdir, []) STRICT_CHECK = False # seems most useful this way for py.test channel.send(("list_done", None)) for path, (mode, time, size) in modifiedfiles: data = cast(bytes, channel.receive()) channel.send(("ack", path[len(destdir) + 1 :])) if data is not None: if STRICT_CHECK and len(data) != size: raise OSError(f"file modified during rsync: {path!r}") f = open(path, "wb") f.write(data) f.close() try: if mode: os.chmod(path, mode) os.utime(path, (time, time)) except OSError: pass del data channel.send(("links", None)) msg = channel.receive() while msg != 42: # we get symlink _type, relpath, linkpoint = cast( "tuple[Literal['linkbase', 'link'], str, str]", msg ) path = os.path.join(destdir, relpath) try: remove(path) except OSError: pass if _type == "linkbase": src = os.path.join(destdir, linkpoint) else: assert _type == "link", _type src = linkpoint os.symlink(src, path) msg = channel.receive() channel.send(("done", None)) if __name__ == "__channelexec__": serve_rsync(channel) # type: ignore[name-defined] # noqa:F821 execnet-2.1.1/src/execnet/script/000077500000000000000000000000001460460142100166725ustar00rootroot00000000000000execnet-2.1.1/src/execnet/script/__init__.py000066400000000000000000000000021460460142100207730ustar00rootroot00000000000000# execnet-2.1.1/src/execnet/script/loop_socketserver.py000066400000000000000000000006421460460142100230160ustar00rootroot00000000000000import os import subprocess import sys if __name__ == "__main__": directory = os.path.dirname(os.path.abspath(sys.argv[0])) script = os.path.join(directory, "socketserver.py") while 1: cmdlist = ["python", script] cmdlist.extend(sys.argv[1:]) text = "starting subcommand: " + " ".join(cmdlist) print(text) process = subprocess.Popen(cmdlist) process.wait() execnet-2.1.1/src/execnet/script/quitserver.py000066400000000000000000000004601460460142100214550ustar00rootroot00000000000000""" send a "quit" signal to a remote server """ from __future__ import annotations import socket import sys host, port = sys.argv[1].split(":") hostport = (host, int(port)) sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.connect(hostport) sock.sendall(b'"raise KeyboardInterrupt"\n') execnet-2.1.1/src/execnet/script/shell.py000066400000000000000000000050511460460142100203540ustar00rootroot00000000000000#! /usr/bin/env python """ a remote python shell for injection into startserver.py """ import os import select import socket import sys from threading import Thread from traceback import print_exc from typing import NoReturn def clientside() -> NoReturn: print("client side starting") host, portstr = sys.argv[1].split(":") port = int(portstr) myself = open(os.path.abspath(sys.argv[0])).read() sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.connect((host, port)) sock.sendall((repr(myself) + "\n").encode()) print("send boot string") inputlist = [sock, sys.stdin] try: while 1: r, w, e = select.select(inputlist, [], []) if sys.stdin in r: line = input() sock.sendall((line + "\n").encode()) if sock in r: line = sock.recv(4096).decode() sys.stdout.write(line) sys.stdout.flush() except BaseException: import traceback traceback.print_exc() sys.exit(1) class promptagent(Thread): def __init__(self, clientsock) -> None: print("server side starting") super.__init__() # type: ignore[call-overload] self.clientsock = clientsock def run(self) -> None: print("Entering thread prompt loop") clientfile = self.clientsock.makefile("w") filein = self.clientsock.makefile("r") loc = self.clientsock.getsockname() while 1: try: clientfile.write("{} {} >>> ".format(*loc)) clientfile.flush() line = filein.readline() if not line: raise EOFError("nothing") if line.strip(): oldout, olderr = sys.stdout, sys.stderr sys.stdout, sys.stderr = clientfile, clientfile try: try: exec(compile(line + "\n", "", "single")) except BaseException: print_exc() finally: sys.stdout = oldout sys.stderr = olderr clientfile.flush() except EOFError: sys.stderr.write("connection close, prompt thread returns") break self.clientsock.close() sock = globals().get("clientsock") if sock is not None: prompter = promptagent(sock) prompter.start() print("promptagent - thread started") else: clientside() execnet-2.1.1/src/execnet/script/socketserver.py000066400000000000000000000075371460460142100217770ustar00rootroot00000000000000#! /usr/bin/env python """ start socket based minimal readline exec server it can exeuted in 2 modes of operation 1. as normal script, that listens for new connections 2. via existing_gateway.remote_exec (as imported module) """ # this part of the program only executes on the server side # from __future__ import annotations import os import sys from typing import TYPE_CHECKING try: import fcntl except ImportError: fcntl = None # type: ignore[assignment] if TYPE_CHECKING: from execnet.gateway_base import Channel from execnet.gateway_base import ExecModel progname = "socket_readline_exec_server-1.2" debug = 0 if debug: # and not os.isatty(sys.stdin.fileno()) f = open("/tmp/execnet-socket-pyout.log", "w") old = sys.stdout, sys.stderr sys.stdout = sys.stderr = f def print_(*args) -> None: print(" ".join(str(arg) for arg in args)) exec( """def exec_(source, locs): exec(source, locs)""" ) def exec_from_one_connection(serversock) -> None: print_(progname, "Entering Accept loop", serversock.getsockname()) clientsock, address = serversock.accept() print_(progname, "got new connection from {} {}".format(*address)) clientfile = clientsock.makefile("rb") print_("reading line") # rstrip so that we can use \r\n for telnet testing source = clientfile.readline().rstrip() clientfile.close() g = {"clientsock": clientsock, "address": address, "execmodel": execmodel} source = eval(source) if source: co = compile(source + "\n", "", "exec") print_(progname, "compiled source, executing") try: exec_(co, g) # type: ignore[name-defined] # noqa: F821 finally: print_(progname, "finished executing code") # background thread might hold a reference to this (!?) # clientsock.close() def bind_and_listen(hostport: str | tuple[str, int], execmodel: ExecModel): socket = execmodel.socket if isinstance(hostport, str): host, port = hostport.split(":") hostport = (host, int(port)) serversock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) # set close-on-exec if hasattr(fcntl, "FD_CLOEXEC"): old = fcntl.fcntl(serversock.fileno(), fcntl.F_GETFD) fcntl.fcntl(serversock.fileno(), fcntl.F_SETFD, old | fcntl.FD_CLOEXEC) # allow the address to be re-used in a reasonable amount of time if os.name == "posix" and sys.platform != "cygwin": serversock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) serversock.bind(hostport) serversock.listen(5) return serversock def startserver(serversock, loop: bool = False) -> None: execute_path = os.getcwd() try: while 1: try: exec_from_one_connection(serversock) except (KeyboardInterrupt, SystemExit): raise except BaseException as exc: if debug: import traceback traceback.print_exc() else: print_("got exception", exc) os.chdir(execute_path) if not loop: break finally: print_("leaving socketserver execloop") serversock.shutdown(2) if __name__ == "__main__": import sys if len(sys.argv) > 1: hostport = sys.argv[1] else: hostport = ":8888" from execnet.gateway_base import get_execmodel execmodel = get_execmodel("thread") serversock = bind_and_listen(hostport, execmodel) startserver(serversock, loop=True) elif __name__ == "__channelexec__": chan: Channel = globals()["channel"] execmodel = chan.gateway.execmodel bindname = chan.receive() assert isinstance(bindname, (str, tuple)) sock = bind_and_listen(bindname, execmodel) port = sock.getsockname() chan.send(port) startserver(sock) execnet-2.1.1/src/execnet/script/socketserverservice.py000066400000000000000000000061551460460142100233530ustar00rootroot00000000000000""" A windows service wrapper for the py.execnet socketserver. To use, run: python socketserverservice.py register net start ExecNetSocketServer """ import sys import threading import servicemanager import win32event import win32evtlogutil import win32service import win32serviceutil from execnet.gateway_base import get_execmodel from . import socketserver appname = "ExecNetSocketServer" class SocketServerService(win32serviceutil.ServiceFramework): _svc_name_ = appname _svc_display_name_ = "%s" % appname _svc_deps_ = ["EventLog"] def __init__(self, args) -> None: # The exe-file has messages for the Event Log Viewer. # Register the exe-file as event source. # # Probably it would be better if this is done at installation time, # so that it also could be removed if the service is uninstalled. # Unfortunately it cannot be done in the 'if __name__ == "__main__"' # block below, because the 'frozen' exe-file does not run this code. # win32evtlogutil.AddSourceToRegistry( self._svc_display_name_, servicemanager.__file__, "Application" ) super.__init__(args) self.hWaitStop = win32event.CreateEvent(None, 0, 0, None) self.WAIT_TIME = 1000 # in milliseconds def SvcStop(self) -> None: self.ReportServiceStatus(win32service.SERVICE_STOP_PENDING) win32event.SetEvent(self.hWaitStop) def SvcDoRun(self) -> None: # Redirect stdout and stderr to prevent "IOError: [Errno 9] # Bad file descriptor". Windows services don't have functional # output streams. sys.stdout = sys.stderr = open("nul", "w") # Write a 'started' event to the event log... win32evtlogutil.ReportEvent( self._svc_display_name_, servicemanager.PYS_SERVICE_STARTED, 0, # category servicemanager.EVENTLOG_INFORMATION_TYPE, (self._svc_name_, ""), ) print("Begin: %s" % self._svc_display_name_) hostport = ":8888" print("Starting py.execnet SocketServer on %s" % hostport) exec_model = get_execmodel("thread") serversock = socketserver.bind_and_listen(hostport, exec_model) thread = threading.Thread( target=socketserver.startserver, args=(serversock,), kwargs={"loop": True} ) thread.setDaemon(True) thread.start() # wait to be stopped or self.WAIT_TIME to pass while True: result = win32event.WaitForSingleObject(self.hWaitStop, self.WAIT_TIME) if result == win32event.WAIT_OBJECT_0: break # write a 'stopped' event to the event log. win32evtlogutil.ReportEvent( self._svc_display_name_, servicemanager.PYS_SERVICE_STOPPED, 0, # category servicemanager.EVENTLOG_INFORMATION_TYPE, (self._svc_name_, ""), ) print("End: %s" % appname) if __name__ == "__main__": # Note that this code will not be run in the 'frozen' exe-file!!! win32serviceutil.HandleCommandLine(SocketServerService) execnet-2.1.1/src/execnet/xspec.py000066400000000000000000000043201460460142100170610ustar00rootroot00000000000000""" (c) 2008-2013, holger krekel """ from __future__ import annotations class XSpec: """Execution Specification: key1=value1//key2=value2 ... * Keys need to be unique within the specification scope * Neither key nor value are allowed to contain "//" * Keys are not allowed to contain "=" * Keys are not allowed to start with underscore * If no "=value" is given, assume a boolean True value """ # XXX allow customization, for only allow specific key names chdir: str | None = None dont_write_bytecode: bool | None = None execmodel: str | None = None id: str | None = None installvia: str | None = None nice: str | None = None popen: bool | None = None python: str | None = None socket: str | None = None ssh: str | None = None ssh_config: str | None = None vagrant_ssh: str | None = None via: str | None = None def __init__(self, string: str) -> None: self._spec = string self.env = {} for keyvalue in string.split("//"): i = keyvalue.find("=") value: str | bool if i == -1: key, value = keyvalue, True else: key, value = keyvalue[:i], keyvalue[i + 1 :] if key[0] == "_": raise AttributeError("%r not a valid XSpec key" % key) if key in self.__dict__: raise ValueError(f"duplicate key: {key!r} in {string!r}") if key.startswith("env:"): self.env[key[4:]] = value else: setattr(self, key, value) def __getattr__(self, name: str) -> None | bool | str: if name[0] == "_": raise AttributeError(name) return None def __repr__(self) -> str: return f"" def __str__(self) -> str: return self._spec def __hash__(self) -> int: return hash(self._spec) def __eq__(self, other: object) -> bool: return self._spec == getattr(other, "_spec", None) def __ne__(self, other: object) -> bool: return self._spec != getattr(other, "_spec", None) def _samefilesystem(self) -> bool: return self.popen is not None and self.chdir is None execnet-2.1.1/testing/000077500000000000000000000000001460460142100146215ustar00rootroot00000000000000execnet-2.1.1/testing/conftest.py000066400000000000000000000142031460460142100170200ustar00rootroot00000000000000from __future__ import annotations import shutil import sys from functools import lru_cache from typing import Callable from typing import Generator from typing import Iterator import execnet import pytest from execnet.gateway import Gateway from execnet.gateway_base import ExecModel from execnet.gateway_base import WorkerPool from execnet.gateway_base import get_execmodel collect_ignore = ["build", "doc/_build"] rsyncdirs = ["conftest.py", "execnet", "testing", "doc"] @pytest.hookimpl(hookwrapper=True) def pytest_runtest_setup(item: pytest.Item) -> Generator[None, None, None]: if item.fspath.purebasename in ("test_group", "test_info"): getspecssh(item.config) # will skip if no gx given yield if "pypy" in item.keywords and not item.config.option.pypy: pytest.skip("pypy tests skipped, use --pypy to run them.") @pytest.fixture def group_function() -> Iterator[execnet.Group]: group = execnet.Group() yield group group.terminate(0.5) @pytest.fixture def makegateway(group_function: execnet.Group) -> Callable[[str], Gateway]: return group_function.makegateway pytest_plugins = ["pytester", "doctest"] # configuration information for tests def pytest_addoption(parser: pytest.Parser) -> None: group = parser.getgroup("execnet", "execnet testing options") group.addoption( "--gx", action="append", dest="gspecs", default=None, help="add a global test environment, XSpec-syntax. ", ) group.addoption( "--pypy", action="store_true", dest="pypy", help="run some tests also against pypy", ) group.addoption( "--broken-isp", action="store_true", dest="broken_isp", help=( "Skips tests that assume your ISP doesn't put up a landing " "page on invalid addresses" ), ) @pytest.fixture def specssh(request: pytest.FixtureRequest) -> execnet.XSpec: return getspecssh(request.config) @pytest.fixture def specsocket(request: pytest.FixtureRequest) -> execnet.XSpec: return getsocketspec(request.config) def getgspecs(config: pytest.Config) -> list[execnet.XSpec]: return [execnet.XSpec(gspec) for gspec in config.getvalueorskip("gspecs")] def getspecssh(config: pytest.Config) -> execnet.XSpec: xspecs = getgspecs(config) for spec in xspecs: if spec.ssh: if not shutil.which("ssh"): pytest.skip("command not found: ssh") return spec pytest.skip("need '--gx ssh=...'") def getsocketspec(config: pytest.Config) -> execnet.XSpec: xspecs = getgspecs(config) for spec in xspecs: if spec.socket: return spec pytest.skip("need '--gx socket=...'") def pytest_generate_tests(metafunc: pytest.Metafunc) -> None: if "gw" in metafunc.fixturenames: assert "anypython" not in metafunc.fixturenames, "need combine?" if hasattr(metafunc.function, "gwtypes"): gwtypes = metafunc.function.gwtypes elif hasattr(metafunc.cls, "gwtype"): gwtypes = [metafunc.cls.gwtype] else: gwtypes = ["popen", "socket", "ssh", "proxy"] metafunc.parametrize("gw", gwtypes, indirect=True) @lru_cache def getexecutable(name: str) -> str | None: if name == "sys.executable": return sys.executable return shutil.which(name) @pytest.fixture(params=("sys.executable", "pypy3")) def anypython(request: pytest.FixtureRequest) -> str: name = request.param executable = getexecutable(name) if executable is None: pytest.skip(f"no {name} found") if "execmodel" in request.fixturenames and name != "sys.executable": backend = request.getfixturevalue("execmodel").backend if backend not in ("thread", "main_thread_only"): pytest.xfail(f"cannot run {backend!r} execmodel with bare {name}") return executable @pytest.fixture(scope="session") def group() -> Iterator[execnet.Group]: g = execnet.Group() yield g g.terminate(timeout=1) @pytest.fixture def gw( request: pytest.FixtureRequest, execmodel: ExecModel, group: execnet.Group, ) -> Gateway: try: return group[request.param] except KeyError: if request.param == "popen": gw = group.makegateway("popen//id=popen//execmodel=%s" % execmodel.backend) elif request.param == "socket": # if execmodel.backend != "thread": # pytest.xfail( # "cannot set remote non-thread execmodel for sockets") pname = "sproxy1" if pname not in group: proxygw = group.makegateway("popen//id=%s" % pname) # assert group['proxygw'].remote_status().receiving gw = group.makegateway( f"socket//id=socket//installvia={pname}" f"//execmodel={execmodel.backend}" ) # TODO(typing): Clarify this assignment. gw.proxygw = proxygw # type: ignore[attr-defined] assert pname in group elif request.param == "ssh": sshhost = request.getfixturevalue("specssh").ssh # we don't use execmodel.backend here # but you can set it when specifying the ssh spec gw = group.makegateway(f"ssh={sshhost}//id=ssh") elif request.param == "proxy": group.makegateway("popen//id=proxy-transport") gw = group.makegateway( "popen//via=proxy-transport//id=proxy" "//execmodel=%s" % execmodel.backend ) else: assert 0, f"unknown execmodel: {request.param}" return gw @pytest.fixture( params=["thread", "main_thread_only", "eventlet", "gevent"], scope="session" ) def execmodel(request: pytest.FixtureRequest) -> ExecModel: if request.param not in ("thread", "main_thread_only"): pytest.importorskip(request.param) if request.param in ("eventlet", "gevent") and sys.platform == "win32": pytest.xfail(request.param + " does not work on win32") return get_execmodel(request.param) @pytest.fixture def pool(execmodel: ExecModel) -> WorkerPool: return WorkerPool(execmodel=execmodel) execnet-2.1.1/testing/test_basics.py000066400000000000000000000321371460460142100175040ustar00rootroot00000000000000# ruff: noqa: B018 from __future__ import annotations import inspect import os import subprocess import sys import textwrap from dataclasses import dataclass from io import BytesIO from pathlib import Path from typing import Any from typing import Callable import execnet import pytest from execnet import gateway from execnet import gateway_base from execnet import gateway_io from execnet.gateway_base import ChannelFactory from execnet.gateway_base import ExecModel from execnet.gateway_base import Message from execnet.gateway_base import Popen2IO skip_win_pypy = pytest.mark.xfail( condition=hasattr(sys, "pypy_version_info") and sys.platform.startswith("win"), reason="failing on Windows on PyPy (#63)", ) @pytest.mark.parametrize("val", ["123", 42, [1, 2, 3], ["23", 25]]) class TestSerializeAPI: def test_serializer_api(self, val: object) -> None: dumped = execnet.dumps(val) val2 = execnet.loads(dumped) assert val == val2 def test_mmap(self, tmp_path: Path, val: object) -> None: mmap = pytest.importorskip("mmap").mmap p = tmp_path / "data.bin" p.write_bytes(execnet.dumps(val)) with p.open("r+b") as f: m = mmap(f.fileno(), 0) val2 = execnet.load(m) assert val == val2 def test_bytesio(self, val: object) -> None: f = BytesIO() execnet.dump(f, val) read = BytesIO(f.getvalue()) val2 = execnet.load(read) assert val == val2 def test_serializer_api_version_error(monkeypatch: pytest.MonkeyPatch) -> None: bchr = gateway_base.bchr monkeypatch.setattr(gateway_base, "DUMPFORMAT_VERSION", bchr(1)) dumped = execnet.dumps(42) monkeypatch.setattr(gateway_base, "DUMPFORMAT_VERSION", bchr(2)) pytest.raises(execnet.DataFormatError, lambda: execnet.loads(dumped)) def test_errors_on_execnet() -> None: assert hasattr(execnet, "RemoteError") assert hasattr(execnet, "TimeoutError") assert hasattr(execnet, "DataFormatError") def test_subprocess_interaction(anypython: str) -> None: line = gateway_io.popen_bootstrapline compile(line, "xyz", "exec") args = [str(anypython), "-c", line] popen = subprocess.Popen( args, bufsize=0, universal_newlines=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE, ) assert popen.stdin is not None assert popen.stdout is not None def send(line: str) -> None: assert popen.stdin is not None popen.stdin.write(line) popen.stdin.flush() def receive() -> str: assert popen.stdout is not None return popen.stdout.readline() try: source = inspect.getsource(read_write_loop) + "read_write_loop()" send(repr(source) + "\n") s = receive() assert s == "ok\n" send("hello\n") s = receive() assert s == "received: hello\n" send("world\n") s = receive() assert s == "received: world\n" send("\n") # terminate loop finally: popen.stdin.close() popen.stdout.close() popen.wait() def read_write_loop() -> None: sys.stdout.write("ok\n") sys.stdout.flush() while 1: try: line = sys.stdin.readline() if not line.strip(): break sys.stdout.write("received: %s" % line) sys.stdout.flush() except (OSError, EOFError): break IO_MESSAGE_EXTRA_SOURCE = """ import sys backend = sys.argv[1] from io import BytesIO import tempfile temp_out = BytesIO() temp_in = BytesIO() io = Popen2IO(temp_out, temp_in, get_execmodel(backend)) for i, handler in enumerate(Message._types): print ("checking", i, handler) for data in "hello", "hello".encode('ascii'): msg1 = Message(i, i, dumps(data)) msg1.to_io(io) x = io.outfile.getvalue() io.outfile.truncate(0) io.outfile.seek(0) io.infile.seek(0) io.infile.write(x) io.infile.seek(0) msg2 = Message.from_io(io) assert msg1.channelid == msg2.channelid, (msg1, msg2) assert msg1.data == msg2.data, (msg1.data, msg2.data) assert msg1.msgcode == msg2.msgcode print ("all passed") """ @dataclass class Checker: python: str path: Path idx: int = 0 def run_check( self, script: str, *extra_args: str, **process_args: Any ) -> subprocess.CompletedProcess[str]: self.idx += 1 check_path = self.path / f"check{self.idx}.py" check_path.write_text(script) return subprocess.run( [self.python, os.fspath(check_path), *extra_args], capture_output=True, text=True, check=True, **process_args, ) @pytest.fixture def checker(anypython: str, tmp_path: Path) -> Checker: return Checker(python=anypython, path=tmp_path) def test_io_message(checker: Checker, execmodel: ExecModel) -> None: out = checker.run_check( inspect.getsource(gateway_base) + IO_MESSAGE_EXTRA_SOURCE, execmodel.backend ) print(out.stdout) assert "all passed" in out.stdout def test_popen_io(checker: Checker, execmodel: ExecModel) -> None: out = checker.run_check( inspect.getsource(gateway_base) + f""" io = init_popen_io(get_execmodel({execmodel.backend!r})) io.write(b"hello") s = io.read(1) assert s == b"x" """, input="x", ) print(out.stderr) assert "hello" in out.stdout def test_popen_io_readloop(execmodel: ExecModel) -> None: sio = BytesIO(b"test") io = Popen2IO(sio, sio, execmodel) real_read = io._read def newread(numbytes: int) -> bytes: if numbytes > 1: numbytes = numbytes - 1 return real_read(numbytes) # type: ignore[no-any-return] io._read = newread result = io.read(3) assert result == b"tes" def test_rinfo_source(checker: Checker) -> None: out = checker.run_check( f""" class Channel: def send(self, data): assert eval(repr(data), {{}}) == data channel = Channel() {inspect.getsource(gateway.rinfo_source)} print ('all passed') """ ) print(out.stdout) assert "all passed" in out.stdout def test_geterrortext(checker: Checker) -> None: out = checker.run_check( inspect.getsource(gateway_base) + """ class Arg(Exception): pass errortext = geterrortext(Arg()) assert "Arg" in errortext try: raise ValueError("17") except ValueError as exc: s = geterrortext(exc) assert "17" in s print ("all passed") """ ) print(out.stdout) assert "all passed" in out.stdout @pytest.mark.skipif("not hasattr(os, 'dup')") def test_stdouterrin_setnull( execmodel: ExecModel, capfd: pytest.CaptureFixture[str] ) -> None: # Backup and restore stdin state, and rely on capfd to handle # this for stdout and stderr. orig_stdin = sys.stdin orig_stdin_fd = os.dup(0) try: # The returned Popen2IO instance can be garbage collected # prematurely since we don't hold a reference here, but we # tolerate this because it is intended to leave behind a # sane state afterwards. gateway_base.init_popen_io(execmodel) os.write(1, b"hello") os.read(0, 1) out, err = capfd.readouterr() assert not out assert not err finally: sys.stdin = orig_stdin os.dup2(orig_stdin_fd, 0) os.close(orig_stdin_fd) class PseudoChannel: class gateway: class _channelfactory: finished = False def __init__(self) -> None: self._sent: list[object] = [] self._closed: list[str | None] = [] self.id = 1000 def send(self, obj: object) -> None: self._sent.append(obj) def close(self, errortext: str | None = None) -> None: self._closed.append(errortext) def test_exectask(execmodel: ExecModel) -> None: io = BytesIO() io.execmodel = execmodel # type: ignore[attr-defined] gw = gateway_base.WorkerGateway(io, id="something") # type: ignore[arg-type] ch = PseudoChannel() gw.executetask((ch, ("raise ValueError()", None, {}))) # type: ignore[arg-type] assert "ValueError" in str(ch._closed[0]) class TestMessage: def test_wire_protocol(self) -> None: for i, handler in enumerate(Message._types): one = BytesIO() data = b"23" # TODO(typing): Maybe make this work. Message(i, 42, data).to_io(one) # type: ignore[arg-type] two = BytesIO(one.getvalue()) msg = Message.from_io(two) assert msg.msgcode == i assert isinstance(msg, Message) assert msg.channelid == 42 assert msg.data == data assert isinstance(repr(msg), str) class TestPureChannel: @pytest.fixture def fac(self, execmodel: ExecModel) -> ChannelFactory: class FakeGateway: def _trace(self, *args) -> None: pass def _send(self, *k) -> None: pass FakeGateway.execmodel = execmodel # type: ignore[attr-defined] return ChannelFactory(FakeGateway()) # type: ignore[arg-type] def test_factory_create(self, fac: ChannelFactory) -> None: chan1 = fac.new() assert chan1.id == 1 chan2 = fac.new() assert chan2.id == 3 def test_factory_getitem(self, fac: ChannelFactory) -> None: chan1 = fac.new() assert fac._channels[chan1.id] == chan1 chan2 = fac.new() assert fac._channels[chan2.id] == chan2 def test_channel_timeouterror(self, fac: ChannelFactory) -> None: channel = fac.new() pytest.raises(IOError, channel.waitclose, timeout=0.01) def test_channel_makefile_incompatmode(self, fac) -> None: channel = fac.new() with pytest.raises(ValueError): channel.makefile("rw") class TestSourceOfFunction: def test_lambda_unsupported(self) -> None: pytest.raises(ValueError, gateway._source_of_function, lambda: 1) def test_wrong_prototype_fails(self) -> None: def prototype(wrong) -> None: pass pytest.raises(ValueError, gateway._source_of_function, prototype) def test_function_without_known_source_fails(self) -> None: # this one won't be able to find the source mess: dict[str, Any] = {} exec("def fail(channel): pass", mess, mess) print(inspect.getsourcefile(mess["fail"])) with pytest.raises(ValueError): gateway._source_of_function(mess["fail"]) def test_function_with_closure_fails(self) -> None: mess: dict[str, Any] = {} def closure(channel: object) -> None: print(mess) with pytest.raises(ValueError): gateway._source_of_function(closure) def test_source_of_nested_function(self) -> None: def working(channel: object) -> None: pass send_source = gateway._source_of_function(working).lstrip("\r\n") expected = "def working(channel: object) -> None:\n pass\n" assert send_source == expected class TestGlobalFinder: def check(self, func) -> list[str]: src = textwrap.dedent(inspect.getsource(func)) code = func.__code__ return gateway._find_non_builtin_globals(src, code) def test_local(self) -> None: def f(a, b, c): d = 3 return d assert self.check(f) == [] def test_global(self) -> None: def f(a, b): sys d = 4 return d assert self.check(f) == ["sys"] def test_builtin(self) -> None: def f() -> None: len assert self.check(f) == [] def test_function_with_global_fails(self) -> None: def func(channel) -> None: sys pytest.raises(ValueError, gateway._source_of_function, func) def test_method_call(self) -> None: # method names are reason # for the simple code object based heusteric failing def f(channel): channel.send(dict(testing=2)) assert self.check(f) == [] @skip_win_pypy def test_remote_exec_function_with_kwargs( anypython: str, makegateway: Callable[[str], gateway.Gateway] ) -> None: def func(channel, data) -> None: channel.send(data) gw = makegateway("popen//python=%s" % anypython) print(f"local version_info {sys.version_info!r}") print(f"remote info: {gw._rinfo()}") ch = gw.remote_exec(func, data=1) result = ch.receive() assert result == 1 def test_remote_exc__no_kwargs(makegateway: Callable[[], gateway.Gateway]) -> None: gw = makegateway() with pytest.raises(TypeError): gw.remote_exec(gateway_base, kwarg=1) with pytest.raises(TypeError): gw.remote_exec("pass", kwarg=1) @skip_win_pypy def test_remote_exec_inspect_stack( makegateway: Callable[[], gateway.Gateway], ) -> None: gw = makegateway() ch = gw.remote_exec( """ import inspect inspect.stack() import traceback channel.send('\\n'.join(traceback.format_stack())) """ ) received = ch.receive() assert isinstance(received, str) assert 'File ""' in received ch.waitclose() execnet-2.1.1/testing/test_channel.py000066400000000000000000000307541460460142100176530ustar00rootroot00000000000000""" mostly functional tests of gateways. """ from __future__ import annotations import time import pytest from execnet.gateway import Gateway from execnet.gateway_base import Channel needs_early_gc = pytest.mark.skipif("not hasattr(sys, 'getrefcount')") needs_osdup = pytest.mark.skipif("not hasattr(os, 'dup')") TESTTIMEOUT = 10.0 # seconds class TestChannelBasicBehaviour: def test_serialize_error(self, gw: Gateway) -> None: ch = gw.remote_exec("channel.send(ValueError(42))") excinfo = pytest.raises(ch.RemoteError, ch.receive) assert "can't serialize" in str(excinfo.value) def test_channel_close_and_then_receive_error(self, gw: Gateway) -> None: channel = gw.remote_exec("raise ValueError") pytest.raises(channel.RemoteError, channel.receive) def test_channel_finish_and_then_EOFError(self, gw: Gateway) -> None: channel = gw.remote_exec("channel.send(42)") x = channel.receive() assert x == 42 pytest.raises(EOFError, channel.receive) pytest.raises(EOFError, channel.receive) pytest.raises(EOFError, channel.receive) def test_waitclose_timeouterror(self, gw: Gateway) -> None: channel = gw.remote_exec("channel.receive()") pytest.raises(channel.TimeoutError, channel.waitclose, 0.02) channel.send(1) channel.waitclose(timeout=TESTTIMEOUT) def test_channel_receive_timeout(self, gw: Gateway) -> None: channel = gw.remote_exec("channel.send(channel.receive())") with pytest.raises(channel.TimeoutError): channel.receive(timeout=0.2) channel.send(1) channel.receive(timeout=TESTTIMEOUT) def test_channel_receive_internal_timeout( self, gw: Gateway, monkeypatch: pytest.MonkeyPatch ) -> None: channel = gw.remote_exec( """ import time time.sleep(0.5) channel.send(1) """ ) monkeypatch.setattr(channel.__class__, "_INTERNALWAKEUP", 0.2) channel.receive() def test_channel_close_and_then_receive_error_multiple(self, gw: Gateway) -> None: channel = gw.remote_exec("channel.send(42) ; raise ValueError") x = channel.receive() assert x == 42 pytest.raises(channel.RemoteError, channel.receive) def test_channel__local_close(self, gw: Gateway) -> None: channel = gw._channelfactory.new() gw._channelfactory._local_close(channel.id) channel.waitclose(0.1) def test_channel__local_close_error(self, gw: Gateway) -> None: channel = gw._channelfactory.new() gw._channelfactory._local_close(channel.id, channel.RemoteError("error")) pytest.raises(channel.RemoteError, channel.waitclose, 0.01) def test_channel_error_reporting(self, gw: Gateway) -> None: channel = gw.remote_exec("def foo():\n return foobar()\nfoo()\n") excinfo = pytest.raises(channel.RemoteError, channel.receive) msg = str(excinfo.value) assert msg.startswith("Traceback (most recent call last):") assert "NameError" in msg assert "foobar" in msg def test_channel_syntax_error(self, gw: Gateway) -> None: # missing colon channel = gw.remote_exec("def foo()\n return 1\nfoo()\n") excinfo = pytest.raises(channel.RemoteError, channel.receive) msg = str(excinfo.value) assert msg.startswith("Traceback (most recent call last):") assert "SyntaxError" in msg def test_channel_iter(self, gw: Gateway) -> None: channel = gw.remote_exec( """ for x in range(3): channel.send(x) """ ) l = list(channel) assert l == [0, 1, 2] def test_channel_pass_in_structure(self, gw: Gateway) -> None: channel = gw.remote_exec( """ ch1, ch2 = channel.receive() data = ch1.receive() ch2.send(data+1) """ ) newchan1 = gw.newchannel() newchan2 = gw.newchannel() channel.send((newchan1, newchan2)) newchan1.send(1) data = newchan2.receive() assert data == 2 def test_channel_multipass(self, gw: Gateway) -> None: channel = gw.remote_exec( """ channel.send(channel) xchan = channel.receive() assert xchan == channel """ ) newchan = channel.receive() assert newchan == channel channel.send(newchan) channel.waitclose() def test_channel_passing_over_channel(self, gw: Gateway) -> None: channel = gw.remote_exec( """ c = channel.gateway.newchannel() channel.send(c) c.send(42) """ ) c = channel.receive() assert isinstance(c, Channel) x = c.receive() assert x == 42 # check that the both sides previous channels are really gone channel.waitclose(TESTTIMEOUT) # assert c.id not in gw._channelfactory newchan = gw.remote_exec( """ assert %d not in channel.gateway._channelfactory._channels """ % channel.id ) newchan.waitclose(TESTTIMEOUT) assert channel.id not in gw._channelfactory._channels def test_channel_receiver_callback(self, gw: Gateway) -> None: l: list[int] = [] # channel = gw.newchannel(receiver=l.append) channel = gw.remote_exec( source=""" channel.send(42) channel.send(13) channel.send(channel.gateway.newchannel()) """ ) channel.setcallback(callback=l.append) pytest.raises(IOError, channel.receive) channel.waitclose(TESTTIMEOUT) assert len(l) == 3 assert l[:2] == [42, 13] assert isinstance(l[2], channel.__class__) def test_channel_callback_after_receive(self, gw: Gateway) -> None: l: list[int] = [] channel = gw.remote_exec( source=""" channel.send(42) channel.send(13) channel.send(channel.gateway.newchannel()) """ ) x = channel.receive() assert x == 42 channel.setcallback(callback=l.append) pytest.raises(IOError, channel.receive) channel.waitclose(TESTTIMEOUT) assert len(l) == 2 assert l[0] == 13 assert isinstance(l[1], channel.__class__) def test_waiting_for_callbacks(self, gw: Gateway) -> None: l = [] def callback(msg) -> None: import time time.sleep(0.2) l.append(msg) channel = gw.remote_exec( source=""" channel.send(42) """ ) channel.setcallback(callback) channel.waitclose(TESTTIMEOUT) assert l == [42] def test_channel_callback_stays_active(self, gw: Gateway) -> None: self.check_channel_callback_stays_active(gw, earlyfree=True) def check_channel_callback_stays_active( self, gw: Gateway, earlyfree: bool = True ) -> Channel | None: if gw.spec.execmodel == "gevent": pytest.xfail("investigate gevent failure") # with 'earlyfree==True', this tests the "sendonly" channel state. l: list[int] = [] channel = gw.remote_exec( source=""" import _thread import time def producer(subchannel): for i in range(5): time.sleep(0.15) subchannel.send(i*100) channel2 = channel.receive() _thread.start_new_thread(producer, (channel2,)) del channel2 """ ) subchannel = gw.newchannel() subchannel.setcallback(l.append) channel.send(subchannel) subchan = None if earlyfree else subchannel counter = 100 while len(l) < 5: if subchan and subchan.isclosed(): break counter -= 1 print(counter) if not counter: pytest.fail("timed out waiting for the answer[%d]" % len(l)) time.sleep(0.04) # busy-wait assert l == [0, 100, 200, 300, 400] return subchan @needs_early_gc def test_channel_callback_remote_freed(self, gw: Gateway) -> None: channel = self.check_channel_callback_stays_active(gw, earlyfree=False) assert channel is not None # freed automatically at the end of producer() channel.waitclose(TESTTIMEOUT) def test_channel_endmarker_callback(self, gw: Gateway) -> None: l: list[int | Channel] = [] channel = gw.remote_exec( source=""" channel.send(42) channel.send(13) channel.send(channel.gateway.newchannel()) """ ) channel.setcallback(l.append, 999) pytest.raises(IOError, channel.receive) channel.waitclose(TESTTIMEOUT) assert len(l) == 4 assert l[:2] == [42, 13] assert isinstance(l[2], channel.__class__) assert l[3] == 999 def test_channel_endmarker_callback_error(self, gw: Gateway) -> None: q = gw.execmodel.queue.Queue() channel = gw.remote_exec( source=""" raise ValueError() """ ) channel.setcallback(q.put, endmarker=999) val = q.get(TESTTIMEOUT) assert val == 999 err = channel._getremoteerror() assert err assert str(err).find("ValueError") != -1 def test_channel_callback_error(self, gw: Gateway) -> None: channel = gw.remote_exec( """ def f(item): raise ValueError(42) ch = channel.gateway.newchannel() ch.setcallback(f) channel.send(ch) channel.receive() assert ch.isclosed() """ ) subchan = channel.receive() assert isinstance(subchan, Channel) subchan.send(1) with pytest.raises(subchan.RemoteError) as excinfo: subchan.waitclose(TESTTIMEOUT) assert "42" in excinfo.value.formatted channel.send(1) channel.waitclose() class TestChannelFile: def test_channel_file_write(self, gw: Gateway) -> None: channel = gw.remote_exec( """ f = channel.makefile() f.write("hello world\\n") f.close() channel.send(42) """ ) first = channel.receive() assert isinstance(first, str) assert first.strip() == "hello world" second = channel.receive() assert second == 42 def test_channel_file_write_error(self, gw: Gateway) -> None: channel = gw.remote_exec("pass") f = channel.makefile() assert not f.isatty() channel.waitclose(TESTTIMEOUT) with pytest.raises(IOError): f.write(b"hello") def test_channel_file_proxyclose(self, gw: Gateway) -> None: channel = gw.remote_exec( """ f = channel.makefile(proxyclose=True) f.write("hello world") f.close() channel.send(42) """ ) first = channel.receive() assert isinstance(first, str) assert first.strip() == "hello world" pytest.raises(channel.RemoteError, channel.receive) def test_channel_file_read(self, gw: Gateway) -> None: channel = gw.remote_exec( """ f = channel.makefile(mode='r') s = f.read(2) channel.send(s) s = f.read(5) channel.send(s) """ ) channel.send("xyabcde") s1 = channel.receive() s2 = channel.receive() assert s1 == "xy" assert s2 == "abcde" def test_channel_file_read_empty(self, gw: Gateway) -> None: channel = gw.remote_exec("pass") f = channel.makefile(mode="r") s = f.read(3) assert s == "" s = f.read(5) assert s == "" def test_channel_file_readline_remote(self, gw: Gateway) -> None: channel = gw.remote_exec( """ channel.send('123\\n45') """ ) channel.waitclose(TESTTIMEOUT) f = channel.makefile(mode="r") s = f.readline() assert s == "123\n" s = f.readline() assert s == "45" def test_channel_makefile_incompatmode(self, gw: Gateway) -> None: channel = gw.newchannel() with pytest.raises(ValueError): channel.makefile("rw") # type: ignore[call-overload] execnet-2.1.1/testing/test_compatibility_regressions.py000066400000000000000000000015171460460142100235320ustar00rootroot00000000000000from execnet import gateway_base def test_opcodes() -> None: data = vars(gateway_base.opcode) computed = {k: v for k, v in data.items() if "__" not in k} assert computed == { "BUILDTUPLE": b"@", "BYTES": b"A", "CHANNEL": b"B", "FALSE": b"C", "FLOAT": b"D", "FROZENSET": b"E", "INT": b"F", "LONG": b"G", "LONGINT": b"H", "LONGLONG": b"I", "NEWDICT": b"J", "NEWLIST": b"K", "NONE": b"L", "PY2STRING": b"M", "PY3STRING": b"N", "SET": b"O", "SETITEM": b"P", "STOP": b"Q", "TRUE": b"R", "UNICODE": b"S", # added in 1.4 # causes a regression since it was ordered in # between CHANNEL and FALSE as "C" moving the other items "COMPLEX": b"T", } execnet-2.1.1/testing/test_gateway.py000066400000000000000000000510301460460142100176720ustar00rootroot00000000000000""" mostly functional tests of gateways. """ from __future__ import annotations import os import pathlib import shutil import signal import sys from textwrap import dedent from typing import Callable import execnet import pytest from execnet import gateway_base from execnet import gateway_io from execnet.gateway import Gateway TESTTIMEOUT = 10.0 # seconds needs_osdup = pytest.mark.skipif("not hasattr(os, 'dup')") flakytest = pytest.mark.xfail( reason="on some systems this test fails due to timing problems" ) skip_win_pypy = pytest.mark.xfail( condition=hasattr(sys, "pypy_version_info") and sys.platform.startswith("win"), reason="failing on Windows on PyPy (#63)", ) class TestBasicGateway: def test_correct_setup(self, gw: Gateway) -> None: assert gw.hasreceiver() assert gw in gw._group assert gw.id in gw._group assert gw.spec def test_repr_doesnt_crash(self, gw: Gateway) -> None: assert isinstance(repr(gw), str) def test_attribute__name__(self, gw: Gateway) -> None: channel = gw.remote_exec("channel.send(__name__)") name = channel.receive() assert name == "__channelexec__" def test_gateway_status_simple(self, gw: Gateway) -> None: status = gw.remote_status() assert status.numexecuting == 0 def test_exc_info_is_clear_after_gateway_startup(self, gw: Gateway) -> None: ch = gw.remote_exec( """ import traceback, sys excinfo = sys.exc_info() if excinfo != (None, None, None): r = traceback.format_exception(*excinfo) else: r = 0 channel.send(r) """ ) res = ch.receive() if res != 0: pytest.fail("remote raised\n%s" % res) def test_gateway_status_no_real_channel(self, gw: Gateway) -> None: numchan = gw._channelfactory.channels() gw.remote_status() numchan2 = gw._channelfactory.channels() # note that on CPython this can not really # fail because refcounting leads to immediate # closure of temporary channels assert numchan2 == numchan @flakytest def test_gateway_status_busy(self, gw: Gateway) -> None: numchannels = gw.remote_status().numchannels ch1 = gw.remote_exec("channel.send(1); channel.receive()") ch2 = gw.remote_exec("channel.receive()") ch1.receive() status = gw.remote_status() assert status.numexecuting == 2 # number of active execution threads assert status.numchannels == numchannels + 2 ch1.send(None) ch2.send(None) ch1.waitclose() ch2.waitclose() for i in range(10): status = gw.remote_status() if status.numexecuting == 0: break else: pytest.fail("did not get correct remote status") # race condition assert status.numchannels <= numchannels def test_remote_exec_module(self, tmp_path: pathlib.Path, gw: Gateway) -> None: p = tmp_path / "remotetest.py" p.write_text("channel.send(1)") mod = type(os)("remotetest") mod.__file__ = str(p) channel = gw.remote_exec(mod) name = channel.receive() assert name == 1 p.write_text("channel.send(2)") channel = gw.remote_exec(mod) name = channel.receive() assert name == 2 def test_remote_exec_module_is_removed( self, gw: Gateway, tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch ) -> None: remotetest = tmp_path / "remote.py" remotetest.write_text( dedent( """ def remote(): return True if __name__ == '__channelexec__': for item in channel: # noqa channel.send(eval(item)) # noqa """ ) ) monkeypatch.syspath_prepend(tmp_path) import remote # type: ignore[import-not-found] ch = gw.remote_exec(remote) # simulate sending the code to a remote location that does not have # access to the source shutil.rmtree(tmp_path) ch.send("remote()") try: result = ch.receive() finally: ch.close() assert result is True def test_remote_exec_module_with_traceback( self, gw: Gateway, tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch, ) -> None: remotetestpy = tmp_path / "remotetest.py" remotetestpy.write_text( dedent( """ def run_me(channel=None): raise ValueError('me') if __name__ == '__channelexec__': run_me() """ ) ) monkeypatch.syspath_prepend(tmp_path) import remotetest # type: ignore[import-not-found] ch = gw.remote_exec(remotetest) try: ch.receive() except execnet.gateway_base.RemoteError as e: assert 'remotetest.py", line 3, in run_me' in str(e) assert "ValueError: me" in str(e) finally: ch.close() ch = gw.remote_exec(remotetest.run_me) try: ch.receive() except execnet.gateway_base.RemoteError as e: assert 'remotetest.py", line 3, in run_me' in str(e) assert "ValueError: me" in str(e) finally: ch.close() def test_correct_setup_no_py(self, gw: Gateway) -> None: channel = gw.remote_exec( """ import sys channel.send(list(sys.modules)) """ ) remotemodules = channel.receive() assert isinstance(remotemodules, list) assert "py" not in remotemodules, "py should not be imported on remote side" def test_remote_exec_waitclose(self, gw: Gateway) -> None: channel = gw.remote_exec("pass") channel.waitclose(TESTTIMEOUT) def test_remote_exec_waitclose_2(self, gw: Gateway) -> None: channel = gw.remote_exec("def gccycle(): pass") channel.waitclose(TESTTIMEOUT) def test_remote_exec_waitclose_noarg(self, gw: Gateway) -> None: channel = gw.remote_exec("pass") channel.waitclose() def test_remote_exec_error_after_close(self, gw: Gateway) -> None: channel = gw.remote_exec("pass") channel.waitclose(TESTTIMEOUT) pytest.raises(IOError, channel.send, 0) def test_remote_exec_no_explicit_close(self, gw: Gateway) -> None: channel = gw.remote_exec("channel.close()") with pytest.raises(channel.RemoteError) as excinfo: channel.waitclose(TESTTIMEOUT) assert "explicit" in excinfo.value.formatted def test_remote_exec_channel_anonymous(self, gw: Gateway) -> None: channel = gw.remote_exec( """ obj = channel.receive() channel.send(obj) """ ) channel.send(42) result = channel.receive() assert result == 42 @needs_osdup def test_confusion_from_os_write_stdout(self, gw: Gateway) -> None: channel = gw.remote_exec( """ import os os.write(1, 'confusion!'.encode('ascii')) channel.send(channel.receive() * 6) channel.send(channel.receive() * 6) """ ) channel.send(3) res = channel.receive() assert res == 18 channel.send(7) res = channel.receive() assert res == 42 @needs_osdup def test_confusion_from_os_write_stderr(self, gw: Gateway) -> None: channel = gw.remote_exec( """ import os os.write(2, 'test'.encode('ascii')) channel.send(channel.receive() * 6) channel.send(channel.receive() * 6) """ ) channel.send(3) res = channel.receive() assert res == 18 channel.send(7) res = channel.receive() assert res == 42 def test__rinfo(self, gw: Gateway) -> None: rinfo = gw._rinfo() assert rinfo.executable assert rinfo.cwd assert rinfo.version_info assert repr(rinfo) old = gw.remote_exec( """ import os.path cwd = os.getcwd() channel.send(os.path.basename(cwd)) os.chdir('..') """ ).receive() try: rinfo2 = gw._rinfo() assert rinfo2.cwd == rinfo.cwd rinfo3 = gw._rinfo(update=True) assert rinfo3.cwd != rinfo2.cwd finally: gw._cache_rinfo = rinfo gw.remote_exec("import os ; os.chdir(%r)" % old).waitclose() class TestPopenGateway: gwtype = "popen" def test_chdir_separation( self, tmp_path: pathlib.Path, makegateway: Callable[[str], Gateway] ) -> None: with pytest.MonkeyPatch.context() as mp: mp.chdir(tmp_path) gw = makegateway("popen") c = gw.remote_exec("import os ; channel.send(os.getcwd())") x = c.receive() assert isinstance(x, str) assert x.lower() == str(tmp_path).lower() def test_remoteerror_readable_traceback(self, gw: Gateway) -> None: with pytest.raises(gateway_base.RemoteError) as e: gw.remote_exec("x y").waitclose() assert "gateway_base" in e.value.formatted def test_many_popen(self, makegateway: Callable[[str], Gateway]) -> None: num = 4 l = [] for i in range(num): l.append(makegateway("popen")) channels = [] for gw in l: channel = gw.remote_exec("""channel.send(42)""") channels.append(channel) while channels: channel = channels.pop() ret = channel.receive() assert ret == 42 def test_rinfo_popen(self, gw: Gateway) -> None: rinfo = gw._rinfo() assert rinfo.executable == sys.executable assert rinfo.cwd == os.getcwd() assert rinfo.version_info == sys.version_info def test_waitclose_on_remote_killed( self, makegateway: Callable[[str], Gateway] ) -> None: gw = makegateway("popen") channel = gw.remote_exec( """ import os import time channel.send(os.getpid()) time.sleep(100) """ ) remotepid = channel.receive() assert isinstance(remotepid, int) os.kill(remotepid, signal.SIGTERM) with pytest.raises(EOFError): channel.waitclose(TESTTIMEOUT) with pytest.raises(IOError): channel.send(None) with pytest.raises(EOFError): channel.receive() def test_receive_on_remote_sysexit(self, gw: Gateway) -> None: channel = gw.remote_exec( """ raise SystemExit() """ ) pytest.raises(channel.RemoteError, channel.receive) def test_dont_write_bytecode(self, makegateway: Callable[[str], Gateway]) -> None: check_sys_dont_write_bytecode = """ import sys channel.send(sys.dont_write_bytecode) """ gw = makegateway("popen") channel = gw.remote_exec(check_sys_dont_write_bytecode) ret = channel.receive() assert not ret gw = makegateway("popen//dont_write_bytecode") channel = gw.remote_exec(check_sys_dont_write_bytecode) ret = channel.receive() assert ret @pytest.mark.skipif("config.option.broken_isp") def test_socket_gw_host_not_found(makegateway: Callable[[str], Gateway]) -> None: with pytest.raises(execnet.HostNotFound): makegateway("socket=qwepoipqwe:9000") class TestSshPopenGateway: gwtype = "ssh" def test_sshconfig_config_parsing( self, monkeypatch: pytest.MonkeyPatch, makegateway: Callable[[str], Gateway] ) -> None: l = [] monkeypatch.setattr( gateway_io, "Popen2IOMaster", lambda *args, **kwargs: l.append(args[0]) ) with pytest.raises(AttributeError): makegateway("ssh=xyz//ssh_config=qwe") assert len(l) == 1 popen_args = l[0] i = popen_args.index("-F") assert popen_args[i + 1] == "qwe" def test_sshaddress(self, gw: Gateway, specssh: execnet.XSpec) -> None: assert gw.remoteaddress == specssh.ssh def test_host_not_found( self, gw: Gateway, makegateway: Callable[[str], Gateway] ) -> None: with pytest.raises(execnet.HostNotFound): makegateway("ssh=nowhere.codespeak.net") class TestThreads: def test_threads(self, makegateway: Callable[[str], Gateway]) -> None: gw = makegateway("popen") gw.remote_init_threads(3) c1 = gw.remote_exec("channel.send(channel.receive())") c2 = gw.remote_exec("channel.send(channel.receive())") c2.send(1) res = c2.receive() assert res == 1 c1.send(42) res = c1.receive() assert res == 42 def test_threads_race_sending(self, makegateway: Callable[[str], Gateway]) -> None: # multiple threads sending data in parallel gw = makegateway("popen") num = 5 gw.remote_init_threads(num) print("remote_init_threads(%d)" % num) channels = [] for x in range(num): ch = gw.remote_exec( """ for x in range(10): channel.send(''*1000) channel.receive() """ ) channels.append(ch) for ch in channels: for x in range(10): ch.receive(TESTTIMEOUT) ch.send(1) for ch in channels: ch.waitclose(TESTTIMEOUT) @flakytest def test_status_with_threads(self, makegateway: Callable[[str], Gateway]) -> None: gw = makegateway("popen") c1 = gw.remote_exec("channel.send(1) ; channel.receive()") c2 = gw.remote_exec("channel.send(2) ; channel.receive()") c1.receive() c2.receive() rstatus = gw.remote_status() assert rstatus.numexecuting == 2 c1.send(1) c2.send(1) c1.waitclose() c2.waitclose() # there is a slight chance that an execution thread # is still active although it's accompanying channel # is already closed. for i in range(10): rstatus = gw.remote_status() if rstatus.numexecuting == 0: return assert 0, "numexecuting didn't drop to zero" class TestTracing: def test_popen_filetracing( self, tmp_path: pathlib.Path, monkeypatch: pytest.MonkeyPatch, makegateway: Callable[[str], Gateway], ) -> None: monkeypatch.setenv("TMP", str(tmp_path)) monkeypatch.setenv("TEMP", str(tmp_path)) # windows monkeypatch.setenv("EXECNET_DEBUG", "1") gw = makegateway("popen") # hack out the debuffilename fn = gw.remote_exec( "import execnet;channel.send(execnet.gateway_base.fn)" ).receive() assert isinstance(fn, str) workerfile = pathlib.Path(fn) assert workerfile.exists() worker_line = "creating workergateway" with workerfile.open() as f: for line in f: if worker_line in line: break else: pytest.fail(f"did not find {worker_line!r} in tracefile") gw.exit() @skip_win_pypy @flakytest def test_popen_stderr_tracing( self, capfd: pytest.CaptureFixture[str], monkeypatch: pytest.MonkeyPatch, makegateway: Callable[[str], Gateway], ) -> None: monkeypatch.setenv("EXECNET_DEBUG", "2") gw = makegateway("popen") pid = gw.remote_exec("import os ; channel.send(os.getpid())").receive() out, err = capfd.readouterr() worker_line = "[%s] creating workergateway" % pid assert worker_line in err gw.exit() def test_no_tracing_by_default(self): assert ( gateway_base.trace == gateway_base.notrace ), "trace does not to default to empty tracing" @pytest.mark.parametrize( "spec, expected_args", [ ("popen//python=python", ["python"]), ("popen//python=sudo -u test python", ["sudo", "-u", "test", "python"]), pytest.param( r"popen//python=/hans\ alt/bin/python", ["/hans alt/bin/python"], marks=pytest.mark.skipif( sys.platform.startswith("win"), reason="invalid spec on Windows" ), ), ('popen//python="/u/test me/python" -e', ["/u/test me/python", "-e"]), ], ) def test_popen_args(spec: str, expected_args: list[str]) -> None: expected_args = [*expected_args, "-u", "-c", gateway_io.popen_bootstrapline] args = gateway_io.popen_args(execnet.XSpec(spec)) assert args == expected_args @pytest.mark.parametrize( "interleave_getstatus", [ pytest.param(True, id="interleave-remote-status"), pytest.param( False, id="no-interleave-remote-status", marks=pytest.mark.xfail( reason="https://github.com/pytest-dev/execnet/issues/123", ), ), ], ) def test_regression_gevent_hangs( group: execnet.Group, interleave_getstatus: bool ) -> None: pytest.importorskip("gevent") gw = group.makegateway("popen//execmodel=gevent") print(gw.remote_status()) def sendback(channel) -> None: channel.send(1234) ch = gw.remote_exec(sendback) if interleave_getstatus: print(gw.remote_status()) assert ch.receive(timeout=0.5) == 1234 def test_assert_main_thread_only( execmodel: gateway_base.ExecModel, makegateway: Callable[[str], Gateway] ) -> None: if execmodel.backend != "main_thread_only": pytest.skip("can only run with main_thread_only") gw = makegateway(f"execmodel={execmodel.backend}//popen") try: # Submit multiple remote_exec requests in quick succession and # assert that all tasks execute in the main thread. It is # necessary to call receive on each channel before the next # remote_exec call, since the channel will raise an error if # concurrent remote_exec requests are submitted as in # test_main_thread_only_concurrent_remote_exec_deadlock. for i in range(10): ch = gw.remote_exec( """ import time, threading time.sleep(0.02) channel.send(threading.current_thread() is threading.main_thread()) """ ) try: res = ch.receive() finally: ch.close() # This doesn't actually block because we closed # the channel already, but it does check for remote # errors and raise them. ch.waitclose() if res is not True: pytest.fail("remote raised\n%s" % res) finally: gw.exit() gw.join() def test_main_thread_only_concurrent_remote_exec_deadlock( execmodel: gateway_base.ExecModel, makegateway: Callable[[str], Gateway] ) -> None: if execmodel.backend != "main_thread_only": pytest.skip("can only run with main_thread_only") gw = makegateway(f"execmodel={execmodel.backend}//popen") channels = [] try: # Submit multiple remote_exec requests in quick succession and # assert that MAIN_THREAD_ONLY_DEADLOCK_TEXT is raised if # concurrent remote_exec requests are submitted for the # main_thread_only execmodel (as compensation for the lack of # back pressure in remote_exec calls which do not attempt to # block until the remote main thread is idle). for i in range(2): channels.append( gw.remote_exec( """ import threading channel.send(threading.current_thread() is threading.main_thread()) # Wait forever, ensuring that the deadlock case triggers. channel.gateway.execmodel.Event().wait() """ ) ) expected_results = ( True, execnet.gateway_base.MAIN_THREAD_ONLY_DEADLOCK_TEXT, ) for expected, ch in zip(expected_results, channels): try: res = ch.receive() except execnet.RemoteError as e: res = e.formatted assert res == expected finally: for ch in channels: ch.close() gw.exit() gw.join() execnet-2.1.1/testing/test_multi.py000066400000000000000000000203631460460142100173700ustar00rootroot00000000000000""" tests for multi channels and gateway Groups """ from __future__ import annotations import gc from time import sleep from typing import Callable import execnet import pytest from execnet import XSpec from execnet.gateway import Gateway from execnet.gateway_base import Channel from execnet.gateway_base import ExecModel from execnet.multi import Group from execnet.multi import safe_terminate class TestMultiChannelAndGateway: def test_multichannel_container_basics( self, gw: Gateway, execmodel: ExecModel ) -> None: mch = execnet.MultiChannel([Channel(gw, i) for i in range(3)]) assert len(mch) == 3 channels = list(mch) assert len(channels) == 3 # ordering for i in range(3): assert channels[i].id == i assert channels[i] == mch[i] assert channels[0] in mch assert channels[1] in mch assert channels[2] in mch def test_multichannel_receive_each(self) -> None: class pseudochannel: def receive(self) -> object: return 12 pc1 = pseudochannel() pc2 = pseudochannel() multichannel = execnet.MultiChannel([pc1, pc2]) # type: ignore[list-item] l = multichannel.receive_each(withchannel=True) assert len(l) == 2 assert l == [(pc1, 12), (pc2, 12)] # type: ignore[comparison-overlap] l2 = multichannel.receive_each(withchannel=False) assert l2 == [12, 12] def test_multichannel_send_each(self) -> None: gm = execnet.Group(["popen"] * 2) mc = gm.remote_exec( """ import os channel.send(channel.receive() + 1) """ ) mc.send_each(41) l = mc.receive_each() assert l == [42, 42] def test_Group_execmodel_setting(self) -> None: gm = execnet.Group() gm.set_execmodel("thread") assert gm.execmodel.backend == "thread" assert gm.remote_execmodel.backend == "thread" gm._gateways.append(1) # type: ignore[arg-type] try: with pytest.raises(ValueError): gm.set_execmodel("eventlet") assert gm.execmodel.backend == "thread" finally: gm._gateways.pop() def test_multichannel_receive_queue_for_two_subprocesses(self) -> None: gm = execnet.Group(["popen"] * 2) mc = gm.remote_exec( """ import os channel.send(os.getpid()) """ ) queue = mc.make_receive_queue() ch, item = queue.get(timeout=10) ch2, item2 = queue.get(timeout=10) assert ch != ch2 assert ch.gateway != ch2.gateway assert item != item2 mc.waitclose() def test_multichannel_waitclose(self) -> None: l = [] class pseudochannel: def waitclose(self) -> None: l.append(0) multichannel = execnet.MultiChannel([pseudochannel(), pseudochannel()]) # type: ignore[list-item] multichannel.waitclose() assert len(l) == 2 class TestGroup: def test_basic_group(self, monkeypatch: pytest.MonkeyPatch) -> None: import atexit atexitlist: list[Callable[[], object]] = [] monkeypatch.setattr(atexit, "register", atexitlist.append) group = Group() assert atexitlist == [group._cleanup_atexit] exitlist = [] joinlist = [] class PseudoIO: def wait(self) -> None: pass class PseudoSpec: via = None class PseudoGW: id = "9999" _io = PseudoIO() spec = PseudoSpec() def exit(self) -> None: exitlist.append(self) group._unregister(self) # type: ignore[arg-type] def join(self) -> None: joinlist.append(self) gw = PseudoGW() group._register(gw) # type: ignore[arg-type] assert len(exitlist) == 0 assert len(joinlist) == 0 group._cleanup_atexit() assert len(exitlist) == 1 assert exitlist == [gw] assert len(joinlist) == 1 assert joinlist == [gw] group._cleanup_atexit() assert len(exitlist) == 1 assert len(joinlist) == 1 def test_group_default_spec(self) -> None: group = Group() group.defaultspec = "not-existing-type" pytest.raises(ValueError, group.makegateway) def test_group_PopenGateway(self) -> None: group = Group() gw = group.makegateway("popen") assert list(group) == [gw] assert group[0] == gw assert len(group) == 1 group._cleanup_atexit() assert not group._gateways def test_group_ordering_and_termination(self) -> None: group = Group() group.makegateway("popen//id=3") group.makegateway("popen//id=2") group.makegateway("popen//id=5") gwlist = list(group) assert len(gwlist) == 3 idlist = [x.id for x in gwlist] assert idlist == list("325") print(group) group.terminate() print(group) assert not group assert repr(group) == "" def test_group_id_allocation(self) -> None: group = Group() specs = [XSpec("popen"), XSpec("popen//id=hello")] group.allocate_id(specs[0]) group.allocate_id(specs[1]) gw = group.makegateway(specs[1]) assert gw.id == "hello" gw = group.makegateway(specs[0]) assert gw.id == "gw0" # pytest.raises(ValueError, # group.allocate_id, XSpec("popen//id=hello")) group.terminate() def test_gateway_and_id(self) -> None: group = Group() gw = group.makegateway("popen//id=hello") assert group["hello"] == gw with pytest.raises((TypeError, AttributeError)): del group["hello"] # type: ignore[attr-defined] with pytest.raises((TypeError, AttributeError)): group["hello"] = 5 # type: ignore[index] assert "hello" in group assert gw in group assert len(group) == 1 gw.exit() assert "hello" not in group with pytest.raises(KeyError): _ = group["hello"] def test_default_group(self) -> None: oldlist = list(execnet.default_group) gw = execnet.makegateway("popen") try: newlist = list(execnet.default_group) assert len(newlist) == len(oldlist) + 1 assert gw in newlist assert gw not in oldlist finally: gw.exit() def test_remote_exec_args(self) -> None: group = Group() group.makegateway("popen") def fun(channel, arg) -> None: channel.send(arg) mch = group.remote_exec(fun, arg=1) result = mch.receive_each() assert result == [1] def test_terminate_with_proxying(self) -> None: group = Group() group.makegateway("popen//id=master") group.makegateway("popen//via=master//id=worker") group.terminate(1.0) @pytest.mark.xfail(reason="active_count() has been broken for some time") def test_safe_terminate(execmodel: ExecModel) -> None: if execmodel.backend not in ("thread", "main_thread_only"): pytest.xfail( "execution model %r does not support task count" % execmodel.backend ) import threading active = threading.active_count() l = [] def term() -> None: sleep(3) def kill() -> None: l.append(1) safe_terminate(execmodel, 1, [(term, kill)] * 10) assert len(l) == 10 sleep(0.1) gc.collect() assert execmodel.active_count() == active # type: ignore[attr-defined] @pytest.mark.xfail(reason="active_count() has been broken for some time") def test_safe_terminate2(execmodel: ExecModel) -> None: if execmodel.backend not in ("thread", "main_thread_only"): pytest.xfail( "execution model %r does not support task count" % execmodel.backend ) import threading active = threading.active_count() l = [] def term() -> None: return def kill() -> None: l.append(1) safe_terminate(execmodel, 3, [(term, kill)] * 10) assert len(l) == 0 sleep(0.1) gc.collect() assert threading.active_count() == active execnet-2.1.1/testing/test_rsync.py000066400000000000000000000243061460460142100173750ustar00rootroot00000000000000import os import pathlib import platform import sys import types import execnet import pytest from execnet import RSync from execnet.gateway import Gateway @pytest.fixture(scope="module") def group(request: pytest.FixtureRequest) -> execnet.Group: group = execnet.Group() request.addfinalizer(group.terminate) return group @pytest.fixture(scope="module") def gw1(request: pytest.FixtureRequest, group: execnet.Group) -> Gateway: gw = group.makegateway("popen//id=gw1") request.addfinalizer(gw.exit) return gw @pytest.fixture(scope="module") def gw2(request: pytest.FixtureRequest, group: execnet.Group) -> Gateway: gw = group.makegateway("popen//id=gw2") request.addfinalizer(gw.exit) return gw needssymlink = pytest.mark.skipif( not hasattr(os, "symlink") or (platform.python_implementation() == "PyPy" and sys.platform == "win32"), reason="os.symlink not available", ) class _dirs(types.SimpleNamespace): source: pathlib.Path dest1: pathlib.Path dest2: pathlib.Path @pytest.fixture def dirs(tmp_path: pathlib.Path) -> _dirs: dirs = _dirs( source=tmp_path / "source", dest1=tmp_path / "dest1", dest2=tmp_path / "dest2", ) dirs.source.mkdir() dirs.dest1.mkdir() dirs.dest2.mkdir() return dirs def are_paths_equal(path1: pathlib.Path, path2: pathlib.Path) -> bool: if os.path.__name__ == "ntpath": # On Windows, os.readlink returns an extended path (\\?\) # for absolute symlinks. However, extended does not compare # equal to non-extended, even when they refer to the same # path otherwise. So we have to fix it up ourselves... is_extended1 = str(path1).startswith("\\\\?\\") is_extended2 = str(path2).startswith("\\\\?\\") if is_extended1 and not is_extended2: path2 = pathlib.Path("\\\\?\\" + str(path2)) if not is_extended1 and is_extended2: path1 = pathlib.Path("\\\\?\\" + str(path1)) return path1 == path2 class TestRSync: def test_notargets(self, dirs: _dirs) -> None: rsync = RSync(dirs.source) with pytest.raises(IOError): rsync.send() assert rsync.send(raises=False) is None # type: ignore[func-returns-value] def test_dirsync(self, dirs: _dirs, gw1: Gateway, gw2: Gateway) -> None: dest = dirs.dest1 dest2 = dirs.dest2 source = dirs.source for s in ("content1", "content2", "content2-a-bit-longer"): subdir = source / "subdir" subdir.mkdir(exist_ok=True) subdir.joinpath("file1").write_text(s) rsync = RSync(dirs.source) rsync.add_target(gw1, dest) rsync.add_target(gw2, dest2) rsync.send() assert dest.joinpath("subdir").is_dir() assert dest.joinpath("subdir", "file1").is_file() assert dest.joinpath("subdir", "file1").read_text() == s assert dest2.joinpath("subdir").is_dir() assert dest2.joinpath("subdir", "file1").is_file() assert dest2.joinpath("subdir", "file1").read_text() == s for x in dest, dest2: fn = x.joinpath("subdir", "file1") os.utime(fn, (0, 0)) source.joinpath("subdir", "file1").unlink() rsync = RSync(source) rsync.add_target(gw2, dest2) rsync.add_target(gw1, dest) rsync.send() assert dest.joinpath("subdir", "file1").is_file() assert dest2.joinpath("subdir", "file1").is_file() rsync = RSync(source) rsync.add_target(gw1, dest, delete=True) rsync.add_target(gw2, dest2) rsync.send() assert not dest.joinpath("subdir", "file1").exists() assert dest2.joinpath("subdir", "file1").exists() def test_dirsync_twice(self, dirs: _dirs, gw1: Gateway, gw2: Gateway) -> None: source = dirs.source source.joinpath("hello").touch() rsync = RSync(source) rsync.add_target(gw1, dirs.dest1) rsync.send() assert dirs.dest1.joinpath("hello").exists() with pytest.raises(IOError): rsync.send() assert rsync.send(raises=False) is None # type: ignore[func-returns-value] rsync.add_target(gw1, dirs.dest2) rsync.send() assert dirs.dest2.joinpath("hello").exists() with pytest.raises(IOError): rsync.send() assert rsync.send(raises=False) is None # type: ignore[func-returns-value] def test_rsync_default_reporting( self, capsys: pytest.CaptureFixture[str], dirs: _dirs, gw1: Gateway ) -> None: source = dirs.source source.joinpath("hello").touch() rsync = RSync(source) rsync.add_target(gw1, dirs.dest1) rsync.send() out, err = capsys.readouterr() assert out.find("hello") != -1 def test_rsync_non_verbose( self, capsys: pytest.CaptureFixture[str], dirs: _dirs, gw1: Gateway ) -> None: source = dirs.source source.joinpath("hello").touch() rsync = RSync(source, verbose=False) rsync.add_target(gw1, dirs.dest1) rsync.send() out, err = capsys.readouterr() assert not out assert not err @pytest.mark.skipif( sys.platform == "win32" or getattr(os, "_name", "") == "nt", reason="irrelevant on windows", ) def test_permissions(self, dirs: _dirs, gw1: Gateway, gw2: Gateway) -> None: source = dirs.source dest = dirs.dest1 onedir = dirs.source / "one" onedir.mkdir() onedir.chmod(448) onefile = dirs.source / "file" onefile.touch() onefile.chmod(504) onefile_mtime = onefile.stat().st_mtime rsync = RSync(source) rsync.add_target(gw1, dest) rsync.send() destdir = dirs.dest1 / onedir.name destfile = dirs.dest1 / onefile.name assert destfile.stat().st_mode & 511 == 504 mode = destdir.stat().st_mode assert mode & 511 == 448 # transfer again with changed permissions onedir.chmod(504) onefile.chmod(448) os.utime(onefile, (onefile_mtime, onefile_mtime)) rsync = RSync(source) rsync.add_target(gw1, dest) rsync.send() mode = destfile.stat().st_mode assert mode & 511 == 448, mode mode = destdir.stat().st_mode assert mode & 511 == 504 @pytest.mark.skipif( sys.platform == "win32" or getattr(os, "_name", "") == "nt", reason="irrelevant on windows", ) def test_read_only_directories(self, dirs: _dirs, gw1: Gateway) -> None: source = dirs.source dest = dirs.dest1 sub = source / "sub" sub.mkdir() subsub = sub / "subsub" subsub.mkdir() sub.chmod(0o500) subsub.chmod(0o500) # The destination directories should be created with the write # permission forced, to avoid raising an EACCES error. rsync = RSync(source) rsync.add_target(gw1, dest) rsync.send() assert dest.joinpath("sub").stat().st_mode & 0o700 assert dest.joinpath("sub", "subsub").stat().st_mode & 0o700 @needssymlink def test_symlink_rsync(self, dirs: _dirs, gw1: Gateway) -> None: source = dirs.source dest = dirs.dest1 subdir = dirs.source / "subdir" subdir.mkdir() sourcefile = subdir / "existent" sourcefile.touch() source.joinpath("rellink").symlink_to(sourcefile.relative_to(source)) source.joinpath("abslink").symlink_to(sourcefile) rsync = RSync(source) rsync.add_target(gw1, dest) rsync.send() rellink = pathlib.Path(os.readlink(str(dest / "rellink"))) assert rellink == pathlib.Path("subdir/existent") abslink = pathlib.Path(os.readlink(str(dest / "abslink"))) expected = dest.joinpath(sourcefile.relative_to(source)) assert are_paths_equal(abslink, expected) @needssymlink def test_symlink2_rsync(self, dirs: _dirs, gw1: Gateway) -> None: source = dirs.source dest = dirs.dest1 subdir = dirs.source / "subdir" subdir.mkdir() sourcefile = subdir / "somefile" sourcefile.touch() subdir.joinpath("link1").symlink_to( subdir.joinpath("link2").relative_to(subdir) ) subdir.joinpath("link2").symlink_to(sourcefile) subdir.joinpath("link3").symlink_to(source.parent) rsync = RSync(source) rsync.add_target(gw1, dest) rsync.send() expected = dest.joinpath(sourcefile.relative_to(dirs.source)) destsub = dest.joinpath("subdir") assert destsub.exists() link1 = pathlib.Path(os.readlink(str(destsub / "link1"))) assert are_paths_equal(link1, pathlib.Path("link2")) link2 = pathlib.Path(os.readlink(str(destsub / "link2"))) assert are_paths_equal(link2, expected) link3 = pathlib.Path(os.readlink(str(destsub / "link3"))) assert are_paths_equal(link3, source.parent) def test_callback(self, dirs: _dirs, gw1: Gateway) -> None: dest = dirs.dest1 source = dirs.source source.joinpath("existent").write_text("a" * 100) source.joinpath("existant2").write_text("a" * 10) total = {} def callback(cmd, lgt, channel): total[(cmd, lgt)] = True rsync = RSync(source, callback=callback) # rsync = RSync() rsync.add_target(gw1, dest) rsync.send() assert total == {("list", 110): True, ("ack", 100): True, ("ack", 10): True} def test_file_disappearing(self, dirs: _dirs, gw1: Gateway) -> None: dest = dirs.dest1 source = dirs.source source.joinpath("ex").write_text("a" * 100) source.joinpath("ex2").write_text("a" * 100) class DRsync(RSync): def filter(self, x: str) -> bool: assert x != str(source) if x.endswith("ex2"): self.x = 1 source.joinpath("ex2").unlink() return True rsync = DRsync(source) rsync.add_target(gw1, dest) rsync.send() assert rsync.x == 1 assert len(list(dest.iterdir())) == 1 assert len(list(source.iterdir())) == 1 execnet-2.1.1/testing/test_serializer.py000066400000000000000000000074771460460142100204220ustar00rootroot00000000000000from __future__ import annotations import os import subprocess import sys from pathlib import Path import execnet import pytest # We use the execnet folder in order to avoid triggering a missing apipkg. pyimportdir = os.fspath(Path(execnet.__file__).parent) class PythonWrapper: def __init__(self, executable: str, tmp_path: Path) -> None: self.executable = executable self.tmp_path = tmp_path def dump(self, obj_rep: str) -> bytes: script_file = self.tmp_path.joinpath("dump.py") script_file.write_text( f""" import sys sys.path.insert(0, {pyimportdir!r}) import gateway_base as serializer sys.stdout = sys.stdout.detach() sys.stdout.write(serializer.dumps_internal({obj_rep})) """ ) res = subprocess.run( [str(self.executable), str(script_file)], capture_output=True, check=True ) return res.stdout def load(self, data: bytes) -> list[str]: script_file = self.tmp_path.joinpath("load.py") script_file.write_text( rf""" import sys sys.path.insert(0, {pyimportdir!r}) import gateway_base as serializer from io import BytesIO data = {data!r} io = BytesIO(data) loader = serializer.Unserializer(io) obj = loader.load() sys.stdout.write(type(obj).__name__ + "\n") sys.stdout.write(repr(obj)) """ ) res = subprocess.run( [str(self.executable), str(script_file)], capture_output=True, check=True, ) return res.stdout.decode("ascii").splitlines() def __repr__(self) -> str: return f"" @pytest.fixture def py3(tmp_path: Path) -> PythonWrapper: return PythonWrapper(sys.executable, tmp_path) @pytest.fixture def dump(py3: PythonWrapper): return py3.dump @pytest.fixture def load(py3: PythonWrapper): return py3.load simple_tests = [ # expected before/after repr ("int", "4"), ("float", "3.25"), ("complex", "(1.78+3.25j)"), ("list", "[1, 2, 3]"), ("tuple", "(1, 2, 3)"), ("dict", "{(1, 2, 3): 32}"), ] @pytest.mark.parametrize(["tp_name", "repr"], simple_tests) def test_simple(tp_name, repr, dump, load) -> None: p = dump(repr) tp, v = load(p) assert tp == tp_name assert v == repr def test_set(load, dump) -> None: p = dump("set((1, 2, 3))") tp, v = load(p) assert tp == "set" # assert v == "{1, 2, 3}" # ordering prevents this assertion assert v.startswith("{") and v.endswith("}") assert "1" in v and "2" in v and "3" in v p = dump("set()") tp, v = load(p) assert tp == "set" assert v == "set()" def test_frozenset(load, dump): p = dump("frozenset((1, 2, 3))") tp, v = load(p) assert tp == "frozenset" assert v == "frozenset({1, 2, 3})" def test_long(load, dump) -> None: really_big = "9223372036854775807324234" p = dump(really_big) tp, v = load(p) assert tp == "int" assert v == really_big def test_bytes(dump, load) -> None: p = dump("b'hi'") tp, v = load(p) assert tp == "bytes" assert v == "b'hi'" def test_str(dump, load) -> None: p = dump("'xyz'") tp, s = load(p) assert tp == "str" assert s == "'xyz'" def test_unicode(load, dump) -> None: p = dump("u'hi'") tp, s = load(p) assert tp == "str" assert s == "'hi'" def test_bool(dump, load) -> None: p = dump("True") tp, s = load(p) assert s == "True" assert tp == "bool" def test_none(dump, load) -> None: p = dump("None") tp, s = load(p) assert s == "None" def test_tuple_nested_with_empty_in_between(dump, load) -> None: p = dump("(1, (), 3)") tp, s = load(p) assert tp == "tuple" assert s == "(1, (), 3)" def test_py2_string_loads() -> None: """Regression test for #267.""" assert execnet.loads(b"\x02M\x00\x00\x00\x01aQ") == b"a" execnet-2.1.1/testing/test_termination.py000066400000000000000000000112611460460142100205640ustar00rootroot00000000000000import os import pathlib import shutil import signal import subprocess import sys from typing import Callable import execnet import pytest from execnet.gateway import Gateway from execnet.gateway_base import ExecModel from execnet.gateway_base import WorkerPool from test_gateway import TESTTIMEOUT execnetdir = pathlib.Path(execnet.__file__).parent.parent skip_win_pypy = pytest.mark.xfail( condition=hasattr(sys, "pypy_version_info") and sys.platform.startswith("win"), reason="failing on Windows on PyPy (#63)", ) def test_exit_blocked_worker_execution_gateway( anypython: str, makegateway: Callable[[str], Gateway], pool: WorkerPool ) -> None: gateway = makegateway("popen//python=%s" % anypython) gateway.remote_exec( """ import time time.sleep(10.0) """ ) def doit() -> int: gateway.exit() return 17 reply = pool.spawn(doit) x = reply.get(timeout=5.0) assert x == 17 def test_endmarker_delivery_on_remote_killterm( makegateway: Callable[[str], Gateway], execmodel: ExecModel ) -> None: if execmodel.backend not in ("thread", "main_thread_only"): pytest.xfail("test and execnet not compatible to greenlets yet") gw = makegateway("popen") q = execmodel.queue.Queue() channel = gw.remote_exec( source=""" import os, time channel.send(os.getpid()) time.sleep(100) """ ) pid = channel.receive() assert isinstance(pid, int) os.kill(pid, signal.SIGTERM) channel.setcallback(q.put, endmarker=999) val = q.get(TESTTIMEOUT) assert val == 999 err = channel._getremoteerror() assert isinstance(err, EOFError) @skip_win_pypy def test_termination_on_remote_channel_receive( monkeypatch: pytest.MonkeyPatch, makegateway: Callable[[str], Gateway] ) -> None: if not shutil.which("ps"): pytest.skip("need 'ps' command to externally check process status") monkeypatch.setenv("EXECNET_DEBUG", "2") gw = makegateway("popen") pid = gw.remote_exec("import os ; channel.send(os.getpid())").receive() gw.remote_exec("channel.receive()") gw._group.terminate() command = ["ps", "-p", str(pid)] output = subprocess.run(command, capture_output=True, text=True, check=False) assert str(pid) not in output.stdout, output def test_close_initiating_remote_no_error( pytester: pytest.Pytester, anypython: str ) -> None: p = pytester.makepyfile( """ import sys sys.path.insert(0, sys.argv[1]) import execnet gw = execnet.makegateway("popen") print ("remote_exec1") ch1 = gw.remote_exec("channel.receive()") print ("remote_exec1") ch2 = gw.remote_exec("channel.receive()") print ("termination") execnet.default_group.terminate() """ ) popen = subprocess.Popen( [anypython, str(p), str(execnetdir)], stdout=None, stderr=subprocess.PIPE ) out, err = popen.communicate() print(err) errstr = err.decode("utf8") lines = [x for x in errstr.splitlines() if "*sys-package" not in x] assert not lines def test_terminate_implicit_does_trykill( pytester: pytest.Pytester, anypython: str, capfd: pytest.CaptureFixture[str], pool: WorkerPool, ) -> None: if pool.execmodel.backend not in ("thread", "main_thread_only"): pytest.xfail("only os threading model supported") if sys.version_info >= (3, 12): pytest.xfail( "since python3.12 this test triggers RuntimeError: can't create new thread at interpreter shutdown" ) p = pytester.makepyfile( """ import sys sys.path.insert(0, %r) import execnet group = execnet.Group() gw = group.makegateway("popen") ch = gw.remote_exec("import time ; channel.send(1) ; time.sleep(100)") ch.receive() # remote execution started sys.stdout.write("1\\n") sys.stdout.flush() sys.stdout.close() class FlushNoOp(object): def flush(self): pass # replace stdout since some python implementations # flush and print errors (for example 3.2) # see Issue #5319 (from the release notes of 3.2 Alpha 2) sys.stdout = FlushNoOp() # use process at-exit group.terminate call """ % str(execnetdir) ) popen = subprocess.Popen([str(anypython), str(p)], stdout=subprocess.PIPE) # sync with start-up assert popen.stdout is not None popen.stdout.readline() reply = pool.spawn(popen.communicate) reply.get(timeout=50) out, err = capfd.readouterr() lines = [x for x in err.splitlines() if "*sys-package" not in x] assert not lines or "Killed" in err execnet-2.1.1/testing/test_threadpool.py000066400000000000000000000125731460460142100204030ustar00rootroot00000000000000import os from pathlib import Path import pytest from execnet.gateway_base import ExecModel from execnet.gateway_base import WorkerPool def test_execmodel(execmodel: ExecModel, tmp_path: Path) -> None: assert execmodel.backend p = tmp_path / "somefile" p.write_text("content") fd = os.open(p, os.O_RDONLY) f = execmodel.fdopen(fd, "r") assert f.read() == "content" f.close() def test_execmodel_basic_attrs(execmodel: ExecModel) -> None: m = execmodel assert callable(m.start) assert m.get_ident() def test_simple(pool: WorkerPool) -> None: reply = pool.spawn(lambda: 42) assert reply.get() == 42 def test_some(pool: WorkerPool, execmodel: ExecModel) -> None: q = execmodel.queue.Queue() num = 4 def f(i: int) -> None: q.put(i) while q.qsize(): execmodel.sleep(0.01) for i in range(num): pool.spawn(f, i) for i in range(num): q.get() # assert len(pool._running) == 4 assert pool.waitall(timeout=1.0) # execmodel.sleep(1) helps on windows? assert len(pool._running) == 0 def test_running_semnatics(pool: WorkerPool, execmodel: ExecModel) -> None: q = execmodel.queue.Queue() def first() -> None: q.get() reply = pool.spawn(first) assert reply.running assert pool.active_count() == 1 q.put(1) assert pool.waitall() assert pool.active_count() == 0 assert not reply.running def test_waitfinish_on_reply(pool: WorkerPool) -> None: l = [] reply = pool.spawn(lambda: l.append(1)) reply.waitfinish() assert l == [1] reply = pool.spawn(lambda: 0 / 0) reply.waitfinish() # no exception raised pytest.raises(ZeroDivisionError, reply.get) @pytest.mark.xfail(reason="WorkerPool does not implement limited size") def test_limited_size(execmodel: ExecModel) -> None: pool = WorkerPool(execmodel, size=1) # type: ignore[call-arg] q = execmodel.queue.Queue() q2 = execmodel.queue.Queue() q3 = execmodel.queue.Queue() def first() -> None: q.put(1) q2.get() pool.spawn(first) assert q.get() == 1 def second() -> None: q3.put(3) # we spawn a second pool to spawn the second function # which should block pool2 = WorkerPool(execmodel) pool2.spawn(pool.spawn, second) assert not pool2.waitall(1.0) assert q3.qsize() == 0 q2.put(2) assert pool2.waitall() assert pool.waitall() def test_get(pool: WorkerPool) -> None: def f() -> int: return 42 reply = pool.spawn(f) result = reply.get() assert result == 42 def test_get_timeout(execmodel: ExecModel, pool: WorkerPool) -> None: def f() -> int: execmodel.sleep(0.2) return 42 reply = pool.spawn(f) with pytest.raises(IOError): reply.get(timeout=0.01) def test_get_excinfo(pool: WorkerPool) -> None: def f() -> None: raise ValueError("42") reply = pool.spawn(f) with pytest.raises(ValueError): reply.get(1.0) with pytest.raises(ValueError): reply.get(1.0) def test_waitall_timeout(pool: WorkerPool, execmodel: ExecModel) -> None: q = execmodel.queue.Queue() def f() -> None: q.get() reply = pool.spawn(f) assert not pool.waitall(0.01) q.put(None) reply.get(timeout=1.0) assert pool.waitall(timeout=0.1) @pytest.mark.skipif(not hasattr(os, "dup"), reason="no os.dup") def test_pool_clean_shutdown( pool: WorkerPool, capfd: pytest.CaptureFixture[str] ) -> None: q = pool.execmodel.queue.Queue() def f() -> None: q.get() pool.spawn(f) assert not pool.waitall(timeout=1.0) pool.trigger_shutdown() with pytest.raises(ValueError): pool.spawn(f) def wait_then_put() -> None: pool.execmodel.sleep(0.1) q.put(1) pool.execmodel.start(wait_then_put) assert pool.waitall() out, err = capfd.readouterr() assert err == "" def test_primary_thread_integration(execmodel: ExecModel) -> None: if execmodel.backend not in ("thread", "main_thread_only"): with pytest.raises(ValueError): WorkerPool(execmodel=execmodel, hasprimary=True) return pool = WorkerPool(execmodel=execmodel, hasprimary=True) queue = execmodel.queue.Queue() def do_integrate() -> None: queue.put(execmodel.get_ident()) pool.integrate_as_primary_thread() execmodel.start(do_integrate) def func() -> None: queue.put(execmodel.get_ident()) pool.spawn(func) ident1 = queue.get() ident2 = queue.get() assert ident1 == ident2 pool.terminate() def test_primary_thread_integration_shutdown(execmodel: ExecModel) -> None: if execmodel.backend not in ("thread", "main_thread_only"): pytest.skip("can only run with threading") pool = WorkerPool(execmodel=execmodel, hasprimary=True) queue = execmodel.queue.Queue() def do_integrate() -> None: queue.put(execmodel.get_ident()) pool.integrate_as_primary_thread() execmodel.start(do_integrate) queue.get() queue2 = execmodel.queue.Queue() def get_two() -> None: queue.put(execmodel.get_ident()) queue2.get() reply = pool.spawn(get_two) # make sure get_two is running and blocked on queue2 queue.get() # then shut down pool.trigger_shutdown() # and let get_two finish queue2.put(1) reply.get() assert pool.waitall(5.0) execnet-2.1.1/testing/test_xspec.py000066400000000000000000000220021460460142100173500ustar00rootroot00000000000000from __future__ import annotations import os import shutil import subprocess import sys from pathlib import Path from typing import Callable import execnet import pytest from execnet import XSpec from execnet.gateway import Gateway from execnet.gateway_io import popen_args from execnet.gateway_io import ssh_args from execnet.gateway_io import vagrant_ssh_args skip_win_pypy = pytest.mark.xfail( condition=hasattr(sys, "pypy_version_info") and sys.platform.startswith("win"), reason="failing on Windows on PyPy (#63)", ) class TestXSpec: def test_norm_attributes(self) -> None: spec = XSpec( r"socket=192.168.102.2:8888//python=c:/this/python3.8//chdir=d:\hello" ) assert spec.socket == "192.168.102.2:8888" assert spec.python == "c:/this/python3.8" assert spec.chdir == r"d:\hello" assert spec.nice is None assert not hasattr(spec, "_xyz") with pytest.raises(AttributeError): spec._hello() # type: ignore[misc,operator] spec = XSpec("socket=192.168.102.2:8888//python=python2.5//nice=3") assert spec.socket == "192.168.102.2:8888" assert spec.python == "python2.5" assert spec.chdir is None assert spec.nice == "3" spec = XSpec("ssh=user@host" "//chdir=/hello/this//python=/usr/bin/python2.5") assert spec.ssh == "user@host" assert spec.python == "/usr/bin/python2.5" assert spec.chdir == "/hello/this" spec = XSpec("popen") assert spec.popen is True def test_ssh_options(self) -> None: spec = XSpec("ssh=-p 22100 user@host//python=python3") assert spec.ssh == "-p 22100 user@host" assert spec.python == "python3" spec = XSpec( "ssh=-i ~/.ssh/id_rsa-passwordless_login -p 22100 user@host" "//python=python3" ) assert spec.ssh == "-i ~/.ssh/id_rsa-passwordless_login -p 22100 user@host" assert spec.python == "python3" def test_execmodel(self) -> None: spec = XSpec("execmodel=thread") assert spec.execmodel == "thread" spec = XSpec("execmodel=eventlet") assert spec.execmodel == "eventlet" def test_ssh_options_and_config(self) -> None: spec = XSpec("ssh=-p 22100 user@host//python=python3") spec.ssh_config = "/home/user/ssh_config" assert ssh_args(spec)[:6] == ["ssh", "-C", "-F", spec.ssh_config, "-p", "22100"] def test_vagrant_options(self) -> None: spec = XSpec("vagrant_ssh=default//python=python3") assert vagrant_ssh_args(spec)[:-1] == ["vagrant", "ssh", "default", "--", "-C"] def test_popen_with_sudo_python(self) -> None: spec = XSpec("popen//python=sudo python3") assert popen_args(spec) == [ "sudo", "python3", "-u", "-c", "import sys;exec(eval(sys.stdin.readline()))", ] def test_env(self) -> None: xspec = XSpec("popen//env:NAME=value1") assert xspec.env["NAME"] == "value1" def test__samefilesystem(self) -> None: assert XSpec("popen")._samefilesystem() assert XSpec("popen//python=123")._samefilesystem() assert not XSpec("popen//chdir=hello")._samefilesystem() def test__spec_spec(self) -> None: for x in ("popen", "popen//python=this"): assert XSpec(x)._spec == x def test_samekeyword_twice_raises(self) -> None: pytest.raises(ValueError, XSpec, "popen//popen") pytest.raises(ValueError, XSpec, "popen//popen=123") def test_unknown_keys_allowed(self) -> None: xspec = XSpec("hello=3") assert xspec.hello == "3" def test_repr_and_string(self) -> None: for x in ("popen", "popen//python=this"): assert repr(XSpec(x)).find("popen") != -1 assert str(XSpec(x)) == x def test_hash_equality(self) -> None: assert XSpec("popen") == XSpec("popen") assert hash(XSpec("popen")) == hash(XSpec("popen")) assert XSpec("popen//python=123") != XSpec("popen") assert hash(XSpec("socket=hello:8080")) != hash(XSpec("popen")) class TestMakegateway: def test_no_type(self, makegateway: Callable[[str], Gateway]) -> None: pytest.raises(ValueError, lambda: makegateway("hello")) @skip_win_pypy def test_popen_default(self, makegateway: Callable[[str], Gateway]) -> None: gw = makegateway("") assert gw.spec.popen assert gw.spec.python is None rinfo = gw._rinfo() # assert rinfo.executable == sys.executable assert rinfo.cwd == os.getcwd() assert rinfo.version_info == sys.version_info @pytest.mark.skipif("not hasattr(os, 'nice')") @pytest.mark.xfail(reason="fails due to timing problems on busy single-core VMs") def test_popen_nice(self, makegateway: Callable[[str], Gateway]) -> None: gw = makegateway("popen") def getnice(channel) -> None: import os if hasattr(os, "nice"): channel.send(os.nice(0)) else: channel.send(None) remotenice = gw.remote_exec(getnice).receive() assert isinstance(remotenice, int) gw.exit() if remotenice is not None: gw = makegateway("popen//nice=5") remotenice2 = gw.remote_exec(getnice).receive() assert remotenice2 == remotenice + 5 def test_popen_env(self, makegateway: Callable[[str], Gateway]) -> None: gw = makegateway("popen//env:NAME123=123") ch = gw.remote_exec( """ import os channel.send(os.environ['NAME123']) """ ) value = ch.receive() assert value == "123" @skip_win_pypy def test_popen_explicit(self, makegateway: Callable[[str], Gateway]) -> None: gw = makegateway("popen//python=%s" % sys.executable) assert gw.spec.python == sys.executable rinfo = gw._rinfo() assert rinfo.executable == sys.executable assert rinfo.cwd == os.getcwd() assert rinfo.version_info == sys.version_info @skip_win_pypy def test_popen_chdir_absolute( self, tmp_path: Path, makegateway: Callable[[str], Gateway] ) -> None: gw = makegateway("popen//chdir=%s" % tmp_path) rinfo = gw._rinfo() assert rinfo.cwd == str(tmp_path.resolve()) @skip_win_pypy def test_popen_chdir_newsub( self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path, makegateway: Callable[[str], Gateway], ) -> None: monkeypatch.chdir(tmp_path) gw = makegateway("popen//chdir=hello") rinfo = gw._rinfo() expected = str(tmp_path.joinpath("hello").resolve()).lower() assert rinfo.cwd.lower() == expected def test_ssh(self, specssh: XSpec, makegateway: Callable[[str], Gateway]) -> None: sshhost = specssh.ssh gw = makegateway("ssh=%s//id=ssh1" % sshhost) assert gw.id == "ssh1" def test_vagrant(self, makegateway: Callable[[str], Gateway]) -> None: vagrant_bin = shutil.which("vagrant") if vagrant_bin is None: pytest.skip("Vagrant binary not in PATH") res = subprocess.run( [vagrant_bin, "status", "default", "--machine-readable"], capture_output=True, encoding="utf-8", errors="replace", check=True, ).stdout print(res) if ",default,state,shutoff\n" in res: pytest.xfail("vm shutoff, run `vagrant up` first") if ",default,state,not_created\n" in res: pytest.xfail("vm not created, run `vagrant up` first") if ",default,state,running\n" not in res: pytest.fail("unknown vm state") gw = makegateway("vagrant_ssh=default//python=python3") rinfo = gw._rinfo() assert rinfo.cwd == "/home/vagrant" assert rinfo.executable == "/usr/bin/python" def test_socket( self, specsocket: XSpec, makegateway: Callable[[str], Gateway] ) -> None: gw = makegateway("socket=%s//id=sock1" % specsocket.socket) rinfo = gw._rinfo() assert rinfo.executable assert rinfo.cwd assert rinfo.version_info assert gw.id == "sock1" # we cannot instantiate a second gateway @pytest.mark.xfail(reason="we can't instantiate a second gateway") def test_socket_second( self, specsocket: XSpec, makegateway: Callable[[str], Gateway] ) -> None: gw = makegateway("socket=%s//id=sock1" % specsocket.socket) gw2 = makegateway("socket=%s//id=sock1" % specsocket.socket) rinfo = gw._rinfo() rinfo2 = gw2._rinfo() assert rinfo.executable == rinfo2.executable assert rinfo.cwd == rinfo2.cwd assert rinfo.version_info == rinfo2.version_info def test_socket_installvia(self) -> None: group = execnet.Group() group.makegateway("popen//id=p1") gw = group.makegateway("socket//installvia=p1//id=s1") assert gw.id == "s1" assert gw.remote_status() group.terminate() execnet-2.1.1/tox.ini000066400000000000000000000011341460460142100144560ustar00rootroot00000000000000[tox] envlist=py{38,39,310,311,312,pypy38},docs,linting isolated_build = true [testenv] usedevelop=true setenv = PYTHONWARNDEFAULTENCODING = 1 deps= pytest pytest-timeout passenv = GITHUB_ACTIONS, HOME, USER, XDG_* commands= python -m pytest {posargs:testing} [testenv:docs] skipsdist = True usedevelop = True changedir = doc deps = sphinx PyYAML commands = sphinx-build -W -b html . _build [testenv:linting] skip_install = True deps = pre-commit>=1.11.0 commands = pre-commit run --all-files --show-diff-on-failure [pytest] timeout = 20 addopts = -ra testpaths = testing