pax_global_header00006660000000000000000000000064144601374740014524gustar00rootroot0000000000000052 comment=a018ae39c54d99a2350c2b930e57b27ae765dafb duet-0.2.9/000077500000000000000000000000001446013747400124755ustar00rootroot00000000000000duet-0.2.9/.github/000077500000000000000000000000001446013747400140355ustar00rootroot00000000000000duet-0.2.9/.github/workflows/000077500000000000000000000000001446013747400160725ustar00rootroot00000000000000duet-0.2.9/.github/workflows/ci.yml000066400000000000000000000060121446013747400172070ustar00rootroot00000000000000name: Continuous Integration on: [pull_request] jobs: format: name: Format check runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 with: fetch-depth: 0 - uses: actions/setup-python@v3 with: python-version: '3.9' architecture: 'x64' - name: Install black run: pip install -r dev/requirements.txt - name: Format run: isort duet --check && black duet --check mypy: name: Type check runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - uses: actions/setup-python@v3 with: python-version: '3.9' architecture: 'x64' - name: Install mypy run: pip install -r dev/requirements.txt - name: Type check run: mypy duet env: PYTHONPATH: '.' lint: name: Lint check runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - uses: actions/setup-python@v3 with: python-version: '3.9' architecture: 'x64' - name: Install pylint run: pip install -r dev/requirements.txt - name: Lint run: pylint duet import: name: Import check strategy: matrix: python-version: ['3.9', '3.10', '3.11'] runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - uses: actions/setup-python@v3 with: python-version: ${{ matrix.python-version }} architecture: 'x64' - name: Install requirements run: pip install -r requirements.txt - name: Import duet run: python -c "import duet" test-linux: name: Pytest Linux strategy: matrix: python-version: ['3.9', '3.10', '3.11'] runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - uses: actions/setup-python@v3 with: python-version: ${{ matrix.python-version }} architecture: 'x64' - name: Install requirements run: pip install -r requirements.txt -r dev/requirements.txt - name: Pytest check run: pytest duet test-windows: name: Pytest Windows strategy: matrix: python-version: ['3.9', '3.10', '3.11'] runs-on: windows-latest steps: - uses: actions/checkout@v3 - uses: actions/setup-python@v3 with: python-version: ${{ matrix.python-version }} architecture: 'x64' - name: Install requirements run: pip install -r requirements.txt -r dev/requirements.txt - name: Pytest Windows run: pytest duet test-macos: name: Pytest MacOS strategy: matrix: python-version: ['3.9', '3.10', '3.11'] runs-on: macos-latest steps: - uses: actions/checkout@v3 - uses: actions/setup-python@v3 with: python-version: ${{ matrix.python-version }} architecture: 'x64' - name: Install requirements run: pip install -r requirements.txt -r dev/requirements.txt - name: Pytest check run: pytest duet duet-0.2.9/.gitignore000066400000000000000000000034331446013747400144700ustar00rootroot00000000000000# Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] *$py.class # C extensions *.so # Distribution / packaging .Python build/ develop-eggs/ dist/ downloads/ eggs/ .eggs/ lib/ lib64/ parts/ sdist/ var/ wheels/ pip-wheel-metadata/ share/python-wheels/ *.egg-info/ .installed.cfg *.egg MANIFEST # PyInstaller # Usually these files are written by a python script from a template # before PyInstaller builds the exe, so as to inject date/other infos into it. *.manifest *.spec # Installer logs pip-log.txt pip-delete-this-directory.txt # Unit test / coverage reports htmlcov/ .tox/ .nox/ .coverage .coverage.* .cache nosetests.xml coverage.xml *.cover *.py,cover .hypothesis/ .pytest_cache/ # Translations *.mo *.pot # Django stuff: *.log local_settings.py db.sqlite3 db.sqlite3-journal # Flask stuff: instance/ .webassets-cache # Scrapy stuff: .scrapy # Sphinx documentation docs/_build/ # PyBuilder target/ # Jupyter Notebook .ipynb_checkpoints # IPython profile_default/ ipython_config.py # pyenv .python-version # pipenv # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. # However, in case of collaboration, if having platform-specific dependencies or dependencies # having no cross-platform support, pipenv may install dependencies that don't work, or not # install all needed dependencies. #Pipfile.lock # PEP 582; used by e.g. github.com/David-OConnor/pyflow __pypackages__/ # Celery stuff celerybeat-schedule celerybeat.pid # SageMath parsed files *.sage.py # Environments .env .venv env/ venv/ ENV/ env.bak/ venv.bak/ # IDEs *.iml .idea # Spyder project settings .spyderproject .spyproject # Rope project settings .ropeproject # mkdocs documentation /site # mypy .mypy_cache/ .dmypy.json dmypy.json # Pyre type checker .pyre/ duet-0.2.9/.mypy.ini000066400000000000000000000005771446013747400142630ustar00rootroot00000000000000[mypy] strict_optional = true plugins = duet.typing show_error_codes = true warn_unused_ignores = true [mypy-__main__] follow_imports = silent ignore_missing_imports = true [mypy-grpc] follow_imports = silent ignore_missing_imports = true [mypy-pytest] follow_imports = silent ignore_missing_imports = true [mypy-aiocontext] follow_imports = silent ignore_missing_imports = true duet-0.2.9/.pylintrc000066400000000000000000000016611446013747400143460ustar00rootroot00000000000000[config] disable=all max-line-length=100 enable= anomalous-backslash-in-string, bad-option-value, bad-reversed-sequence, bad-super-call, continue-in-finally, dangerous-default-value, duplicate-argument-name, expression-not-assigned, f-string-without-interpolation, function-redefined, init-is-generator, line-too-long, mixed-indentation, nonexistent-operator, not-in-loop, no-value-for-parameter pointless-statement, pointless-string-statement, redefined-builtin, relative-import, return-arg-in-generator, return-in-init, return-outside-function, singleton-comparison, syntax-error, undefined-variable, unnecessary-pass, unreachable, unrecognized-inline-option, unused-import, unused-variable, unused-wildcard-import, wildcard-import, wrong-import-order, wrong-import-position, yield-outside-function duet-0.2.9/AUTHORS000066400000000000000000000004501446013747400135440ustar00rootroot00000000000000# This is the list of Duet's significant contributors. # # This does not necessarily list everyone who has contributed code, # especially since many employees of one corporation may be contributing. # To see the full list of contributors, see the revision history in # source control. Google LLC duet-0.2.9/CONTRIBUTING.md000066400000000000000000000021171446013747400147270ustar00rootroot00000000000000# How to Contribute We'd love to accept your patches and contributions to this project. There are just a few small guidelines you need to follow. ## Contributor License Agreement Contributions to this project must be accompanied by a Contributor License Agreement (CLA). You (or your employer) retain the copyright to your contribution; this simply gives us permission to use and redistribute your contributions as part of the project. Head over to to see your current agreements on file or to sign a new one. You generally only need to submit a CLA once, so if you've already submitted one (even if it was for a different project), you probably don't need to do it again. ## Code Reviews All submissions, including submissions by project members, require review. We use GitHub pull requests for this purpose. Consult [GitHub Help](https://help.github.com/articles/about-pull-requests/) for more information on using pull requests. ## Community Guidelines This project follows [Google's Open Source Community Guidelines](https://opensource.google/conduct/). duet-0.2.9/LICENSE000066400000000000000000000261351446013747400135110ustar00rootroot00000000000000 Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright [yyyy] [name of copyright owner] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. duet-0.2.9/MANIFEST.in000066400000000000000000000001061446013747400142300ustar00rootroot00000000000000include LICENSE include requirements.txt include dev/requirements.txt duet-0.2.9/README.md000066400000000000000000000027301446013747400137560ustar00rootroot00000000000000# duet A simple future-based async library for python Duet takes inspiration from the amazing [trio](https://trio.readthedocs.io/en/stable/) library and the [structured concurrency](https://vorpus.org/blog/notes-on-structured-concurrency-or-go-statement-considered-harmful/) approach to async programming that it uses. However, duet differs from trio in two major ways: - Instead of a full-blown implementation of asynchronous IO, duet relies on the `Future` interface for parallelism, and provides a way to run async/await coroutines around those `Future`s. This is useful if you are using an API that returns futures, such as RPC libraries like gRPC. The standard `Future` interface does not implement `__await__` directly, so `Future` instances must be wrapped in `duet.AwaitableFuture`. - duet is re-entrant. At the top level, you run async code by calling `duet.run(foo)`. Inside `foo` suppose you call a function that has not yet been fully refactored to be asynchronous, but itself calls `duet.run(bar)`. Most async libraries, including `trio` and `asyncio`, will raise an exception if you try to "re-enter" the event loop in this way, but duet allows it. We have found that this can simplify the process of refactoring code to be asynchronous because you don't have to completely separate the sync and async parts of your codebase all at once. ## Installation Install from pypi: ``` pip install duet ``` ## Note duet is not an official Google project. duet-0.2.9/dev/000077500000000000000000000000001446013747400132535ustar00rootroot00000000000000duet-0.2.9/dev/build000077500000000000000000000012601446013747400142770ustar00rootroot00000000000000#!/usr/bin/env sh if [ -d "dist" ]; then rm -rf dist/ fi for arg in "$@"; do case $arg in --pre) SRC_VERSION_LINE=$(cat "duet/_version.py" | tail -n 1) SRC_VERSION=$(echo $SRC_VERSION_LINE | cut -d'"' -f 2) if [[ ${SRC_VERSION} != *"dev" ]]; then echo "Version doesn't end in dev: ${SRC_VERSION_LINE}" >&2 exit 1 fi export DUET_PRE_RELEASE_VERSION="${SRC_VERSION}$(date "+%Y%m%d%H%M%S")" echo "pre-release version: ${DUET_PRE_RELEASE_VERSION}" ;; --upload) export UPLOAD="yes" ;; esac done python setup.py sdist bdist_wheel if [ -n "${UPLOAD}" ]; then echo "uploading..." twine upload dist/* fi duet-0.2.9/dev/check000077500000000000000000000001201446013747400142470ustar00rootroot00000000000000#!/usr/bin/env sh black duet/ isort duet/ mypy duet/ pylint duet/ pytest duet/ duet-0.2.9/dev/requirements.txt000066400000000000000000000001451446013747400165370ustar00rootroot00000000000000black == 22.3.0 isort == 5.7.* mypy == 0.931.* pylint == 2.10.* pytest == 6.2.* twine == 3.3.* wheel duet-0.2.9/duet/000077500000000000000000000000001446013747400134365ustar00rootroot00000000000000duet-0.2.9/duet/__init__.py000066400000000000000000000043071446013747400155530ustar00rootroot00000000000000# Copyright 2021 The Duet Authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # https://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Run asynchronous coroutines using Futures. Coroutines using async/await provide a way to write computations that can be paused and later resumed. This module provides a way to manage the execution of multiple such coroutines using Futures to provide concurrency. In other words, while one coroutine is waiting for a particular Future to complete, other coroutines can run. Other libraries for dealing with async/await, such as asyncio in the standard library or the third-party trio library, are focused on providing fully asynchronous I/O capabilities. Here we focus solely on managing coroutines and rely on Futures (themselves backed by either threads or a separate async I/O library) to provide concurrency. This module differs from those other libraries in two big ways: first, it is reentrant, meaning we can call `duet.run` recursively, which makes it much easier to refactor our code incrementally to be asynchronous; second, we can run the event loop manually one tick at a time, which makes it possible to implement things like the pmap function below which wraps async code into a generator interface. """ from concurrent.futures import CancelledError from duet._version import __version__ from duet.aitertools import aenumerate, aiter, AnyIterable, AsyncCollector, azip from duet.api import ( awaitable, awaitable_func, deadline_scope, LimitedScope, Limiter, new_scope, pmap, pmap_aiter, pmap_async, pstarmap, pstarmap_aiter, pstarmap_async, run, Scope, sleep, sync, timeout_scope, ) from duet.futuretools import AwaitableFuture, BufferedFuture, completed_future, failed_future duet-0.2.9/duet/_version.py000066400000000000000000000011341446013747400156330ustar00rootroot00000000000000# Copyright 2021 The Duet Authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # https://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. __version__ = "0.2.9" duet-0.2.9/duet/aitertools.py000066400000000000000000000067271446013747400162110ustar00rootroot00000000000000# Copyright 2021 The Duet Authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # https://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import collections from typing import ( AsyncIterable, AsyncIterator, Deque, Generic, Iterable, Optional, Tuple, TypeVar, Union, ) import duet.futuretools as futuretools T = TypeVar("T") AnyIterable = Union[Iterable[T], AsyncIterable[T]] async def aenumerate(iterable: AnyIterable[T], start: int = 0) -> AsyncIterator[Tuple[int, T]]: i = start async for value in aiter(iterable): yield (i, value) i += 1 async def aiter(iterable: AnyIterable[T]) -> AsyncIterator[T]: if isinstance(iterable, Iterable): for value in iterable: yield value else: async for value in iterable: yield value async def azip(*iterables: AnyIterable) -> AsyncIterator[Tuple]: iters = [aiter(iterable) for iterable in iterables] while True: values = [] for it in iters: try: value = await it.__anext__() values.append(value) except StopAsyncIteration: return yield tuple(values) class AsyncCollector(Generic[T]): """Allows async iteration over values dynamically added by the client. This class is useful for creating an asynchronous iterator that is "fed" by one process (the "producer") and iterated over by another process (the "consumer"). The producer calls `.add` repeatedly to add values to be iterated over, and then calls either `.done` or `.error` to stop the iteration or raise an error, respectively. The consumer can use `async for` or direct calls to `__anext__` to iterate over the produced values. """ def __init__(self): self._buffer: Deque[T] = collections.deque() self._waiter: Optional[futuretools.AwaitableFuture[None]] = None self._done: bool = False self._error: Optional[Exception] = None def add(self, value: T) -> None: if self._done: raise RuntimeError("already done.") self._buffer.append(value) if self._waiter: self._waiter.try_set_result(None) def done(self) -> None: if self._done: raise RuntimeError("already done.") self._done = True if self._waiter: self._waiter.try_set_result(None) def error(self, error: Exception) -> None: if self._done: raise RuntimeError("already done.") self._done = True self._error = error if self._waiter: self._waiter.try_set_result(None) def __aiter__(self) -> AsyncIterator[T]: return self async def __anext__(self) -> T: if not self._done and not self._buffer: self._waiter = futuretools.AwaitableFuture() await self._waiter self._waiter = None if self._buffer: return self._buffer.popleft() if self._error: raise self._error raise StopAsyncIteration() duet-0.2.9/duet/api.py000066400000000000000000000414651446013747400145730ustar00rootroot00000000000000# Copyright 2021 The Duet Authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # https://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import abc import collections import contextlib import functools import inspect from concurrent.futures import CancelledError from typing import ( Any, AsyncIterator, Awaitable, Callable, Deque, Dict, List, Optional, Set, Tuple, TypeVar, ) import duet.impl as impl from duet.aitertools import aenumerate, aiter, AnyIterable, AsyncCollector from duet.futuretools import AwaitableFuture T = TypeVar("T") U = TypeVar("U") def run(func: Callable[..., Awaitable[T]], *args, **kwds) -> T: """Run an async function to completion. Args: func: The async function to run. *args: Positional arguments to pass to func. **kwds: Keyword arguments to pass to func. Returns: The final result of the async function. """ scheduler = impl.Scheduler() scheduler.init_signals() try: task = scheduler.spawn(func(*args, **kwds)) try: while scheduler.active_tasks: scheduler.tick() except BaseException as exc: for task in scheduler.active_tasks: task.interrupt(None, exc) while scheduler.active_tasks: try: scheduler.tick() except BaseException: pass raise return task.result finally: scheduler.cleanup_signals() def sync(f: Callable[..., Awaitable[T]]) -> Callable[..., T]: """Decorator that adds a sync version of async function or method.""" if isinstance(f, classmethod): raise TypeError(f"duet.sync cannot be applied to classmethod {f.__func__}") sig = inspect.signature(f) first_arg = next(iter(sig.parameters), None) if first_arg == "self": # For class or instance methods, look up the method to call on the given # class or instance. This ensures that we call the right method even it # has been overridden in a subclass. To illustrate, consider: # # class Parent: # async def foo(self): ... # foo_sync = duet.sync(foo) # # class Child(Parent): # async def foo(self): ... # # A manual implementation of foo_sync would call duet.run(self.foo) so # that Child().foo_sync() would call Child.foo instead of Parent.foo. # We want the foo_sync wrapper to work the same way. But the wrapper # was called with Parent.foo only, so we must look up the appropriate # function by name at runtime, using getattr. @functools.wraps(f) def wrapped(self, *args, **kw): method = getattr(self, f.__name__) if inspect.ismethod(method) and id(method.__func__) == wrapped_id: return run(f, self, *args, **kw) return run(method, *args, **kw) wrapped_id = id(wrapped) if getattr(wrapped, "__isabstractmethod__", False): wrapped.__isabstractmethod__ = False # type: ignore[attr-defined] else: @functools.wraps(f) def wrapped(*args, **kw): return run(f, *args, **kw) return wrapped def awaitable(value): """Wraps a value to ensure that it is awaitable.""" if inspect.isawaitable(value): return value if AwaitableFuture.isfuture(value): return AwaitableFuture.wrap(value) return _awaitable_value(value) async def _awaitable_value(value): return value def awaitable_func(function): """Wraps a function to ensure that it returns an awaitable.""" if inspect.iscoroutinefunction(function): return function if inspect.isgeneratorfunction(function): raise TypeError( "cannot use generator function with duet; please convert to " f"async function instead: {function.__name__}" ) @functools.wraps(function) async def wrapped(*args, **kw): return await awaitable(function(*args, **kw)) return wrapped async def pmap_async( func: Callable[[T], Awaitable[U]], iterable: AnyIterable[T], limit: Optional[int] = None ) -> List[U]: """Apply an async function to every item in iterable. Args: func: Async function called for each element in iterable. iterable: Iterated over to produce values that are fed to func. limit: The maximum number of function calls to make concurrently. Returns: List of results of all function calls. """ async with new_scope() as scope: return [x async for x in pmap_aiter(scope, func, iterable, limit)] pmap = sync(pmap_async) async def pstarmap_async( func: Callable[..., Awaitable[U]], iterable: AnyIterable[Any], limit: Optional[int] = None ) -> List[U]: """Apply an async function to every tuple of args in iterable. Args: func: Async function called with each tuple of args in iterable. iterable: Iterated over to produce arg tuples that are fed to func. limit: The maximum number of function calls to make concurrently. Returns: List of results of all function calls. """ return await pmap_async(lambda args: func(*args), iterable, limit) pstarmap = sync(pstarmap_async) async def pmap_aiter( scope: "Scope", func: Callable[[T], Awaitable[U]], iterable: AnyIterable[T], limit: Optional[int] = None, ) -> AsyncIterator[U]: """Apply an async function to every item in iterable. Args: scope: Scope in which the returned async iterator must be used. func: Async function called for each element in iterable. iterable: Iterated over to produce values that are fed to func. limit: The maximum number of function calls to make concurrently. Returns: Asynchronous iterator that yields results in order as they become available. """ collector = AsyncCollector[Tuple[int, U]]() async def task(i, arg, slot): try: value = await func(arg) collector.add((i, value)) finally: slot.release() async def generate(): try: limiter = Limiter(limit) async with new_scope() as gen_scope: async for i, arg in aenumerate(iterable): slot = await limiter.acquire() gen_scope.spawn(task, i, arg, slot) except BaseException as e: collector.error(e) if isinstance(e, GeneratorExit): # Raise to avoid "coroutine ignored GeneratorExit" errors. raise else: collector.done() scope.spawn(generate) buffer: Dict[int, U] = {} next_idx = 0 async for i, value in collector: buffer[i] = value while next_idx in buffer: yield buffer.pop(next_idx) next_idx += 1 while buffer: yield buffer.pop(next_idx) next_idx += 1 def pstarmap_aiter( scope: "Scope", func: Callable[..., Awaitable[U]], iterable: AnyIterable[Any], limit: Optional[int] = None, ) -> AsyncIterator[U]: """Apply an async function to every tuple of args in iterable. Args: scope: Scope in which the returned async iterator must be used. func: Async function called with each tuple of args in iterable. iterable: Iterated over to produce arg tuples that are fed to func. limit: The maximum number of function calls to make concurrently. Returns: Asynchronous iterator that yields results in order as they become available. """ return pmap_aiter(scope, lambda args: func(*args), iterable, limit) async def sleep(time: float) -> None: """Sleeps for the given length of time in seconds.""" async with new_scope(timeout=time) as scope: try: await AwaitableFuture() except TimeoutError as e: if e is not scope._timeout_error: raise @contextlib.asynccontextmanager async def deadline_scope(deadline: float) -> AsyncIterator[None]: """Enter a scope that will exit when the deadline elapses. Args: deadline: Absolute time in epoch seconds when the scope should exit. """ async with new_scope(deadline=deadline): yield @contextlib.asynccontextmanager async def timeout_scope(timeout: float) -> AsyncIterator[None]: """Enter a scope that will exit when the timeout elapses. Args: timeout: Time in seconds from now when the scope should exit. """ async with new_scope(timeout=timeout): yield @contextlib.asynccontextmanager async def new_scope( *, deadline: Optional[float] = None, timeout: Optional[float] = None ) -> AsyncIterator["Scope"]: """Creates a scope in which asynchronous tasks can be launched. This is inspired by the concept of "nurseries" in trio: https://trio.readthedocs.io/en/latest/reference-core.html#nurseries-and-spawning We define the lifetime of a scope using an `async with` statement. Inside this block we can then spawn new asynchronous tasks which will run in the background, and the block will only exit when all spawned tasks are done. If an error is raised by the code in the block itself or by any of the spawned tasks, all other background tasks will be interrupted and the block will raise an error. Args: deadline: Absolute time in epoch seconds when the scope should exit. timeout: Time in seconds from now when the scope should exit. If both deadline and timeout are given, the actual deadline will be whichever one will elapse first. """ main_task = impl.current_task() scheduler = main_task.scheduler tasks: Set[impl.Task] = set() async def finish_tasks(): while True: await impl.any_ready(tasks) tasks.intersection_update(scheduler.active_tasks) if not tasks: break if timeout is not None: if deadline is None: deadline = scheduler.time() + timeout else: deadline = min(deadline, scheduler.time() + timeout) scope = Scope(main_task, scheduler, tasks) if deadline is not None: main_task.push_deadline(deadline, scope._timeout_error) try: yield scope await finish_tasks() except BaseException as exc: # Interrupt remaining tasks. for task in tasks: if not task.done: task.interrupt(main_task, RuntimeError("scope exited")) # Finish remaining tasks while ignoring further interrupts. main_task.interruptible = False await finish_tasks() main_task.interruptible = True # If interrupted, raise the underlying error but suppress the context # (the Interrupt itself) when displaying the traceback. if isinstance(exc, impl.Interrupt): exc = exc.error exc.__suppress_context__ = True raise exc finally: if deadline is not None: main_task.pop_deadline() class Scope: """Bounds the lifetime of async tasks spawned in the background.""" def __init__( self, main_task: impl.Task, scheduler: impl.Scheduler, tasks: Set[impl.Task] ) -> None: self._main_task = main_task self._scheduler = scheduler self._tasks = tasks self._timeout_error = TimeoutError() def cancel(self) -> None: self._main_task.interrupt(self._main_task, CancelledError()) def spawn(self, func: Callable[..., Awaitable[Any]], *args, **kwds) -> None: """Starts a background task that will run the given function.""" task = self._scheduler.spawn(self._run(func, *args, **kwds), main_task=self._main_task) self._tasks.add(task) async def _run(self, func: Callable[..., Awaitable[Any]], *args, **kwds) -> None: task = impl.current_task() try: await func(*args, **kwds) finally: self._tasks.discard(task) class Limiter: """Limits concurrent access to critical resources or code blocks. A Limiter is created with a fixed capacity (or None to indicate no limit), and can then be used with async with blocks to limit access, e.g.: limiter = Limiter(10) ... async with limiter: # At most 10 async calls can be in this section at once. ... In certain situations, it may not be possible to use async with blocks to demarcate the critical section. In that case, one can instead call acquire to get a "slot" that must be released later when done using the resource: limiter = Limiter(10) ... slot = await limiter.acquire() ... slot.release() """ def __init__(self, capacity: Optional[int]) -> None: self._capacity = capacity self._count = 0 self._waiters: Deque[AwaitableFuture[None]] = collections.deque() self._available_waiters: List[AwaitableFuture[None]] = [] def is_available(self) -> bool: """Returns True if the limiter is available, False otherwise.""" return self._capacity is None or self._count < self._capacity async def __aenter__(self) -> None: if not self.is_available(): f = AwaitableFuture[None]() self._waiters.append(f) await f self._count += 1 async def acquire(self) -> "Slot": await self.__aenter__() return Slot(self._release) async def __aexit__(self, exc_type, exc, tb) -> None: self._release() def _release(self): self._count -= 1 # Release the first waiter that has not yet been cancelled. while self._waiters: f = self._waiters.popleft() if f.try_set_result(None): break if self._available_waiters: for f in self._available_waiters: f.try_set_result(None) self._available_waiters.clear() async def available(self) -> None: """Wait until this limiter is available (i.e. not full to capacity). Note that this always yields control to the scheduler, even if the limiter is currently available, to ensure that throttled iterators do not race ahead of downstream work. """ f = AwaitableFuture[None]() if self.is_available(): f.set_result(None) else: self._available_waiters.append(f) await f async def throttle(self, iterable: AnyIterable[T]) -> AsyncIterator[T]: async for value in aiter(iterable): await self.available() yield value @property def capacity(self) -> Optional[int]: return self._capacity @capacity.setter def capacity(self, capacity: int) -> None: self._capacity = capacity class Slot: def __init__(self, release_func): self.release_func = release_func self.called = False def release(self): if self.called: raise Exception("Already released.") self.called = True self.release_func() class LimitedScope(abc.ABC): """Combined Scope (for running async iters) and Limiter (for throttling). Provides convenience methods for running coroutines in parallel within this scope while throttling to prevent iterators from running too far ahead. """ @property @abc.abstractmethod def scope(self) -> Scope: pass @property @abc.abstractmethod def limiter(self) -> Limiter: pass def spawn(self, func: Callable[..., Awaitable[Any]], *args, **kwds) -> None: """Starts a background task that will run the given function.""" self.scope.spawn(func, *args, **kwds) async def pmap_async( self, func: Callable[[T], Awaitable[U]], iterable: AnyIterable[T] ) -> List[U]: return [x async for x in self.pmap_aiter(func, iterable)] def pmap_aiter( self, func: Callable[[T], Awaitable[U]], iterable: AnyIterable[T] ) -> AsyncIterator[U]: return pmap_aiter(self.scope, func, self.limiter.throttle(iterable)) async def pstarmap_async( self, func: Callable[..., Awaitable[U]], iterable: AnyIterable[Any] ) -> List[U]: return [x async for x in self.pstarmap_aiter(func, iterable)] def pstarmap_aiter( self, func: Callable[..., Awaitable[U]], iterable: AnyIterable[Any] ) -> AsyncIterator[U]: return pstarmap_aiter(self.scope, func, self.limiter.throttle(iterable)) duet-0.2.9/duet/api_test.py000066400000000000000000000462471446013747400156350ustar00rootroot00000000000000# Copyright 2021 The Duet Authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # https://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import abc import concurrent.futures import contextlib import inspect import sys import time import traceback from typing import List, Tuple import pytest import duet import duet.impl as impl async def mul(a, b): await duet.completed_future(None) return a * b async def add(a, b): await duet.completed_future(None) return a + b class Fail(Exception): pass async def fail_after_await(): await duet.completed_future(None) raise Fail() async def fail_before_await(): raise Fail() fail_funcs = [fail_after_await, fail_before_await] class TestAwaitableFunc: def test_wrap_async_func(self): async def async_func(a, b): await duet.completed_future(None) return a + b assert duet.awaitable_func(async_func) is async_func assert duet.run(async_func, 1, 2) == 3 def test_wrap_sync_func(self): def sync_func(a, b): return a + b wrapped = duet.awaitable_func(sync_func) assert inspect.iscoroutinefunction(wrapped) assert duet.awaitable_func(wrapped) is wrapped # Don't double-wrap assert duet.run(wrapped, 1, 2) == 3 class TestRun: def test_future(self): def func(value): return duet.completed_future(value * 2) assert duet.run(func, 1) == 2 def test_function(self): async def func(value): value = await duet.completed_future(value * 2) return value * 3 assert duet.run(func, 1) == 2 * 3 def test_function_returning_none(self): side_effects = [] async def func(value): value = await duet.completed_future(value * 2) value = await duet.completed_future(value * 3) side_effects.append(value) assert duet.run(func, 1) is None assert side_effects == [2 * 3] # make sure func ran to completion def test_nested_functions(self): async def func(value): value = await sub_func(value * 2) return value * 3 async def sub_func(value): value = await duet.completed_future(value * 5) return value * 7 assert duet.run(func, 1) == 2 * 3 * 5 * 7 def test_nested_functions_returning_none(self): side_effects = [] async def func(value): value2 = await sub_func(value * 2) return value * 3, value2 async def sub_func(value): value = await duet.completed_future(value * 5) value = await duet.completed_future(value * 7) side_effects.append(value) assert duet.run(func, 1) == (3, None) assert side_effects == [2 * 5 * 7] def test_failed_future(self): async def func(value): try: await duet.failed_future(Exception()) return value * 2 except Exception: return value * 3 assert duet.run(func, 1) == 3 def test_failed_nested_generator(self): side_effects = [] async def func(value): try: await sub_func(value * 2) return value * 3 except Exception: return value * 5 async def sub_func(value): await duet.failed_future(Exception()) side_effects.append(value * 7) assert duet.run(func, 1) == 5 assert side_effects == [] @pytest.mark.parametrize("fail_func", fail_funcs) def test_failure_propagates(self, fail_func): with pytest.raises(Fail): duet.run(fail_func) class TestPmap: def test_ordering(self): """pmap results are in order, even if funcs finish out of order.""" finished = [] async def func(value): iterations = 10 - value for i in range(iterations): await duet.completed_future(i) finished.append(value) return value * 2 results = duet.pmap(func, range(10), limit=10) assert results == [i * 2 for i in range(10)] assert finished == list(reversed(range(10))) @pytest.mark.parametrize("limit", [3, 10, None]) def test_failure(self, limit): async def foo(i): if i == 7: raise ValueError("I do not like 7 :-(") return 7 * i with pytest.raises(ValueError): duet.pmap(foo, range(100), limit=limit) class TestPstarmap: def test_ordering(self): """pstarmap results are in order, even if funcs finish out of order.""" finished = [] async def func(a, b): value = 5 * a + b iterations = 10 - value for i in range(iterations): await duet.completed_future(i) finished.append(value) return value * 2 args_iter = ((a, b) for a in range(2) for b in range(5)) results = duet.pstarmap(func, args_iter, limit=10) assert results == [i * 2 for i in range(10)] assert finished == list(reversed(range(10))) class TestPmapAsync: @duet.sync async def test_ordering(self): """pmap_async results in order, even if funcs finish out of order.""" finished = [] async def func(value): iterations = 10 - value for i in range(iterations): await duet.completed_future(i) finished.append(value) return value * 2 results = await duet.pmap_async(func, range(10), limit=10) assert results == [i * 2 for i in range(10)] assert finished == list(reversed(range(10))) @duet.sync async def test_laziness(self): live = set() async def func(i): num_live = len(live) live.add(i) await duet.completed_future(i) live.remove(i) return num_live num_lives = await duet.pmap_async(func, range(100), limit=10) assert all(num_live <= 10 for num_live in num_lives) class TestPstarmapAsync: @duet.sync async def test_ordering(self): """pstarmap_async results in order, even if funcs finish out of order.""" finished = [] async def func(a, b): value = 5 * a + b iterations = 10 - value for i in range(iterations): await duet.completed_future(i) finished.append(value) return value * 2 args_iter = ((a, b) for a in range(2) for b in range(5)) results = await duet.pstarmap_async(func, args_iter, limit=10) assert results == [i * 2 for i in range(10)] assert finished == list(reversed(range(10))) class TestLimiter: @duet.sync async def test_ordering(self): """Check that waiting coroutines acquire limiter in order.""" limiter = duet.Limiter(1) acquired = [] async def func(i): async with limiter: acquired.append(i) await duet.completed_future(None) async with duet.new_scope() as scope: for i in range(10): scope.spawn(func, i) assert acquired == sorted(acquired) @duet.sync async def test_resize_capacity(self) -> None: """Check that resizing correctly lets running tasks complete.""" limiter = duet.Limiter(3) async with duet.new_scope() as scope: acqs: List[duet.AwaitableFuture[None]] = [] completed: List[int] = [] unlocks: List[duet.AwaitableFuture[None]] = [] dones: List[duet.AwaitableFuture[None]] = [] def spawn(i: int) -> None: """Spawn a new "controllable" async task. We can await limiter slot acquisition, and await when the task completes. """ acq = duet.AwaitableFuture[None]() done = duet.AwaitableFuture[None]() unlock = duet.AwaitableFuture[None]() acqs.append(acq) dones.append(done) unlocks.append(unlock) async def func(): async with limiter: acq.set_result(None) await unlock done.set_result(None) completed.append(i) scope.spawn(func) # Spawn three tasks. for i in range(3): spawn(i) # Wait for the last task to acquire the limiter. await acqs[-1] # Resize the limiter down to 2. limiter.capacity = 2 assert not limiter.is_available() # unlock one, and ensure the limiter is still unavailable. unlocks.pop(0).set_result(None) assert not limiter.is_available() await dones.pop(0) # unlock one more, which should free a slot. unlocks.pop(0).set_result(None) await dones.pop(0) assert limiter.is_available() # acquire again, this time hitting the limit of 2 again. spawn(3) await acqs[-1] assert not limiter.is_available() # complete all tasks. for f in unlocks: f.set_result(None) # Ensure that all spawned tasks completed in the right order. assert completed == list(range(4)) @duet.sync async def test_cancel(self) -> None: limiter = duet.Limiter(1) async def func( ready: duet.AwaitableFuture[duet.Scope], done: duet.AwaitableFuture[Tuple[bool, bool]] ) -> None: """Acquired and release the lock, and record what happened.""" async with duet.new_scope(timeout=1) as scope: ready.set_result(scope) acquired = False cancelled = False try: async with limiter: acquired = True except duet.CancelledError: cancelled = True done.set_result((acquired, cancelled)) async with contextlib.AsyncExitStack() as exit_stack: scope = await exit_stack.enter_async_context(duet.new_scope()) # first acquire the lock await exit_stack.enter_async_context(limiter) # now spawn two coroutines that will attempt to acquire the lock ready1 = duet.AwaitableFuture[duet.Scope]() done1 = duet.AwaitableFuture[Tuple[bool, bool]]() scope.spawn(func, ready1, done1) scope1 = await ready1 ready2 = duet.AwaitableFuture[duet.Scope]() done2 = duet.AwaitableFuture[Tuple[bool, bool]]() scope.spawn(func, ready2, done2) _scope2 = await ready2 # cancel the first waiting coroutine scope1.cancel() # ensure that first coroutine was cancelled and second coroutine got the lock. async with duet.new_scope(timeout=0.1): acquired1, cancelled1 = await done1 acquired2, cancelled2 = await done2 assert cancelled1 and not acquired1 assert acquired2 and not cancelled2 @duet.sync async def test_sleep(): start = time.time() await duet.sleep(0.5) assert abs((time.time() - start) - 0.5) < 0.3 @duet.sync async def test_sleep_with_timeout(): start = time.time() with pytest.raises(TimeoutError): async with duet.timeout_scope(0.5): await duet.sleep(10) assert abs((time.time() - start) - 0.5) < 0.3 @duet.sync async def test_repeated_sleep(): start = time.time() for _ in range(5): await duet.sleep(0.1) assert abs((time.time() - start) - 0.5) < 0.3 @duet.sync async def test_repeated_sleep_with_timeout(): start = time.time() with pytest.raises(TimeoutError): async with duet.timeout_scope(0.5): for _ in range(5): await duet.sleep(0.2) assert abs((time.time() - start) - 0.5) < 0.3 class TestScope: @duet.sync async def test_run_all(self): results = {} async def func(a, b): results[a, b] = await mul(a, b) async with duet.new_scope() as scope: for a in range(10): for b in range(10): scope.spawn(func, a, b) assert results == {(a, b): a * b for a in range(10) for b in range(10)} @pytest.mark.parametrize("fail_func", fail_funcs) @duet.sync async def test_failure_in_spawned_task(self, fail_func): after_fail = False with pytest.raises(Fail): async with duet.new_scope() as scope: for a in range(10): scope.spawn(mul, a, a) scope.spawn(fail_func) after_fail = True # This should still run. assert after_fail @duet.sync async def test_sync_failure_in_main_task(self): # pylint: disable=unreachable after_await = False with pytest.raises(Fail): async with duet.new_scope() as scope: scope.spawn(mul, 2, 3) raise Fail() after_await = True # This should not run. assert not after_await # pyline: enable=unreachable @duet.sync async def test_async_failure_in_main_task(self): after_await = False with pytest.raises(Fail): async with duet.new_scope() as scope: scope.spawn(mul, 2, 3) await duet.failed_future(Fail()) after_await = True # This should not run. assert not after_await def test_interrupt_not_included_in_stack_trace(self): async def func(): async with duet.new_scope() as scope: f = duet.AwaitableFuture() scope.spawn(lambda: f) f.set_exception(ValueError("oops!")) await duet.AwaitableFuture() with pytest.raises(ValueError, match="oops!") as exc_info: duet.run(func) stack_trace = "".join( traceback.format_exception(exc_info.type, exc_info.value, exc_info.tb) ) assert "Interrupt" not in stack_trace assert isinstance(exc_info.value.__context__, impl.Interrupt) assert exc_info.value.__suppress_context__ @duet.sync async def test_timeout(self): future = duet.AwaitableFuture() start = time.time() with pytest.raises(TimeoutError): async with duet.timeout_scope(0.5): await future assert abs((time.time() - start) - 0.5) < 0.2 assert future.cancelled() @duet.sync async def test_deadline(self): future = duet.AwaitableFuture() start = time.time() with pytest.raises(TimeoutError): async with duet.deadline_scope(time.time() + 0.5): await future assert abs((time.time() - start) - 0.5) < 0.2 assert future.cancelled() @duet.sync async def test_timeout_completes_within_timeout(self): with concurrent.futures.ThreadPoolExecutor() as executor: start = time.time() async with duet.timeout_scope(10): future = executor.submit(time.sleep, 0.5) await duet.awaitable(future) assert abs((time.time() - start) - 0.5) < 0.2 @duet.sync async def test_scope_timeout_cancels_all_subtasks(self): futures = [] task_timeouts = [] async def task(): try: f = duet.AwaitableFuture() futures.append(f) await f except TimeoutError: task_timeouts.append(True) else: task_timeouts.append(False) start = time.time() with pytest.raises(TimeoutError): async with duet.new_scope(timeout=0.5) as scope: scope.spawn(task) scope.spawn(task) await duet.AwaitableFuture() assert abs((time.time() - start) - 0.5) < 0.2 assert task_timeouts == [True, True] assert all(f.cancelled() for f in futures) @duet.sync async def test_cancel(self): task_future = duet.AwaitableFuture() scope_future = duet.AwaitableFuture() async def main_task(): with pytest.raises(duet.CancelledError): async with duet.new_scope() as scope: scope_future.set_result(scope) await task_future async def cancel_task(): scope = await scope_future scope.cancel() async with duet.new_scope() as scope: scope.spawn(main_task) scope.spawn(cancel_task) assert task_future.cancelled() @pytest.mark.skipif( sys.version_info >= (3, 8), reason="inapplicable for python 3.8+ (can be removed)" ) @duet.sync async def test_multiple_calls_to_future_set_result(): """This checks a scenario that caused deadlocks in earlier versions.""" async def set_results(*fs): for f in fs: await duet.completed_future(None) f.set_result(None) async with duet.new_scope() as scope: f0 = duet.AwaitableFuture() f1 = duet.AwaitableFuture() scope.spawn(set_results, f0) await f0 # Calling f0.set_result again should not mark this main task as ready. # If it does, then the duet scheduler will try to advance the task and # will block on getting the result of f1. This prevents the background # `set_results` task from advancing and actually calling f1.set_result, # so we would deadlock. scope.spawn(set_results, f0, f1) await f1 class TestSync: def test_sync_on_overridden_method(self): class Foo: async def foo_async(self, a: int) -> int: return a * 2 foo = duet.sync(foo_async) class Bar(Foo): async def foo_async(self, a: int) -> int: return a * 3 assert Foo().foo(5) == 10 assert Bar().foo(5) == 15 def test_sync_on_abstract_method(self): class Foo(abc.ABC): @abc.abstractmethod async def foo_async(self, a: int) -> int: pass foo = duet.sync(foo_async) class Bar(Foo): async def foo_async(self, a: int) -> int: return a * 3 with pytest.raises(TypeError, match="Can't instantiate abstract class Foo.*foo_async"): _ = Foo() assert Bar().foo(5) == 15 def test_sync_on_classmethod(self): with pytest.raises(TypeError, match="duet.sync cannot be applied to classmethod"): class _Foo: @classmethod async def foo_async(cls, a: int) -> int: return a * 2 foo = duet.sync(foo_async) duet-0.2.9/duet/futuretools.py000066400000000000000000000144061446013747400164100ustar00rootroot00000000000000# Copyright 2021 The Duet Authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # https://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import threading from concurrent.futures import Future from typing import Any, Callable, Generator, Generic, Optional, Tuple, Type, TypeVar try: from typing import Protocol except ImportError: from typing_extensions import Protocol # type: ignore[misc] try: import grpc FutureClasses: Tuple[Type, ...] = (Future, grpc.Future) except ImportError: FutureClasses = (Future,) T = TypeVar("T") class FutureLike(Protocol[T]): def result(self) -> T: ... def exception(self) -> Optional[BaseException]: ... def add_done_callback(self, fn: Callable[["FutureLike[T]"], Any]) -> None: ... def cancel(self) -> bool: ... def cancelled(self) -> bool: ... class AwaitableFuture(Future, Generic[T]): """A Future that can be awaited.""" # This is an internal variable in the Future class. # We add an annotation here so mypy will let us use it. _condition: threading.Condition @staticmethod def isfuture(value: Any) -> bool: return isinstance(value, FutureClasses) @staticmethod def wrap(future: FutureLike[T]) -> "AwaitableFuture[T]": """Creates an awaitable future that wraps the given source future.""" awaitable = AwaitableFuture[T]() def cancel(awaitable_future: Future): if awaitable_future.cancelled(): future.cancel() def callback(future: FutureLike[T]): if future.cancelled(): awaitable.cancel() else: error = future.exception() if error is None: awaitable.try_set_result(future.result()) else: awaitable.try_set_exception(error) awaitable.add_done_callback(cancel) future.add_done_callback(callback) return awaitable def __await__(self) -> Generator["AwaitableFuture[T]", None, T]: yield self return self.result() def try_set_result(self, result: T) -> bool: """Sets the result on this future if not already done. Returns: True if we set the result, False if the future was already done. """ with self._condition: if self.done(): return False self.set_result(result) return True def try_set_exception(self, exception: Optional[BaseException]) -> bool: """Sets an exception on this future if not already done. Returns: True if we set the exception, False if the future was already done. """ with self._condition: if self.done(): return False self.set_exception(exception) return True class BufferedFuture(AwaitableFuture): """A future whose async operation may be buffered until flush is called. Calling the flush method starts the asynchronous operation associated with this future, if it has not been started already. By default, calling result or exception will also call flush so that the async operation will start and we do not deadlock waiting for a result. """ def flush(self): pass def result(self, timeout=None): self.flush() return super().result(timeout) def exception(self, timeout=None): self.flush() return super().exception(timeout) class BufferGroup: """A group of buffered futures that need to be flushed.""" def __init__(self, latch=False): """ Args: latch: If True, we set a flag the first time the group is flushed; we then immediately flush any futures added after that point. If False, the default, we store all added futures in a list and flush them the next time the group is flushed, regardless of whether the group has been flushed before. """ self._latch = latch self._flushed = False self._futures = [] def add(self, future): if not isinstance(future, BufferedFuture): return if self._latch and self._flushed: future.flush() else: self._futures.append(future) def flush(self): for f in self._futures: f.flush() self._futures.clear() if self._latch: self._flushed = True class FutureList(BufferedFuture): """A Future that waits for a list of other Futures.""" def __init__(self, futures): super().__init__() if not len(futures): self.set_result([]) return self._results = [None] * len(futures) self._outstanding = len(futures) self._lock = threading.Lock() self._buffer = BufferGroup() for i, f in enumerate(futures): self._buffer.add(f) f.add_done_callback(lambda f, idx=i: self._handle_result(f, idx)) def _handle_result(self, future, index): if self.done(): return error = future.exception() if error is not None: self.try_set_exception(error) return result = future.result() with self._lock: self._results[index] = result self._outstanding -= 1 if not self._outstanding: self.try_set_result(self._results) def flush(self): self._buffer.flush() def completed_future(data: T) -> AwaitableFuture[T]: """Return a future with the given data as its result.""" f = AwaitableFuture[T]() f.set_result(data) return f def failed_future(error: BaseException) -> AwaitableFuture[Any]: """Return a future that will fail with the given error.""" f = AwaitableFuture[Any]() f.set_exception(error) return f duet-0.2.9/duet/futuretools_test.py000066400000000000000000000034401446013747400174430ustar00rootroot00000000000000# Copyright 2021 The Duet Authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # https://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. from concurrent.futures import Future from typing import Any, Callable, Optional import pytest import duet try: import grpc except ImportError: grpc = None def test_awaitable_future(): assert isinstance(duet.awaitable(Future()), duet.AwaitableFuture) def test_awaitable_future_propagates_cancellation(): f = Future() awaitable = duet.AwaitableFuture.wrap(f) awaitable.cancel() assert f.cancelled() @pytest.mark.skipif(grpc is None, reason="only run if grpc is installed") def test_awaitable_grpc_future(): class ConcreteGrpcFuture(grpc.Future): def cancel(self) -> bool: return True def cancelled(self) -> bool: return True def running(self) -> bool: return True def done(self) -> bool: return True def result(self, timeout: Optional[int] = None) -> Any: return 1234 def exception(self, timeout=None) -> Optional[BaseException]: return None def add_done_callback(self, fn: Callable[[Any], Any]) -> None: pass def traceback(self, timeout=None): pass assert isinstance(duet.awaitable(ConcreteGrpcFuture()), duet.AwaitableFuture) duet-0.2.9/duet/impl.py000066400000000000000000000371671446013747400147670ustar00rootroot00000000000000# Copyright 2021 The Duet Authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # https://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Internal implementation details for duet.""" import enum import functools import heapq import itertools import signal import threading import time from concurrent.futures import Future from contextvars import ContextVar from typing import ( Any, Awaitable, Callable, cast, Coroutine, Generic, Iterator, List, Optional, Set, TypeVar, ) import duet.futuretools as futuretools T = TypeVar("T") class Interrupt(BaseException): def __init__(self, task, error): self.task = task self.error = error class TaskState(enum.Enum): WAITING = 0 SUCCEEDED = 1 FAILED = 2 class TaskStateError(Exception): def __init__(self, state: TaskState, expected_state: TaskState) -> None: self.state = state self.expected_state = expected_state super().__init__(f"state: {state}, expected: {expected_state}") # Sentinel local variable name that we insert into coroutines. # This allows us to detect whether a task is running when we get Ctrl-C. LOCALS_TASK_SCHEDULER = "__duet_task_scheduler__" class Task(Generic[T]): def __init__( self, awaitable: Awaitable[T], scheduler: "Scheduler", main_task: Optional["Task"] ) -> None: self.scheduler = scheduler self.main_task = main_task self._state = TaskState.WAITING self._future: Optional[Future] = None self._ready_future = futuretools.AwaitableFuture[None]() self._ready_future.set_result(None) # Ready to advance. self.interruptible = True self._interrupt: Optional[Interrupt] = None self._result: Optional[T] = None self._error: Optional[Exception] = None self._deadlines: List[DeadlineEntry] = [] if main_task and main_task.deadline_entry is not None: entry = main_task.deadline_entry self.push_deadline(deadline=entry.deadline, timeout_error=entry.timeout_error) self._generator = awaitable.__await__() # Returns coroutine generator. if isinstance(awaitable, Coroutine): awaitable.cr_frame.f_locals.setdefault(LOCALS_TASK_SCHEDULER, scheduler) def _check_state(self, expected_state: TaskState) -> None: if self._state != expected_state: raise TaskStateError(self._state, expected_state) @property def future(self) -> Optional[Future]: self._check_state(TaskState.WAITING) return self._future @property def result(self) -> T: self._check_state(TaskState.SUCCEEDED) return cast(T, self._result) @property def done(self) -> bool: return self._state == TaskState.SUCCEEDED or self._state == TaskState.FAILED def add_ready_callback(self, callback: Callable[["Task"], Any]) -> None: self._check_state(TaskState.WAITING) self._ready_future.add_done_callback(lambda _: callback(self)) def advance(self): if self.done: return if self._state == TaskState.WAITING: self._ready_future.result() token = _current_task.set(self) try: if self._interrupt: interrupt = self._interrupt self._interrupt = None if interrupt.task is self: error = interrupt.error else: error = interrupt f = self._generator.throw(error) else: f = next(self._generator) except StopIteration as e: self._result = e.value self._state = TaskState.SUCCEEDED return except BaseException as error: self._error = error self._state = TaskState.FAILED if self.main_task: self.main_task.interrupt(self, error) return else: raise else: if not isinstance(f, Future): raise TypeError(f"expected Future, got {type(f)}: {f}") ready_future = futuretools.AwaitableFuture() f.add_done_callback(lambda _: ready_future.try_set_result(None)) self._future = f self._ready_future = ready_future self._state = TaskState.WAITING finally: _current_task.reset(token) def push_deadline(self, deadline: float, timeout_error: TimeoutError) -> None: if self._deadlines: entry = self._deadlines[-1] if entry.deadline < deadline: deadline = entry.deadline timeout_error = entry.timeout_error entry = DeadlineEntry(self, deadline, timeout_error) self.scheduler.add_deadline(entry) self._deadlines.append(entry) def pop_deadline(self) -> None: entry = self._deadlines.pop(-1) entry.valid = False @property def deadline_entry(self) -> Optional["DeadlineEntry"]: return self._deadlines[-1] if self._deadlines else None def interrupt(self, task, error): if self.done or not self.interruptible or self._interrupt: return self._interrupt = Interrupt(task, error) self._ready_future.try_set_result(None) if self._future: self._future.cancel() def close(self): self._generator.close() self.scheduler = None self.main_task = None _current_task: ContextVar[Task] = ContextVar("current_task") def current_task() -> Task: """Gets the currently-running duet task. This must be called from within a running async function, or else it will raise a RuntimeError. """ try: return _current_task.get() except LookupError: raise RuntimeError("Can only be called from an async function.") def current_scheduler() -> "Scheduler": """Gets the currently-running duet scheduler. This must be called from within a running async function, or else it will raise a RuntimeError. """ return current_task().scheduler def any_ready(tasks: Set[Task]) -> futuretools.AwaitableFuture[None]: """Returns a Future that will fire when any of the given tasks is ready.""" if not tasks or any(task.done for task in tasks): return futuretools.completed_future(None) f = futuretools.AwaitableFuture[None]() for task in tasks: task.add_ready_callback(lambda _: f.try_set_result(None)) return f class ReadySet: """Container for an ordered set of tasks that are ready to advance.""" def __init__(self): self._cond = threading.Condition() self._buffer = futuretools.BufferGroup() self._tasks: List[Task] = [] self._task_set: Set[Task] = set() def register(self, task: Task) -> None: """Registers task to be added to this set when it is ready.""" self._buffer.add(task.future) task.add_ready_callback(self._add) def _add(self, task: Task) -> None: """Adds the given task to the ready set, if it is not already there.""" with self._cond: if task not in self._task_set: self._task_set.add(task) self._tasks.append(task) self._cond.notify() def get_all(self, timeout: Optional[float] = None) -> List[Task]: """Gets all ready tasks and clears the ready set. If no tasks are ready yet, we flush buffered futures to notify them that they should proceed, and then block until one or more tasks become ready. Raises: ValueError if timeout is < 0 or > threading.TIMEOUT_MAX """ if timeout is not None and (timeout < 0 or timeout > threading.TIMEOUT_MAX): raise ValueError(f"invalid timeout: {timeout}") with self._cond: if self._tasks: return self._pop_tasks() # Flush buffered futures to ensure we make progress. Note that we must # release the condition lock before flushing to avoid a deadlock if # buffered futures complete and trigger a call to self._add. self._buffer.flush() with self._cond: if not self._tasks: if not self._cond.wait(timeout): raise TimeoutError() return self._pop_tasks() def _pop_tasks(self) -> List[Task]: tasks = self._tasks self._tasks = [] self._task_set.clear() return tasks def interrupt(self) -> None: with self._cond: self._cond.notify() @functools.total_ordering class DeadlineEntry: """A entry for one Deadline in the Scheduler's priority queue. This follows the implementation notes in the stdlib heapq docs: https://docs.python.org/3/library/heapq.html#priority-queue-implementation-notes Attributes: task: The task associated with this deadline. deadline: Absolute time when the deadline will elapse. count: Monotonically-increasing counter to preserve creation order when comparing entries with the same deadline. valid: Flag indicating whether the deadline is still valid. If the task exits its scope before the deadline elapses, we mark the deadline as invalid but leave it in the scheduler's priority queue since removal would require an O(n) scan. The scheduler ignores invalid deadlines when they elapse. """ _counter = itertools.count() def __init__(self, task: Task, deadline: float, timeout_error: TimeoutError): self.task = task self.deadline = deadline self.timeout_error = timeout_error self.count = next(self._counter) self._cmp_val = (deadline, self.count) self.valid = True def __eq__(self, other: Any) -> bool: if not isinstance(other, DeadlineEntry): return NotImplemented return self._cmp_val == other._cmp_val def __lt__(self, other: Any) -> bool: if not isinstance(other, DeadlineEntry): return NotImplemented return self._cmp_val < other._cmp_val def __repr__(self) -> str: return f"DeadlineEntry({self.task}, {self.deadline}, {self.count})" class Scheduler: def __init__(self) -> None: self.active_tasks: Set[Task] = set() self._ready_tasks = ReadySet() self._prev_signal: Optional[Callable] = None self._interrupted = False self._deadlines: List[DeadlineEntry] = [] def spawn(self, awaitable: Awaitable[Any], main_task: Optional[Task] = None) -> Task: """Spawns a new Task to run an awaitable in this Scheduler. Note that the task will not be advanced until the next scheduler tick. Also, note that this function is safe to call from sync code (such as duet.run) or async code (such as within a scope). Args: func: The async function to run. *args: Args for func. **kwds: Keyword args for func. Returns: A Task to run the given awaitable. """ task = Task(awaitable, scheduler=self, main_task=main_task) self.active_tasks.add(task) self._ready_tasks.register(task) return task def time(self) -> float: return time.time() def add_deadline(self, entry: DeadlineEntry) -> None: heapq.heappush(self._deadlines, entry) def get_next_deadline(self) -> Optional[float]: while self._deadlines: if not self._deadlines[0].valid: heapq.heappop(self._deadlines) continue return self._deadlines[0].deadline return None def get_deadline_entries(self, deadline: float) -> Iterator[DeadlineEntry]: while self._deadlines and self._deadlines[0].deadline <= deadline: entry = heapq.heappop(self._deadlines) if entry.valid: yield entry def tick(self): """Runs the scheduler ahead by one tick. This waits for at least one active task to complete, then advances all ready tasks and sets up a new future to be notified later by tasks that are still active (or yet to be spawned). Raises a RuntimeError if there are no currently active tasks. """ if not self.active_tasks: raise RuntimeError("tick called with no active tasks") if self._interrupted: task = next(iter(self.active_tasks)) task.interrupt(task, KeyboardInterrupt) self._interrupted = False deadline = self.get_next_deadline() if deadline is None: ready_tasks = self._ready_tasks.get_all(None) else: ready_tasks: List[Task] = [] for i in itertools.count(): timeout = deadline - self.time() if i and timeout < 0: break try: ready_tasks = self._ready_tasks.get_all( max(0, min(timeout, threading.TIMEOUT_MAX)) ) break except TimeoutError: pass if not ready_tasks: for entry in self.get_deadline_entries(deadline): entry.task.interrupt(entry.task, entry.timeout_error) ready_tasks = self._ready_tasks.get_all(None) for task in ready_tasks: try: task.advance() finally: if task.done: task.close() self.active_tasks.discard(task) else: self._ready_tasks.register(task) def _interrupt(self, signum: int, frame: Optional[Any]) -> None: """Interrupt signal handler used while this scheduler is running. This is inspired by trio's interrupt handling, described here: https://vorpus.org/blog/control-c-handling-in-python-and-trio/ If the interrupted frame is inside a running task, which we detect by looking for a special local variable inserted into the task coroutine, we simply raise a KeyboardInterrupt as usual. Otherwise we set a flag which will get checked on the next tick() and cause a task to be interrupted. One important difference from trio is that duet is reentrant, so when detecting whether we are in a task we have to check whether the task's scheduler is self. If the interrupted frame is running in a task of a different scheduler, that should not raise KeyboardInterrupt directly. """ if self._in_task(frame): raise KeyboardInterrupt else: self._interrupted = True self._ready_tasks.interrupt() def _in_task(self, frame) -> bool: while frame is not None: if frame.f_locals.get(LOCALS_TASK_SCHEDULER, None) is self: return True frame = frame.f_back return False def init_signals(self): if ( threading.current_thread() == threading.main_thread() and signal.getsignal(signal.SIGINT) == signal.default_int_handler ): self._prev_signal = signal.signal(signal.SIGINT, self._interrupt) def cleanup_signals(self): if self._prev_signal: signal.signal(signal.SIGINT, self._prev_signal) duet-0.2.9/duet/impl_test.py000066400000000000000000000046661446013747400160240ustar00rootroot00000000000000# Copyright 2021 The Duet Authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # https://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import duet import duet.impl as impl class CompleteOnFlush(duet.BufferedFuture): def __init__(self): super().__init__() self.flushed = False def flush(self): self.flushed = True self.set_result(None) def make_task(future: duet.AwaitableFuture) -> impl.Task: """Make a task from the given future. We advance the task once, which just starts the generator and yields the future itself. """ task = impl.Task(future, impl.Scheduler(), None) task.advance() return task class TestReadySet: def test_get_all_returns_all_ready_tasks(self): task1 = make_task(duet.completed_future(None)) task2 = make_task(duet.completed_future(None)) task3 = make_task(duet.AwaitableFuture()) task4 = make_task(duet.completed_future(None)) rs = impl.ReadySet() rs.register(task1) rs.register(task2) rs.register(task3) rs.register(task4) tasks = rs.get_all() assert tasks == [task1, task2, task4] def test_task_added_at_most_once(self): task = make_task(duet.completed_future(None)) rs = impl.ReadySet() rs.register(task) rs.register(task) tasks = rs.get_all() assert tasks == [task] def test_futures_flushed_if_no_task_ready(self): future = CompleteOnFlush() task = make_task(future) rs = impl.ReadySet() rs.register(task) tasks = rs.get_all() assert tasks == [task] assert future.flushed def test_futures_not_flushed_if_tasks_ready(self): future = CompleteOnFlush() task1 = make_task(future) task2 = make_task(duet.completed_future(None)) rs = impl.ReadySet() rs.register(task1) rs.register(task2) tasks = rs.get_all() assert tasks == [task2] assert not future.flushed duet-0.2.9/duet/py.typed000066400000000000000000000000731446013747400151350ustar00rootroot00000000000000# Marker file for PEP 561. This package uses inline types. duet-0.2.9/duet/typing.py000066400000000000000000000063411446013747400153260ustar00rootroot00000000000000# Copyright 2021 The Duet Authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # https://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Mypy plugin to provide better typechecking of duet functions. For more information about mypy plugins see: https://mypy.readthedocs.io/en/stable/extending_mypy.html#extending-mypy-using-plugins """ from typing import Callable, List, Optional from mypy.plugin import FunctionContext, Plugin from mypy.types import CallableType, get_proper_type, Instance, Overloaded, Type def duet_sync_callback(ctx: FunctionContext) -> Type: """Callback to provide an accurate signature for duet.sync. The duet.sync function wraps an async callable in a synchronous wrapper: def sync(f: Callable[..., Awaitable[T]]) -> Callable[..., T]: This plugin basically tells mypy that the two ellipses are exactly the same, that is, that the new synchronous callable accepts exactly the same args as the original function. This allows for precise typechecking of calls to functions wrapped by duet.sync. """ func_type = get_proper_type(ctx.arg_types[0][0]) if not isinstance(func_type, (CallableType, Overloaded)): ctx.api.msg.fail(f"expected Callable[..., Awaitable[T]], got {func_type}", ctx.context) return ctx.default_return_type if isinstance(func_type, CallableType): return modify_callable(func_type, ctx) # func_type is overloaded overloaded_callables: List[CallableType] = [] for ft in func_type.items: overload_type = modify_callable(ft, ctx) if not isinstance(overload_type, CallableType): ctx.api.msg.fail( f"expected overloaded type to be callable, got {overload_type}", ctx.context ) return ctx.default_return_type overloaded_callables.append(overload_type) return Overloaded(overloaded_callables) def modify_callable(func_type: CallableType, ctx: FunctionContext) -> Type: # Note that the return type of an async function is Coroutine[Any, Any, T], # which is a subtype of Awaitable[T]. See: # https://mypy.readthedocs.io/en/stable/more_types.html#typing-async-await ret_type = get_proper_type(func_type.ret_type) if not (isinstance(ret_type, Instance) and ret_type.type.name == "Coroutine"): if not func_type.implicit: ctx.api.msg.fail(f"expected return type Awaitable[T], got {ret_type}", ctx.context) return ctx.default_return_type result_type = ret_type.args[-1] return func_type.copy_modified(ret_type=result_type) class DuetPlugin(Plugin): def get_function_hook(self, fullname: str) -> Optional[Callable[[FunctionContext], Type]]: if fullname == "duet.api.sync": return duet_sync_callback return None def plugin(version: str): return DuetPlugin duet-0.2.9/pyproject.toml000066400000000000000000000003671446013747400154170ustar00rootroot00000000000000[tool.black] line-length = 100 target_version = ['py37', 'py38'] skip-magic-trailing-comma = true [tool.isort] profile = 'black' # Sort alphabetically, irrespective of case. order_by_type = false line_length = 100 remove_redundant_aliases = true duet-0.2.9/requirements.txt000066400000000000000000000000651446013747400157620ustar00rootroot00000000000000typing-extensions >= 3.10.0; python_version <= '3.7' duet-0.2.9/setup.cfg000066400000000000000000000000421446013747400143120ustar00rootroot00000000000000[metadata] license_file = LICENSE duet-0.2.9/setup.py000066400000000000000000000045231446013747400142130ustar00rootroot00000000000000# Copyright 2021 The Duet Authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # https://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import os import pathlib from setuptools import setup # This reads the __version__ variable from duet/_version.py __version__ = "" exec(pathlib.Path("duet/_version.py").read_text()) name = "duet" description = "A simple future-based async library for python." # README file as long_description. long_description = pathlib.Path("README.md").read_text() # If DUET_PRE_RELEASE_VERSION is set then we update the version to this value. # It is assumed that it ends with one of `.devN`, `.aN`, `.bN`, `.rcN` and hence # it will be a pre-release version on PyPi. See # https://packaging.python.org/guides/distributing-packages-using-setuptools/#pre-release-versioning # for more details. if "DUET_PRE_RELEASE_VERSION" in os.environ: __version__ = os.environ["DUET_PRE_RELEASE_VERSION"] long_description = "\n\n".join( [ "This is a development version of Duet and may be unstable.", "For the latest stable release see https://pypi.org/project/duet/.", long_description, ] ) # Sanity check assert __version__, "Version string cannot be empty" # Read requirements requirements = [line.strip() for line in open("requirements.txt").readlines()] dev_requirements = [line.strip() for line in open("dev/requirements.txt").readlines()] setup( name=name, version=__version__, url="http://github.com/google/duet", author="The Duet Authors", author_email="maffoo@google.com", python_requires=">=3.9.0", install_requires=requirements, extras_require={ "dev_env": dev_requirements, }, license="Apache 2", description=description, long_description=long_description, long_description_content_type='text/markdown', packages=["duet"], package_data={ "duet": ["py.typed"], }, )