pax_global_header00006660000000000000000000000064144161251010014505gustar00rootroot0000000000000052 comment=a480af9e4c8c4b8286a4fd5da9263dcf87ad5c13 simonw-asyncinject-a480af9/000077500000000000000000000000001441612510100157475ustar00rootroot00000000000000simonw-asyncinject-a480af9/.github/000077500000000000000000000000001441612510100173075ustar00rootroot00000000000000simonw-asyncinject-a480af9/.github/workflows/000077500000000000000000000000001441612510100213445ustar00rootroot00000000000000simonw-asyncinject-a480af9/.github/workflows/publish.yml000066400000000000000000000026651441612510100235460ustar00rootroot00000000000000name: Publish Python Package on: release: types: [created] jobs: test: runs-on: ubuntu-latest strategy: matrix: python-version: ["3.7", "3.8", "3.9", "3.10"] steps: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} - uses: actions/cache@v2 name: Configure pip caching with: path: ~/.cache/pip key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }} restore-keys: | ${{ runner.os }}-pip- - name: Install dependencies run: | pip install -e '.[test]' - name: Run tests run: | pytest deploy: runs-on: ubuntu-latest needs: [test] steps: - uses: actions/checkout@v2 - name: Set up Python uses: actions/setup-python@v2 with: python-version: "3.10" - uses: actions/cache@v2 name: Configure pip caching with: path: ~/.cache/pip key: ${{ runner.os }}-publish-pip-${{ hashFiles('**/setup.py') }} restore-keys: | ${{ runner.os }}-publish-pip- - name: Install dependencies run: | pip install setuptools wheel twine build - name: Publish env: TWINE_USERNAME: __token__ TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }} run: | python -m build twine upload dist/* simonw-asyncinject-a480af9/.github/workflows/test.yml000066400000000000000000000013121441612510100230430ustar00rootroot00000000000000name: Test on: [push, pull_request] jobs: test: runs-on: ubuntu-latest strategy: matrix: python-version: ["3.7", "3.8", "3.9", "3.10"] steps: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} - uses: actions/cache@v2 name: Configure pip caching with: path: ~/.cache/pip key: ${{ runner.os }}-pip-${{ hashFiles('**/setup.py') }} restore-keys: | ${{ runner.os }}-pip- - name: Install dependencies run: | pip install -e '.[test]' - name: Run tests run: | pytest simonw-asyncinject-a480af9/.gitignore000066400000000000000000000001411441612510100177330ustar00rootroot00000000000000.venv __pycache__/ *.py[cod] *$py.class venv .eggs .pytest_cache *.egg-info .DS_Store dist build simonw-asyncinject-a480af9/LICENSE000066400000000000000000000261351441612510100167630ustar00rootroot00000000000000 Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright [yyyy] [name of copyright owner] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. simonw-asyncinject-a480af9/README.md000066400000000000000000000140471441612510100172340ustar00rootroot00000000000000# asyncinject [![PyPI](https://img.shields.io/pypi/v/asyncinject.svg)](https://pypi.org/project/asyncinject/) [![Changelog](https://img.shields.io/github/v/release/simonw/asyncinject?include_prereleases&label=changelog)](https://github.com/simonw/asyncinject/releases) [![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://github.com/simonw/asyncinject/blob/main/LICENSE) Run async workflows using pytest-fixtures-style dependency injection ## Installation Install this library using `pip`: $ pip install asyncinject ## Usage This library is inspired by [pytest fixtures](https://docs.pytest.org/en/6.2.x/fixture.html). The idea is to simplify executing parallel `asyncio` operations by allowing them to be defined using a collection of functions, where the function arguments represent dependent functions that need to be executed first. The library can then create and execute a plan for executing the required functions in parallel in the most efficient sequence possible. Here's an example, using the [httpx](https://www.python-httpx.org/) HTTP library. ```python from asyncinject import Registry import httpx async def get(url): async with httpx.AsyncClient() as client: return (await client.get(url)).text async def example(): return await get("http://www.example.com/") async def simonwillison(): return await get("https://simonwillison.net/search/?tag=empty") async def both(example, simonwillison): return example + "\n\n" + simonwillison registry = Registry(example, simonwillison, both) combined = await registry.resolve(both) print(combined) ``` If you run this in `ipython` or `python -m asyncio` (to enable top-level await in the console) you will see output that combines HTML from both of those pages. The HTTP requests to `www.example.com` and `simonwillison.net` will be performed in parallel. The library notices that `both()` takes two arguments which are the names of other registered `async def` functions, and will construct an execution plan that executes those two functions in parallel, then passes their results to the `both()` method. ### Registry.from_dict() Passing a list of functions to the `Registry` constructor will register each function under their introspected function name, using `fn.__name__`. You can set explicit names instead using a dictionary: ```python registry = Registry.from_dict({ "example": example, "simonwillison": simonwillison, "both": both }) ``` Those string names will be used to match parameters, so each function will need to accept parameters named after the keys used in that dictionary. ### Registering additional functions Functions that are registered can be regular functions or `async def` functions. In addition to registering functions by passing them to the constructor, you can also add them to a registry using the `.register()` method: ```python async def another(): return "another" registry.register(another) ``` To register them with a name other than the name of the function, pass the `name=` argument: ```python async def another(): return "another 2" registry.register(another, name="another_2") ``` ### Resolving an unregistered function You don't need to register the final function that you pass to `.resolve()` - if you pass an unregistered function, the library will introspect the function's parameters and resolve them directly. This works with both regular and async functions: ```python async def one(): return 1 async def two(): return 2 registry = Registry(one, two) # async def works here too: def three(one, two): return one + two print(await registry.resolve(three)) # Prints 3 ``` ### Parameters are passed through Your dependent functions can require keyword arguments which have been passed to the `.resolve()` call: ```python async def get_param_1(param1): return await get(param1) async def get_param_2(param2): return await get(param2) async def both(get_param_1, get_param_2): return get_param_1 + "\n\n" + get_param_2 combined = await Registry(get_param_1, get_param_2, both).resolve( both, param1 = "http://www.example.com/", param2 = "https://simonwillison.net/search/?tag=empty" ) print(combined) ``` ### Parameters with default values are ignored You can opt a parameter out of the dependency injection mechanism by assigning it a default value: ```python async def go(calc1, x=5): return calc1 + x async def calc1(): return 5 print(await Registry(calc1, go).resolve(go)) # Prints 10 ``` ### Tracking with a timer You can pass a `timer=` callable to the `Registry` constructor to gather timing information about executed tasks.. Your function should take three positional arguments: - `name` - the name of the function that is being timed - `start` - the time that it started executing, using `time.perf_counter()` ([perf_counter() docs](https://docs.python.org/3/library/time.html#time.perf_counter)) - `end` - the time that it finished executing You can use `print` here too: ```python combined = await Registry( get_param_1, get_param_2, both, timer=print ).resolve( both, param1 = "http://www.example.com/", param2 = "https://simonwillison.net/search/?tag=empty" ) ``` This will output: ``` get_param_1 436633.584580685 436633.797921747 get_param_2 436633.641832699 436634.196364347 both 436634.196570217 436634.196575639 ``` ### Turning off parallel execution By default, functions that can run in parallel according to the execution plan will run in parallel using `asyncio.gather()`. You can disable this parallel exection by passing `parallel=False` to the `Registry` constructor, or by setting `registry.parallel = False` after the registry object has been created. This is mainly useful for benchmarking the difference between parallel and serial execution for your project. ## Development To contribute to this library, first checkout the code. Then create a new virtual environment: cd asyncinject python -m venv venv source venv/bin/activate Now install the dependencies and test dependencies: pip install -e '.[test]' To run the tests: pytest simonw-asyncinject-a480af9/asyncinject/000077500000000000000000000000001441612510100202615ustar00rootroot00000000000000simonw-asyncinject-a480af9/asyncinject/__init__.py000066400000000000000000000110741441612510100223750ustar00rootroot00000000000000import inspect import time try: import graphlib except ImportError: from . import vendored_graphlib as graphlib import asyncio class Registry: def __init__(self, *fns, parallel=True, timer=None): self._registry = {} self._graph = None self._reversed = None self.parallel = parallel self.timer = timer for fn in fns: self.register(fn) @classmethod def from_dict(cls, d, parallel=True, timer=None): instance = cls(parallel=parallel, timer=timer) for key, fn in d.items(): instance.register(fn, name=key) return instance def register(self, fn, *, name=None): self._registry[name or fn.__name__] = fn # Clear caches: self._graph = None self._reversed = None def _make_time_logger(self, awaitable): async def inner(): start = time.perf_counter() result = await awaitable end = time.perf_counter() self.timer(awaitable.__name__, start, end) return result return inner() @property def graph(self): if self._graph is None: self._graph = { key: set(inspect.signature(fn).parameters.keys()) for key, fn in self._registry.items() } return self._graph @property def reversed(self): if self._reversed is None: self._reversed = dict(reversed(pair) for pair in self._registry.items()) return self._reversed async def resolve(self, fn, **kwargs): if not isinstance(fn, str): # It's a fn - is it a registered one? name = self.reversed.get(fn) if name is None: # Special case - since it is not registered we need to # introspect its parameters here and use resolve_multi params = inspect.signature(fn).parameters.keys() to_resolve = {p for p in params if p not in kwargs} resolved = await self.resolve_multi(to_resolve, results=kwargs) result = fn(**{param: resolved[param] for param in params}) if asyncio.iscoroutine(result): result = await result return result else: name = fn results = await self.resolve_multi([name], results=kwargs) return results[name] def _plan(self, names, results=None): if results is None: results = {} ts = graphlib.TopologicalSorter() to_do = set(names) done = set(results.keys()) while to_do: item = to_do.pop() dependencies = self.graph.get(item) or set() ts.add(item, *dependencies) done.add(item) # Add any not-done dependencies to the queue to_do.update({k for k in dependencies if k not in done}) return ts def _get_awaitable(self, name, results): fn = self._registry[name] kwargs = {k: v for k, v in results.items() if k in self.graph[name]} awaitable_fn = fn if not asyncio.iscoroutinefunction(fn): async def _awaitable(*args, **kwargs): return fn(*args, **kwargs) _awaitable.__name__ = fn.__name__ awaitable_fn = _awaitable aw = awaitable_fn(**kwargs) if self.timer: aw = self._make_time_logger(aw) return aw async def _execute_sequential(self, results, ts): for name in ts.static_order(): if name not in self._registry: continue results[name] = await self._get_awaitable(name, results) async def _execute_parallel(self, results, ts): ts.prepare() tasks = [] def schedule(): for name in ts.get_ready(): if name not in self._registry: ts.done(name) continue tasks.append(asyncio.create_task(worker(name))) async def worker(name): res = await self._get_awaitable(name, results) results[name] = res ts.done(name) schedule() schedule() while tasks: await asyncio.gather(*[tasks.pop() for _ in range(len(tasks))]) async def resolve_multi(self, names, results=None): if results is None: results = {} ts = self._plan(names, results) if self.parallel: await self._execute_parallel(results, ts) else: await self._execute_sequential(results, ts) return results simonw-asyncinject-a480af9/asyncinject/vendored_graphlib.py000066400000000000000000000231211441612510100243100ustar00rootroot00000000000000# Vendored from https://raw.githubusercontent.com/python/cpython/3.10/Lib/graphlib.py # Modified to work on Python 3.6 (I removed := operator) # License: https://github.com/python/cpython/blob/main/LICENSE __all__ = ["TopologicalSorter", "CycleError"] _NODE_OUT = -1 _NODE_DONE = -2 class _NodeInfo: __slots__ = "node", "npredecessors", "successors" def __init__(self, node): # The node this class is augmenting. self.node = node # Number of predecessors, generally >= 0. When this value falls to 0, # and is returned by get_ready(), this is set to _NODE_OUT and when the # node is marked done by a call to done(), set to _NODE_DONE. self.npredecessors = 0 # List of successor nodes. The list can contain duplicated elements as # long as they're all reflected in the successor's npredecessors attribute. self.successors = [] class CycleError(ValueError): """Subclass of ValueError raised by TopologicalSorter.prepare if cycles exist in the working graph. If multiple cycles exist, only one undefined choice among them will be reported and included in the exception. The detected cycle can be accessed via the second element in the *args* attribute of the exception instance and consists in a list of nodes, such that each node is, in the graph, an immediate predecessor of the next node in the list. In the reported list, the first and the last node will be the same, to make it clear that it is cyclic. """ pass class TopologicalSorter: """Provides functionality to topologically sort a graph of hashable nodes""" def __init__(self, graph=None): self._node2info = {} self._ready_nodes = None self._npassedout = 0 self._nfinished = 0 if graph is not None: for node, predecessors in graph.items(): self.add(node, *predecessors) def _get_nodeinfo(self, node): result = self._node2info.get(node) if result is None: self._node2info[node] = result = _NodeInfo(node) return result def add(self, node, *predecessors): """Add a new node and its predecessors to the graph. Both the *node* and all elements in *predecessors* must be hashable. If called multiple times with the same node argument, the set of dependencies will be the union of all dependencies passed in. It is possible to add a node with no dependencies (*predecessors* is not provided) as well as provide a dependency twice. If a node that has not been provided before is included among *predecessors* it will be automatically added to the graph with no predecessors of its own. Raises ValueError if called after "prepare". """ if self._ready_nodes is not None: raise ValueError("Nodes cannot be added after a call to prepare()") # Create the node -> predecessor edges nodeinfo = self._get_nodeinfo(node) nodeinfo.npredecessors += len(predecessors) # Create the predecessor -> node edges for pred in predecessors: pred_info = self._get_nodeinfo(pred) pred_info.successors.append(node) def prepare(self): """Mark the graph as finished and check for cycles in the graph. If any cycle is detected, "CycleError" will be raised, but "get_ready" can still be used to obtain as many nodes as possible until cycles block more progress. After a call to this function, the graph cannot be modified and therefore no more nodes can be added using "add". """ if self._ready_nodes is not None: raise ValueError("cannot prepare() more than once") self._ready_nodes = [ i.node for i in self._node2info.values() if i.npredecessors == 0 ] # ready_nodes is set before we look for cycles on purpose: # if the user wants to catch the CycleError, that's fine, # they can continue using the instance to grab as many # nodes as possible before cycles block more progress cycle = self._find_cycle() if cycle: raise CycleError(f"nodes are in a cycle", cycle) def get_ready(self): """Return a tuple of all the nodes that are ready. Initially it returns all nodes with no predecessors; once those are marked as processed by calling "done", further calls will return all new nodes that have all their predecessors already processed. Once no more progress can be made, empty tuples are returned. Raises ValueError if called without calling "prepare" previously. """ if self._ready_nodes is None: raise ValueError("prepare() must be called first") # Get the nodes that are ready and mark them result = tuple(self._ready_nodes) n2i = self._node2info for node in result: n2i[node].npredecessors = _NODE_OUT # Clean the list of nodes that are ready and update # the counter of nodes that we have returned. self._ready_nodes.clear() self._npassedout += len(result) return result def is_active(self): """Return ``True`` if more progress can be made and ``False`` otherwise. Progress can be made if cycles do not block the resolution and either there are still nodes ready that haven't yet been returned by "get_ready" or the number of nodes marked "done" is less than the number that have been returned by "get_ready". Raises ValueError if called without calling "prepare" previously. """ if self._ready_nodes is None: raise ValueError("prepare() must be called first") return self._nfinished < self._npassedout or bool(self._ready_nodes) def __bool__(self): return self.is_active() def done(self, *nodes): """Marks a set of nodes returned by "get_ready" as processed. This method unblocks any successor of each node in *nodes* for being returned in the future by a call to "get_ready". Raises :exec:`ValueError` if any node in *nodes* has already been marked as processed by a previous call to this method, if a node was not added to the graph by using "add" or if called without calling "prepare" previously or if node has not yet been returned by "get_ready". """ if self._ready_nodes is None: raise ValueError("prepare() must be called first") n2i = self._node2info for node in nodes: # Check if we know about this node (it was added previously using add() nodeinfo = n2i.get(node) if nodeinfo is None: raise ValueError(f"node {node!r} was not added using add()") # If the node has not being returned (marked as ready) previously, inform the user. stat = nodeinfo.npredecessors if stat != _NODE_OUT: if stat >= 0: raise ValueError( f"node {node!r} was not passed out (still not ready)" ) elif stat == _NODE_DONE: raise ValueError(f"node {node!r} was already marked done") else: assert False, f"node {node!r}: unknown status {stat}" # Mark the node as processed nodeinfo.npredecessors = _NODE_DONE # Go to all the successors and reduce the number of predecessors, collecting all the ones # that are ready to be returned in the next get_ready() call. for successor in nodeinfo.successors: successor_info = n2i[successor] successor_info.npredecessors -= 1 if successor_info.npredecessors == 0: self._ready_nodes.append(successor) self._nfinished += 1 def _find_cycle(self): n2i = self._node2info stack = [] itstack = [] seen = set() node2stacki = {} for node in n2i: if node in seen: continue while True: if node in seen: # If we have seen already the node and is in the # current stack we have found a cycle. if node in node2stacki: return stack[node2stacki[node] :] + [node] # else go on to get next successor else: seen.add(node) itstack.append(iter(n2i[node].successors).__next__) node2stacki[node] = len(stack) stack.append(node) # Backtrack to the topmost stack entry with # at least another successor. while stack: try: node = itstack[-1]() break except StopIteration: del node2stacki[stack.pop()] itstack.pop() else: break return None def static_order(self): """Returns an iterable of nodes in a topological order. The particular order that is returned may depend on the specific order in which the items were inserted in the graph. Using this method does not require to call "prepare" or "done". If any cycle is detected, :exc:`CycleError` will be raised. """ self.prepare() while self.is_active(): node_group = self.get_ready() yield from node_group self.done(*node_group) simonw-asyncinject-a480af9/pytest.ini000066400000000000000000000000371441612510100200000ustar00rootroot00000000000000[pytest] asyncio_mode = strict simonw-asyncinject-a480af9/setup.py000066400000000000000000000017151441612510100174650ustar00rootroot00000000000000from setuptools import setup import os VERSION = "0.6" def get_long_description(): with open( os.path.join(os.path.dirname(os.path.abspath(__file__)), "README.md"), encoding="utf8", ) as fp: return fp.read() setup( name="asyncinject", description="Run async workflows using pytest-fixtures-style dependency injection", long_description=get_long_description(), long_description_content_type="text/markdown", author="Simon Willison", url="https://github.com/simonw/asyncinject", project_urls={ "Issues": "https://github.com/simonw/asyncinject/issues", "CI": "https://github.com/simonw/asyncinject/actions", "Changelog": "https://github.com/simonw/asyncinject/releases", }, license="Apache License, Version 2.0", version=VERSION, packages=["asyncinject"], install_requires=[], extras_require={"test": ["pytest", "pytest-asyncio"]}, python_requires=">=3.7", ) simonw-asyncinject-a480af9/tests/000077500000000000000000000000001441612510100171115ustar00rootroot00000000000000simonw-asyncinject-a480af9/tests/test_asyncinject.py000066400000000000000000000141311441612510100230340ustar00rootroot00000000000000import asyncio import pytest from asyncinject import Registry from random import random import time @pytest.fixture def complex_registry(): async def log(): return [] async def d(log): await asyncio.sleep(0.1 + random() * 0.5) log.append("d") async def c(log): await asyncio.sleep(0.1 + random() * 0.5) log.append("c") async def b(log, c, d): log.append("b") async def a(log, b, c): log.append("a") async def go(log, a): log.append("go") return log return Registry(log, d, c, b, a, go) @pytest.mark.asyncio async def test_complex(complex_registry): result = await complex_registry.resolve("go") # 'c' should only be called once assert tuple(result) in ( # c and d could happen in either order ("c", "d", "b", "a", "go"), ("d", "c", "b", "a", "go"), ) @pytest.mark.asyncio async def test_with_parameters(): async def go(calc1, calc2, param1): return param1 + calc1 + calc2 async def calc1(): return 5 async def calc2(): return 6 registry = Registry(go, calc1, calc2) result = await registry.resolve(go, param1=4) assert result == 15 # Should throw an error if that parameter is missing with pytest.raises(TypeError) as e: result = await registry.resolve(go) assert "go() missing 1 required positional" in e.args[0] @pytest.mark.asyncio async def test_parameters_passed_through(): async def go(calc1, calc2, param1): return calc1 + calc2 async def calc1(): return 5 async def calc2(param1): return 6 + param1 registry = Registry(go, calc1, calc2) result = await registry.resolve(go, param1=1) assert result == 12 @pytest.mark.asyncio async def test_ignore_default_parameters(): async def go(calc1, x=5): return calc1 + x async def calc1(): return 5 registry = Registry(go, calc1) result = await registry.resolve(go) assert result == 10 @pytest.mark.asyncio async def test_timer(complex_registry): collected = [] complex_registry.timer = lambda name, start, end: collected.append( (name, start, end) ) await complex_registry.resolve("go") assert len(collected) == 6 names = [c[0] for c in collected] starts = [c[1] for c in collected] ends = [c[2] for c in collected] assert all(isinstance(n, float) for n in starts) assert all(isinstance(n, float) for n in ends) assert names[0] == "log" assert names[5] == "go" assert sorted(names[1:5]) == ["a", "b", "c", "d"] @pytest.mark.asyncio async def test_parallel(complex_registry): collected = [] complex_registry.timer = lambda name, start, end: collected.append( (name, start, end) ) # Run it once in parallel=True mode await complex_registry.resolve("go") parallel_timings = {c[0]: (c[1], c[2]) for c in collected} # 'c' and 'd' should have started within 0.05s c_start, d_start = parallel_timings["c"][0], parallel_timings["d"][0] assert abs(c_start - d_start) < 0.05 # And again in parallel=False mode collected.clear() complex_registry.parallel = False await complex_registry.resolve("go") serial_timings = {c[0]: (c[1], c[2]) for c in collected} # 'c' and 'd' should have started at least 0.1s apart c_start_serial, d_start_serial = serial_timings["c"][0], serial_timings["d"][0] assert abs(c_start_serial - d_start_serial) > 0.1 @pytest.mark.asyncio async def test_optimal_concurrency(): # https://github.com/simonw/asyncinject/issues/10 async def a(): await asyncio.sleep(0.1) async def b(): await asyncio.sleep(0.2) async def c(a): await asyncio.sleep(0.1) async def d(b, c): pass registry = Registry(a, b, c, d) start = time.perf_counter() await registry.resolve(d) end = time.perf_counter() # Should have taken ~0.2s assert 0.18 < (end - start) < 0.22 @pytest.mark.asyncio @pytest.mark.parametrize("use_async", (True, False)) async def test_resolve_unregistered_function(use_async): # https://github.com/simonw/asyncinject/issues/13 async def one(): return 1 async def two(): return 2 registry = Registry(one, two) async def three_async(one, two): return one + two def three_not_async(one, two): return one + two fn = three_async if use_async else three_not_async result = await registry.resolve(fn) assert result == 3 # Test that passing parameters works too result2 = await registry.resolve(fn, one=2) assert result2 == 4 @pytest.mark.asyncio async def test_register(): registry = Registry() # Mix in a non-async function too: def one(): return "one" async def two_(): return "two" async def three(one, two): return one + two registry.register(one) # Should raise an error if you don't use name= with pytest.raises(TypeError): registry.register(two_, "two") registry.register(two_, name="two") result = await registry.resolve(three) assert result == "onetwo" @pytest.mark.asyncio @pytest.mark.parametrize("parallel", (True, False)) async def test_just_sync_functions(parallel): def one(): return 1 def two(): return 2 def three(one, two): return one + two timed = [] registry = Registry( one, two, three, parallel=parallel, timer=lambda *args: timed.append(args) ) result = await registry.resolve(three) assert result == 3 assert {t[0] for t in timed} == {"two", "one", "three"} @pytest.mark.asyncio @pytest.mark.parametrize("use_string_name", (True, False)) async def test_registry_from_dict(use_string_name): async def _one(): return 1 async def _two(): return 2 async def _three(one, two): return one + two registry = Registry.from_dict({"one": _one, "two": _two, "three": _three}) if use_string_name: result = await registry.resolve("three") else: result = await registry.resolve(_three) assert result == 3