pax_global_header00006660000000000000000000000064143212625070014514gustar00rootroot0000000000000052 comment=ba2169891f5b32a5c59e48ca185dd8e68e44ded7 .github/000077500000000000000000000000001432126250700124205ustar00rootroot00000000000000.github/workflows/000077500000000000000000000000001432126250700144555ustar00rootroot00000000000000.github/workflows/publish-to-pypi.yml000066400000000000000000000014531432126250700202500ustar00rootroot00000000000000name: Publish package to PyPI on: push: branches: [master] release: types: [ released ] jobs: build-n-publish: name: Build and publish Python package to PyPI runs-on: ubuntu-18.04 steps: - uses: actions/checkout@master - name: Set up Python 3.7 uses: actions/setup-python@v1 with: python-version: 3.7 - name: Install pypa/build run: >- python3 -m pip install --user --upgrade setuptools wheel - name: Build a binary wheel and a source tarball run: >- python3 setup.py sdist bdist_wheel - name: Publish distribution 📦 to PyPI if: startsWith(github.ref, 'refs/tags') uses: pypa/gh-action-pypi-publish@master with: password: ${{ secrets.PYPI_API_TOKEN }}.github/workflows/python-package.yml000066400000000000000000000021341432126250700201120ustar00rootroot00000000000000name: Test package on: push: branches: [ master ] pull_request: branches: [ master ] jobs: build: runs-on: ubuntu-latest strategy: matrix: python-version: [3.7, 3.8, 3.9] steps: - uses: actions/checkout@master - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@master with: python-version: ${{ matrix.python-version }} - name: Install dependencies run: | python -m pip install --upgrade pip pip install -r requirements_ci.txt - name: Lint with pre-commit config run: pre-commit run --all-files - name: Test with mypy run: mypy -m aiohttp_retry - name: Test with pytest env: PYTHONPATH: . run: pytest --cov=./ --cov-report=xml --asyncio-mode=auto - name: Upload coverage to Codecov uses: codecov/codecov-action@v1 with: token: ${{ secrets.CODECOV_TOKEN }} files: ./coverage.xml flags: pytest env_vars: OS,PYTHON name: codecov-umbrella fail_ci_if_error: true verbose: true .gitignore000066400000000000000000000001461432126250700130510ustar00rootroot00000000000000.idea/ venv/ env/ __pycache__/ .mypy_cache/ .pytest_cache/ aiohttp_retry.egg-info/ build/ dist/ var/ .pre-commit-config.yaml000066400000000000000000000011131432126250700153350ustar00rootroot00000000000000repos: - repo: https://github.com/pycqa/isort rev: 5.10.1 hooks: - id: isort files: 'aiohttp_retry/.*' stages: - commit - push - id: isort files: 'tests/.*' stages: - commit - push - repo: https://github.com/pre-commit/mirrors-autopep8 rev: 'v1.6.0' # Use the sha / tag you want to point at hooks: - id: autopep8 files: 'aiohttp_retry' stages: - commit - push - id: autopep8 files: 'tests' stages: - commit - push AUTHORS000066400000000000000000000000461432126250700121300ustar00rootroot00000000000000Dmitry Inyutin LICENSE000066400000000000000000000020771432126250700120730ustar00rootroot00000000000000The MIT License (MIT) Copyright (c) 2020 aiohttp_retry Authors Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. README.md000066400000000000000000000201771432126250700123460ustar00rootroot00000000000000# Simple aiohttp retry client [![codecov](https://codecov.io/gh/inyutin/aiohttp_retry/branch/master/graph/badge.svg?token=ZWGAXSF1SP)](https://codecov.io/gh/inyutin/aiohttp_retry) Python 3.7 or higher. **Install**: `pip install aiohttp-retry`. [!["Buy Me A Coffee"](https://www.buymeacoffee.com/assets/img/custom_images/orange_img.png)](https://www.buymeacoffee.com/inyutin) ### Breaking API changes - Everything between [2.7.0 - 2.8.3) is yanked. There is a bug with evaluate_response_callback, it led to infinite retries - 2.8.0 is incorrect and yanked. https://github.com/inyutin/aiohttp_retry/issues/79 - Since 2.5.6 this is a new parameter in ```get_timeout``` func called "response". If you have defined your own ```RetryOptions```, you should add this param into it. Issue about this: https://github.com/inyutin/aiohttp_retry/issues/59 ### Examples of usage: ```python from aiohttp_retry import RetryClient, ExponentialRetry async def main(): retry_options = ExponentialRetry(attempts=1) retry_client = RetryClient(raise_for_status=False, retry_options=retry_options) async with retry_client.get('https://ya.ru') as response: print(response.status) await retry_client.close() ``` ```python from aiohttp import ClientSession from aiohttp_retry import RetryClient async def main(): client_session = ClientSession() retry_client = RetryClient(client_session=client_session) async with retry_client.get('https://ya.ru') as response: print(response.status) await client_session.close() ``` ```python from aiohttp_retry import RetryClient, RandomRetry async def main(): retry_options = RandomRetry(attempts=1) retry_client = RetryClient(raise_for_status=False, retry_options=retry_options) response = await retry_client.get('/ping') print(response.status) await retry_client.close() ``` ```python from aiohttp_retry import RetryClient async def main(): async with RetryClient() as client: async with client.get('https://ya.ru') as response: print(response.status) ``` You can change parameters between attempts by passing multiple requests params: ```python from aiohttp_retry import RetryClient, RequestParams, ExponentialRetry async def main(): retry_client = RetryClient(raise_for_status=False) async with retry_client.requests( params_list=[ RequestParams( method='GET', url='https://ya.ru', ), RequestParams( method='GET', url='https://ya.ru', headers={'some_header': 'some_value'}, ), ] ) as response: print(response.status) await retry_client.close() ``` You can also add some logic, F.E. logging, on failures by using trace mechanic. ```python import logging import sys from types import SimpleNamespace from aiohttp import ClientSession, TraceConfig, TraceRequestStartParams from aiohttp_retry import RetryClient, ExponentialRetry handler = logging.StreamHandler(sys.stdout) logging.basicConfig(handlers=[handler]) logger = logging.getLogger(__name__) retry_options = ExponentialRetry(attempts=2) async def on_request_start( session: ClientSession, trace_config_ctx: SimpleNamespace, params: TraceRequestStartParams, ) -> None: current_attempt = trace_config_ctx.trace_request_ctx['current_attempt'] if retry_options.attempts <= current_attempt: logger.warning('Wow! We are in last attempt') async def main(): trace_config = TraceConfig() trace_config.on_request_start.append(on_request_start) retry_client = RetryClient(retry_options=retry_options, trace_configs=[trace_config]) response = await retry_client.get('https://httpstat.us/503', ssl=False) print(response.status) await retry_client.close() ``` Look tests for more examples. \ **Be aware: last request returns as it is.** **If the last request ended with exception, that this exception will be raised from RetryClient request** ### Documentation `RetryClient` takes the same arguments as ClientSession[[docs](https://docs.aiohttp.org/en/stable/client_reference.html)] \ `RetryClient` has methods: - request - get - options - head - post - put - patch - put - delete They are same as for `ClientSession`, but take one possible additional argument: ```python class RetryOptionsBase: def __init__( self, attempts: int = 3, # How many times we should retry statuses: Optional[Iterable[int]] = None, # On which statuses we should retry exceptions: Optional[Iterable[Type[Exception]]] = None, # On which exceptions we should retry retry_all_server_errors: bool = True, # If should retry all 500 errors or not # a callback that will run on response to decide if retry evaluate_response_callback: Optional[EvaluateResponseCallbackType] = None, ): ... @abc.abstractmethod def get_timeout(self, attempt: int, response: Optional[Response] = None) -> float: raise NotImplementedError ``` You can specify `RetryOptions` both for `RetryClient` and it's methods. `RetryOptions` in methods override `RetryOptions` defined in `RetryClient` constructor. **Important**: by default all 5xx responses are retried + statuses you specified as ```statuses``` param If you will pass ```retry_all_server_errors=False``` than you can manually set what 5xx errors to retry. You can define your own timeouts logic or use: - ```ExponentialRetry``` with exponential backoff - ```RandomRetry``` for random backoff - ```ListRetry``` with backoff you predefine by list - ```FibonacciRetry``` with backoff that looks like fibonacci sequence - ```JitterRetry``` exponential retry with a bit of randomness **Important**: you can proceed server response as an parameter for calculating next timeout. However this response can be None, server didn't make a response or you have set up ```raise_for_status=True``` Look here for an example: https://github.com/inyutin/aiohttp_retry/issues/59 Additionally, you can specify ```evaluate_response_callback```. It receive a ```ClientResponse``` and decide to retry or not by returning a bool. It can be useful, if server API sometimes response with malformed data. #### Request Trace Context `RetryClient` add *current attempt number* to `request_trace_ctx` (see examples, for more info see [aiohttp doc](https://docs.aiohttp.org/en/stable/client_advanced.html#aiohttp-client-tracing)). ### Change parameters between retries `RetryClient` also has a method called `requests`. This method should be used if you want to make requests with different params. ```python @dataclass class RequestParams: method: str url: _RAW_URL_TYPE trace_request_ctx: Optional[Dict[str, Any]] = None kwargs: Optional[Dict[str, Any]] = None ``` ```python def requests( self, params_list: List[RequestParams], retry_options: Optional[RetryOptionsBase] = None, raise_for_status: Optional[bool] = None, ) -> _RequestContext: ``` You can find an example of usage above or in tests. But basically `RequestParams` is a structure to define params for `ClientSession.request` func. `method`, `url`, `headers` `trace_request_ctx` defined outside kwargs, because they are popular. There is also an old way to change URL between retries by specifying ```url``` as list of urls. Example: ```python from aiohttp_retry import RetryClient retry_client = RetryClient() async with retry_client.get(url=['/internal_error', '/ping']) as response: text = await response.text() assert response.status == 200 assert text == 'Ok!' await retry_client.close() ``` In this example we request ```/interval_error```, fail and then successfully request ```/ping```. If you specify less urls than ```attempts``` number in ```RetryOptions```, ```RetryClient``` will request last url at last attempts. This means that in example above we would request ```/ping``` once again in case of failure. ### Types `aiohttp_retry` is a typed project. It should be fully compatablie with mypy. It also introduce one special type: ``` ClientType = Union[ClientSession, RetryClient] ``` This type can be imported by ```from aiohttp_retry.types import ClientType``` aiohttp_retry/000077500000000000000000000000001432126250700137555ustar00rootroot00000000000000aiohttp_retry/__init__.py000066400000000000000000000001331432126250700160630ustar00rootroot00000000000000from .client import * # noqa: F401, F403 from .retry_options import * # noqa: F401, F403 aiohttp_retry/client.py000066400000000000000000000275271432126250700156220ustar00rootroot00000000000000import asyncio import logging import sys from abc import abstractmethod from dataclasses import dataclass from types import TracebackType from typing import ( Any, Awaitable, Callable, Dict, Generator, List, Optional, Tuple, Type, Union, ) from aiohttp import ClientResponse, ClientSession, hdrs from aiohttp.typedefs import StrOrURL from yarl import URL as YARL_URL from .retry_options import ExponentialRetry, RetryOptionsBase if sys.version_info >= (3, 8): from typing import Protocol else: from typing_extensions import Protocol class _Logger(Protocol): """ _Logger defines which methods logger object should have """ @abstractmethod def debug(self, msg: str, *args: Any, **kwargs: Any) -> None: pass @abstractmethod def warning(self, msg: str, *args: Any, **kwargs: Any) -> None: pass @abstractmethod def exception(self, msg: str, *args: Any, **kwargs: Any) -> None: pass # url itself or list of urls for changing between retries _RAW_URL_TYPE = Union[StrOrURL, YARL_URL] _URL_TYPE = Union[_RAW_URL_TYPE, List[_RAW_URL_TYPE], Tuple[_RAW_URL_TYPE, ...]] _LoggerType = Union[_Logger, logging.Logger] RequestFunc = Callable[..., Awaitable[ClientResponse]] @dataclass class RequestParams: method: str url: _RAW_URL_TYPE headers: Optional[Dict[str, Any]] = None trace_request_ctx: Optional[Dict[str, Any]] = None kwargs: Optional[Dict[str, Any]] = None class _RequestContext: def __init__( self, request_func: RequestFunc, params_list: List[RequestParams], logger: _LoggerType, retry_options: RetryOptionsBase, raise_for_status: bool = False, ) -> None: assert len(params_list) > 0 self._request_func = request_func self._params_list = params_list self._logger = logger self._retry_options = retry_options self._raise_for_status = raise_for_status self._response: Optional[ClientResponse] = None async def _is_skip_retry(self, current_attempt: int, response: ClientResponse) -> bool: if current_attempt == self._retry_options.attempts: return True if response.status >= 500 and self._retry_options.retry_all_server_errors: return False if response.status in self._retry_options.statuses: return False if self._retry_options.evaluate_response_callback is None: return True return await self._retry_options.evaluate_response_callback(response) async def _do_request(self) -> ClientResponse: current_attempt = 0 while True: self._logger.debug(f"Attempt {current_attempt+1} out of {self._retry_options.attempts}") current_attempt += 1 try: try: params = self._params_list[current_attempt - 1] except IndexError: params = self._params_list[-1] response: ClientResponse = await self._request_func( params.method, params.url, headers=params.headers, trace_request_ctx={ 'current_attempt': current_attempt, **(params.trace_request_ctx or {}), }, **(params.kwargs or {}), ) debug_message = f"Retrying after response code: {response.status}" skip_retry = await self._is_skip_retry(current_attempt, response) if skip_retry: if self._raise_for_status: response.raise_for_status() self._response = response return self._response else: retry_wait = self._retry_options.get_timeout(attempt=current_attempt, response=response) except Exception as e: if current_attempt >= self._retry_options.attempts: raise e is_exc_valid = any([isinstance(e, exc) for exc in self._retry_options.exceptions]) if not is_exc_valid: raise e debug_message = f"Retrying after exception: {repr(e)}" retry_wait = self._retry_options.get_timeout(attempt=current_attempt, response=None) self._logger.debug(debug_message) await asyncio.sleep(retry_wait) def __await__(self) -> Generator[Any, None, ClientResponse]: return self.__aenter__().__await__() async def __aenter__(self) -> ClientResponse: return await self._do_request() async def __aexit__( self, exc_type: Optional[Type[BaseException]], exc_val: Optional[BaseException], exc_tb: Optional[TracebackType], ) -> None: if self._response is not None: if not self._response.closed: self._response.close() def _url_to_urls(url: _URL_TYPE) -> Tuple[StrOrURL, ...]: if isinstance(url, str) or isinstance(url, YARL_URL): return (url,) if isinstance(url, list): urls = tuple(url) elif isinstance(url, tuple): urls = url else: raise ValueError("you can pass url only by str or list/tuple") if len(urls) == 0: raise ValueError("you can pass url by str or list/tuple with attempts count size") return urls class RetryClient: def __init__( self, client_session: Optional[ClientSession] = None, logger: Optional[_LoggerType] = None, retry_options: Optional[RetryOptionsBase] = None, raise_for_status: bool = False, *args: Any, **kwargs: Any, ) -> None: if client_session is not None: client = client_session closed = None else: client = ClientSession(*args, **kwargs) closed = False self._client = client self._closed = closed self._logger: _LoggerType = logger or logging.getLogger("aiohttp_retry") self._retry_options: RetryOptionsBase = retry_options or ExponentialRetry() self._raise_for_status = raise_for_status @property def retry_options(self) -> RetryOptionsBase: return self._retry_options def requests( self, params_list: List[RequestParams], retry_options: Optional[RetryOptionsBase] = None, raise_for_status: Optional[bool] = None, ) -> _RequestContext: return self._make_requests( params_list=params_list, retry_options=retry_options, raise_for_status=raise_for_status, ) def request( self, method: str, url: StrOrURL, retry_options: Optional[RetryOptionsBase] = None, raise_for_status: Optional[bool] = None, **kwargs: Any, ) -> _RequestContext: return self._make_request( method=method, url=url, retry_options=retry_options, raise_for_status=raise_for_status, **kwargs, ) def get( self, url: _URL_TYPE, retry_options: Optional[RetryOptionsBase] = None, raise_for_status: Optional[bool] = None, **kwargs: Any, ) -> _RequestContext: return self._make_request( method=hdrs.METH_GET, url=url, retry_options=retry_options, raise_for_status=raise_for_status, **kwargs, ) def options( self, url: _URL_TYPE, retry_options: Optional[RetryOptionsBase] = None, raise_for_status: Optional[bool] = None, **kwargs: Any, ) -> _RequestContext: return self._make_request( method=hdrs.METH_OPTIONS, url=url, retry_options=retry_options, raise_for_status=raise_for_status, **kwargs, ) def head( self, url: _URL_TYPE, retry_options: Optional[RetryOptionsBase] = None, raise_for_status: Optional[bool] = None, **kwargs: Any, ) -> _RequestContext: return self._make_request( method=hdrs.METH_HEAD, url=url, retry_options=retry_options, raise_for_status=raise_for_status, **kwargs, ) def post( self, url: _URL_TYPE, retry_options: Optional[RetryOptionsBase] = None, raise_for_status: Optional[bool] = None, **kwargs: Any, ) -> _RequestContext: return self._make_request( method=hdrs.METH_POST, url=url, retry_options=retry_options, raise_for_status=raise_for_status, **kwargs, ) def put( self, url: _URL_TYPE, retry_options: Optional[RetryOptionsBase] = None, raise_for_status: Optional[bool] = None, **kwargs: Any, ) -> _RequestContext: return self._make_request( method=hdrs.METH_PUT, url=url, retry_options=retry_options, raise_for_status=raise_for_status, **kwargs, ) def patch( self, url: _URL_TYPE, retry_options: Optional[RetryOptionsBase] = None, raise_for_status: Optional[bool] = None, **kwargs: Any, ) -> _RequestContext: return self._make_request( method=hdrs.METH_PATCH, url=url, retry_options=retry_options, raise_for_status=raise_for_status, **kwargs, ) def delete( self, url: _URL_TYPE, retry_options: Optional[RetryOptionsBase] = None, raise_for_status: Optional[bool] = None, **kwargs: Any, ) -> _RequestContext: return self._make_request( method=hdrs.METH_DELETE, url=url, retry_options=retry_options, raise_for_status=raise_for_status, **kwargs, ) async def close(self) -> None: await self._client.close() self._closed = True def _make_request( self, method: str, url: _URL_TYPE, retry_options: Optional[RetryOptionsBase] = None, raise_for_status: Optional[bool] = None, **kwargs: Any, ) -> _RequestContext: url_list = _url_to_urls(url) params_list = [RequestParams( method=method, url=url, headers=kwargs.pop('headers', {}), trace_request_ctx=kwargs.pop('trace_request_ctx', None), kwargs=kwargs, ) for url in url_list] return self._make_requests( params_list=params_list, retry_options=retry_options, raise_for_status=raise_for_status, ) def _make_requests( self, params_list: List[RequestParams], retry_options: Optional[RetryOptionsBase] = None, raise_for_status: Optional[bool] = None, ) -> _RequestContext: if retry_options is None: retry_options = self._retry_options if raise_for_status is None: raise_for_status = self._raise_for_status return _RequestContext( request_func=self._client.request, params_list=params_list, logger=self._logger, retry_options=retry_options, raise_for_status=raise_for_status, ) async def __aenter__(self) -> 'RetryClient': return self async def __aexit__( self, exc_type: Optional[Type[BaseException]], exc_val: Optional[BaseException], exc_tb: Optional[TracebackType], ) -> None: await self.close() def __del__(self) -> None: if getattr(self, '_closed', None) is None: # in case object was not initialized (__init__ raised an exception) return if not self._closed: self._logger.warning("Aiohttp retry client was not closed") aiohttp_retry/py.typed000066400000000000000000000000001432126250700154420ustar00rootroot00000000000000aiohttp_retry/retry_options.py000066400000000000000000000173251432126250700172570ustar00rootroot00000000000000import abc import random from typing import Any, Awaitable, Callable, Iterable, List, Optional, Set, Type from warnings import warn from aiohttp import ClientResponse EvaluateResponseCallbackType = Callable[[ClientResponse], Awaitable[bool]] class RetryOptionsBase: def __init__( self, attempts: int = 3, # How many times we should retry statuses: Optional[Iterable[int]] = None, # On which statuses we should retry exceptions: Optional[Iterable[Type[Exception]]] = None, # On which exceptions we should retry retry_all_server_errors: bool = True, # If should retry all 500 errors or not # a callback that will run on response to decide if retry evaluate_response_callback: Optional[EvaluateResponseCallbackType] = None, ): self.attempts: int = attempts if statuses is None: statuses = set() self.statuses: Iterable[int] = statuses if exceptions is None: exceptions = set() self.exceptions: Iterable[Type[Exception]] = exceptions self.retry_all_server_errors = retry_all_server_errors self.evaluate_response_callback = evaluate_response_callback @abc.abstractmethod def get_timeout(self, attempt: int, response: Optional[ClientResponse] = None) -> float: raise NotImplementedError class ExponentialRetry(RetryOptionsBase): def __init__( self, attempts: int = 3, # How many times we should retry start_timeout: float = 0.1, # Base timeout time, then it exponentially grow max_timeout: float = 30.0, # Max possible timeout between tries factor: float = 2.0, # How much we increase timeout each time statuses: Optional[Set[int]] = None, # On which statuses we should retry exceptions: Optional[Set[Type[Exception]]] = None, # On which exceptions we should retry retry_all_server_errors: bool = True, evaluate_response_callback: Optional[EvaluateResponseCallbackType] = None, ): super().__init__( attempts=attempts, statuses=statuses, exceptions=exceptions, retry_all_server_errors=retry_all_server_errors, evaluate_response_callback=evaluate_response_callback, ) self._start_timeout: float = start_timeout self._max_timeout: float = max_timeout self._factor: float = factor def get_timeout(self, attempt: int, response: Optional[ClientResponse] = None) -> float: """Return timeout with exponential backoff.""" timeout = self._start_timeout * (self._factor ** attempt) return min(timeout, self._max_timeout) def RetryOptions(*args: Any, **kwargs: Any) -> ExponentialRetry: warn("RetryOptions is deprecated, use ExponentialRetry") return ExponentialRetry(*args, **kwargs) class RandomRetry(RetryOptionsBase): def __init__( self, attempts: int = 3, # How many times we should retry statuses: Optional[Iterable[int]] = None, # On which statuses we should retry exceptions: Optional[Iterable[Type[Exception]]] = None, # On which exceptions we should retry min_timeout: float = 0.1, # Minimum possible timeout max_timeout: float = 3.0, # Maximum possible timeout between tries random_func: Callable[[], float] = random.random, # Random number generator retry_all_server_errors: bool = True, evaluate_response_callback: Optional[EvaluateResponseCallbackType] = None, ): super().__init__( attempts=attempts, statuses=statuses, exceptions=exceptions, retry_all_server_errors=retry_all_server_errors, evaluate_response_callback=evaluate_response_callback, ) self.attempts: int = attempts self.min_timeout: float = min_timeout self.max_timeout: float = max_timeout self.random = random_func def get_timeout(self, attempt: int, response: Optional[ClientResponse] = None) -> float: """Generate random timeouts.""" return self.min_timeout + self.random() * (self.max_timeout - self.min_timeout) class ListRetry(RetryOptionsBase): def __init__( self, timeouts: List[float], statuses: Optional[Iterable[int]] = None, # On which statuses we should retry exceptions: Optional[Iterable[Type[Exception]]] = None, # On which exceptions we should retry retry_all_server_errors: bool = True, evaluate_response_callback: Optional[EvaluateResponseCallbackType] = None, ): super().__init__( attempts=len(timeouts), statuses=statuses, exceptions=exceptions, retry_all_server_errors=retry_all_server_errors, evaluate_response_callback=evaluate_response_callback, ) self.timeouts = timeouts def get_timeout(self, attempt: int, response: Optional[ClientResponse] = None) -> float: """timeouts from a defined list.""" return self.timeouts[attempt] class FibonacciRetry(RetryOptionsBase): def __init__( self, attempts: int = 3, multiplier: float = 1.0, statuses: Optional[Iterable[int]] = None, exceptions: Optional[Iterable[Type[Exception]]] = None, max_timeout: float = 3.0, # Maximum possible timeout between tries retry_all_server_errors: bool = True, evaluate_response_callback: Optional[EvaluateResponseCallbackType] = None, ): super().__init__( attempts=attempts, statuses=statuses, exceptions=exceptions, retry_all_server_errors=retry_all_server_errors, evaluate_response_callback=evaluate_response_callback, ) self.max_timeout = max_timeout self.multiplier = multiplier self.prev_step = 1.0 self.current_step = 1.0 def get_timeout(self, attempt: int, response: Optional[ClientResponse] = None) -> float: new_current_step = self.prev_step + self.current_step self.prev_step = self.current_step self.current_step = new_current_step return min(self.multiplier * new_current_step, self.max_timeout) class JitterRetry(ExponentialRetry): """https://github.com/inyutin/aiohttp_retry/issues/44""" def __init__( self, attempts: int = 3, # How many times we should retry start_timeout: float = 0.1, # Base timeout time, then it exponentially grow max_timeout: float = 30.0, # Max possible timeout between tries factor: float = 2.0, # How much we increase timeout each time statuses: Optional[Set[int]] = None, # On which statuses we should retry exceptions: Optional[Set[Type[Exception]]] = None, # On which exceptions we should retry random_interval_size: float = 2.0, # size of interval for random component retry_all_server_errors: bool = True, evaluate_response_callback: Optional[EvaluateResponseCallbackType] = None, ): super().__init__( attempts=attempts, start_timeout=start_timeout, max_timeout=max_timeout, factor=factor, statuses=statuses, exceptions=exceptions, retry_all_server_errors=retry_all_server_errors, evaluate_response_callback=evaluate_response_callback, ) self._start_timeout: float = start_timeout self._max_timeout: float = max_timeout self._factor: float = factor self._random_interval_size = random_interval_size def get_timeout(self, attempt: int, response: Optional[ClientResponse] = None) -> float: timeout: float = super().get_timeout(attempt) + random.uniform(0, self._random_interval_size) ** self._factor return timeout aiohttp_retry/types.py000066400000000000000000000002151432126250700154710ustar00rootroot00000000000000from typing import Union from aiohttp import ClientSession from .client import RetryClient ClientType = Union[ClientSession, RetryClient] codecov.yml000066400000000000000000000006231432126250700132260ustar00rootroot00000000000000codecov: require_ci_to_pass: yes coverage: precision: 2 round: down range: "90...100" status: project: default: threshold: 90% parsers: gcov: branch_detection: conditional: yes loop: yes method: no macro: no comment: layout: "reach,diff,flags,files,footer" behavior: default require_changes: no ignore: - "setup.py" - "tests/*.py" pyproject.toml000066400000000000000000000007521432126250700140000ustar00rootroot00000000000000[tool.mypy] disallow_untyped_calls = true disallow_untyped_defs = true disallow_incomplete_defs = true check_untyped_defs = true disallow_untyped_decorators = true warn_redundant_casts = true warn_unused_ignores = true warn_no_return = true warn_return_any = true warn_unreachable = true [tool.isort] multi_line_output = 3 line_length = 88 include_trailing_comma = true force_grid_wrap = 0 combine_as_imports = true [tool.autopep8] max_line_length = 127 in-place = true recursive = true requirements.txt000066400000000000000000000000151432126250700143400ustar00rootroot00000000000000aiohttp yarl requirements_ci.txt000066400000000000000000000001651432126250700150210ustar00rootroot00000000000000autopep8==1.6.0 pytest==7.1.2 pytest-aiohttp==1.0.4 pytest-cov==3.0.0 mypy==0.961 isort==5.10.1 pre-commit==2.20.0 setup.py000066400000000000000000000016431432126250700125760ustar00rootroot00000000000000from setuptools import find_packages, setup with open("README.md", "r", encoding="utf-8") as fh: long_description = fh.read() setup( name='aiohttp_retry', version='2.8.3', description='Simple retry client for aiohttp', long_description=long_description, long_description_content_type="text/markdown", classifiers=[ "Programming Language :: Python :: 3", "License :: OSI Approved :: MIT License", "Operating System :: OS Independent", ], keywords='aiohttp retry client', author='Dmitry Inyutin', author_email='inyutin.da@gmail.com', url='https://github.com/inyutin/aiohttp_retry', license='MIT', include_package_data=True, packages=find_packages(exclude=["tests", "tests.*"]), platforms=['any'], python_requires='>=3.7', install_requires=[ 'aiohttp', ], package_data={ "aiohttp_retry": ["py.typed"], } ) tests/000077500000000000000000000000001432126250700122225ustar00rootroot00000000000000tests/app.py000066400000000000000000000050521432126250700133560ustar00rootroot00000000000000from aiohttp import web class App: def __init__(self): self.counter = 0 app = web.Application() app.router.add_get('/ping', self.ping_handler) app.router.add_get('/internal_error', self.internal_error_handler) app.router.add_get('/not_found_error', self.not_found_error_handler) app.router.add_get('/sometimes_error', self.sometimes_error) app.router.add_get('/sometimes_json', self.sometimes_json) app.router.add_get('/check_headers', self.check_headers) app.router.add_get('/with_auth', self.with_auth) app.router.add_options('/options_handler', self.ping_handler) app.router.add_head('/head_handler', self.ping_handler) app.router.add_post('/post_handler', self.ping_handler) app.router.add_put('/put_handler', self.ping_handler) app.router.add_patch('/patch_handler', self.ping_handler) app.router.add_delete('/delete_handler', self.ping_handler) self._web_app = app async def ping_handler(self, _: web.Request) -> web.Response: self.counter += 1 return web.Response(text='Ok!', status=200) async def internal_error_handler(self, _: web.Request) -> web.Response: self.counter += 1 raise web.HTTPInternalServerError() async def not_found_error_handler(self, _: web.Request) -> web.Response: self.counter += 1 raise web.HTTPNotFound() async def sometimes_error(self, _: web.Request) -> web.Response: self.counter += 1 if self.counter == 3: return web.Response(text='Ok!', status=200) raise web.HTTPInternalServerError() async def sometimes_json(self, _: web.Request) -> web.Response: self.counter += 1 if self.counter == 3: return web.json_response(data={'status': 'Ok!'}, status=200) return web.Response(text='Ok!', status=200) async def check_headers(self, request: web.Request) -> web.Response: self.counter += 1 if request.headers.get('correct_headers') != 'True': raise web.HTTPNotAcceptable() return web.Response(text='Ok!', status=200) async def with_auth(self, request: web.Request) -> web.Response: self.counter += 1 # BasicAuth("username", "password") if request.headers.get('Authorization') != 'Basic dXNlcm5hbWU6cGFzc3dvcmQ=': return web.Response(text='incorrect auth', status=403) return web.Response(text='Ok!', status=200) @property def web_app(self) -> web.Application: return self._web_app tests/test_client.py000066400000000000000000000361361432126250700151220ustar00rootroot00000000000000from types import SimpleNamespace from typing import Optional, Tuple import pytest from aiohttp import ( BasicAuth, ClientResponse, ClientResponseError, ClientSession, TraceConfig, TraceRequestStartParams, hdrs, ) from yarl import URL from aiohttp_retry import ExponentialRetry, ListRetry, RetryClient from aiohttp_retry.client import RequestParams from aiohttp_retry.retry_options import RetryOptionsBase from tests.app import App async def get_retry_client_and_test_app_for_test( aiohttp_client, raise_for_status: bool = False, retry_options: Optional[RetryOptionsBase] = None, ) -> Tuple[RetryClient, App]: test_app = App() app = test_app.web_app() client = await aiohttp_client(app, raise_for_status=raise_for_status) retry_client = RetryClient(client_session=client, retry_options=retry_options) return retry_client, test_app async def test_hello(aiohttp_client): retry_client, test_app = await get_retry_client_and_test_app_for_test(aiohttp_client) async with retry_client.get('/ping') as response: text = await response.text() assert response.status == 200 assert text == 'Ok!' assert test_app.counter == 1 await retry_client.close() async def test_hello_by_request(aiohttp_client): retry_client, test_app = await get_retry_client_and_test_app_for_test(aiohttp_client) async with retry_client.request(method=hdrs.METH_GET, url='/ping') as response: text = await response.text() assert response.status == 200 assert text == 'Ok!' assert test_app.counter == 1 await retry_client.close() async def test_hello_with_context(aiohttp_client): test_app = App() app = test_app.web_app() client = await aiohttp_client(app) async with RetryClient() as retry_client: retry_client._client = client async with retry_client.get('/ping') as response: text = await response.text() assert response.status == 200 assert text == 'Ok!' assert test_app.counter == 1 async def test_internal_error(aiohttp_client): retry_client, test_app = await get_retry_client_and_test_app_for_test(aiohttp_client) retry_options = ExponentialRetry(attempts=5) async with retry_client.get('/internal_error', retry_options) as response: assert response.status == 500 assert test_app.counter == 5 await retry_client.close() async def test_not_found_error(aiohttp_client): retry_client, test_app = await get_retry_client_and_test_app_for_test(aiohttp_client) retry_options = ExponentialRetry(attempts=5, statuses={404}) async with retry_client.get('/not_found_error', retry_options) as response: assert response.status == 404 assert test_app.counter == 5 await retry_client.close() async def test_sometimes_error(aiohttp_client): retry_client, test_app = await get_retry_client_and_test_app_for_test(aiohttp_client) retry_options = ExponentialRetry(attempts=5) async with retry_client.get('/sometimes_error', retry_options) as response: text = await response.text() assert response.status == 200 assert text == 'Ok!' assert test_app.counter == 3 await retry_client.close() async def test_sometimes_error_with_raise_for_status(aiohttp_client): retry_client, test_app = await get_retry_client_and_test_app_for_test(aiohttp_client, raise_for_status=True) retry_options = ExponentialRetry(attempts=5, exceptions={ClientResponseError}) async with retry_client.get('/sometimes_error', retry_options) as response: text = await response.text() assert response.status == 200 assert text == 'Ok!' assert test_app.counter == 3 await retry_client.close() async def test_override_options(aiohttp_client): retry_client, test_app = await get_retry_client_and_test_app_for_test( aiohttp_client, retry_options=ExponentialRetry(attempts=1) ) retry_options = ExponentialRetry(attempts=5) async with retry_client.get('/sometimes_error', retry_options) as response: text = await response.text() assert response.status == 200 assert text == 'Ok!' assert test_app.counter == 3 await retry_client.close() async def test_hello_awaitable(aiohttp_client): retry_client, test_app = await get_retry_client_and_test_app_for_test(aiohttp_client) response = await retry_client.get('/ping') text = await response.text() assert response.status == 200 assert text == 'Ok!' assert test_app.counter == 1 await retry_client.close() async def test_add_trace_request_ctx(aiohttp_client): actual_request_contexts = [] async def on_request_start( _: ClientSession, trace_config_ctx: SimpleNamespace, __: TraceRequestStartParams, ) -> None: actual_request_contexts.append(trace_config_ctx) test_app = App() trace_config = TraceConfig() trace_config.on_request_start.append(on_request_start) # type: ignore retry_client = RetryClient() retry_client._client = await aiohttp_client( test_app.web_app(), trace_configs=[trace_config] ) async with retry_client.get('/sometimes_error', trace_request_ctx={'foo': 'bar'}): assert test_app.counter == 3 assert actual_request_contexts == [ SimpleNamespace( trace_request_ctx={ 'foo': 'bar', 'current_attempt': i + 1, }, ) for i in range(3) ] @pytest.mark.parametrize("attempts", [2, 3]) async def test_change_urls_in_request(aiohttp_client, attempts): retry_client, test_app = await get_retry_client_and_test_app_for_test( aiohttp_client, retry_options=ExponentialRetry(attempts=attempts) ) async with retry_client.get(url=['/internal_error', '/ping']) as response: text = await response.text() assert response.status == 200 assert text == 'Ok!' assert test_app.counter == 2 await retry_client.close() @pytest.mark.parametrize("attempts", [2, 3]) async def test_change_urls_as_tuple_in_request(aiohttp_client, attempts): retry_client, test_app = await get_retry_client_and_test_app_for_test( aiohttp_client, retry_options=ExponentialRetry(attempts=attempts) ) async with retry_client.get(url=('/internal_error', '/ping')) as response: text = await response.text() assert response.status == 200 assert text == 'Ok!' assert test_app.counter == 2 await retry_client.close() @pytest.mark.parametrize("url", [{"/ping", "/internal_error"}, []]) async def test_pass_bad_urls(aiohttp_client, url): retry_client, _ = await get_retry_client_and_test_app_for_test(aiohttp_client) with pytest.raises(ValueError): async with retry_client.get(url=url): pass await retry_client.close() @pytest.mark.parametrize("url, method", [ ("/options_handler", 'options'), ("/head_handler", 'head'), ("/post_handler", 'post'), ("/put_handler", 'put'), ("/patch_handler", 'patch'), ("/delete_handler", 'delete'), ]) async def test_methods(aiohttp_client, url, method): retry_client, _ = await get_retry_client_and_test_app_for_test(aiohttp_client) method_func = getattr(retry_client, method) async with method_func(url) as response: assert response.method.lower() == method await retry_client.close() async def test_not_found_error_with_retry_client_raise_for_status(aiohttp_client): test_app = App() app = test_app.web_app client = await aiohttp_client(app) retry_client = RetryClient(raise_for_status=True) retry_client._client = client retry_options = ExponentialRetry(attempts=5, statuses={404}) override_response = retry_client.get('/not_found_error', retry_options, raise_for_status=False) assert not override_response._raise_for_status response = retry_client.get('/not_found_error', retry_options) assert response._raise_for_status try: async with response: pass except ClientResponseError as exc: assert exc.status == 404 assert test_app.counter == 5 else: raise AssertionError('Expected ClientResponseError not raised') await retry_client.close() await client.close() async def test_request(aiohttp_client): retry_client, test_app = await get_retry_client_and_test_app_for_test(aiohttp_client) async with retry_client.request(hdrs.METH_GET, '/ping') as response: text = await response.text() assert response.status == 200 assert text == 'Ok!' assert test_app.counter == 1 await retry_client.close() async def test_url_as_yarl(aiohttp_client): """https://github.com/inyutin/aiohttp_retry/issues/41""" retry_client, test_app = await get_retry_client_and_test_app_for_test(aiohttp_client) async with retry_client.get(URL('/ping')) as response: text = await response.text() assert response.status == 200 assert text == 'Ok!' assert test_app.counter == 1 await retry_client.close() async def test_change_client_retry_options(aiohttp_client): retry_options = ExponentialRetry(attempts=5) retry_client, test_app = await get_retry_client_and_test_app_for_test(aiohttp_client, retry_options=retry_options) # first time with 5 attempts is okay async with retry_client.get('/sometimes_error') as response: text = await response.text() assert response.status == 200 assert text == 'Ok!' assert test_app.counter == 3 test_app.counter = 0 retry_client.retry_options.attempts = 2 # second time with 5 attempts is error async with retry_client.get('/sometimes_error') as response: text = await response.text() assert response.status == 500 assert test_app.counter == 2 await retry_client.close() async def test_not_retry_server_errors(aiohttp_client): retry_options = ExponentialRetry(retry_all_server_errors=False) retry_client, test_app = await get_retry_client_and_test_app_for_test(aiohttp_client) async with retry_client.get('/internal_error', retry_options) as response: assert response.status == 500 assert test_app.counter == 1 await retry_client.close() async def test_list_retry_works_for_multiple_attempts(aiohttp_client): retry_options = ListRetry(timeouts=[0]*3) retry_client, test_app = await get_retry_client_and_test_app_for_test(aiohttp_client) async with retry_client.get('/internal_error', retry_options) as response: assert response.status == 500 assert test_app.counter == 3 await retry_client.close() async def test_implicit_client(aiohttp_client): # check that if client not passed that it created implicitly test_app = App() retry_client = RetryClient() assert retry_client._client is not None retry_client._client = await aiohttp_client(test_app.web_app()) async with retry_client.get('/ping') as response: assert response.status == 200 await retry_client.close() async def test_evaluate_response_callback(aiohttp_client): async def evaluate_response(response: ClientResponse) -> bool: try: await response.json() except: return False return True retry_options = ExponentialRetry(attempts=5, evaluate_response_callback=evaluate_response) retry_client, test_app = await get_retry_client_and_test_app_for_test(aiohttp_client, retry_options=retry_options) async with retry_client.get('/sometimes_json') as response: body = await response.json() assert response.status == 200 assert body == {'status': 'Ok!'} assert test_app.counter == 3 async def test_multiply_urls_by_requests(aiohttp_client): retry_client, test_app = await get_retry_client_and_test_app_for_test(aiohttp_client) async with retry_client.requests( params_list=[ RequestParams( method='GET', url='/internal_error', ), RequestParams( method='GET', url='/ping', ), ] ) as response: text = await response.text() assert response.status == 200 assert text == 'Ok!' assert test_app.counter == 2 await retry_client.close() async def test_multiply_methods_by_requests(aiohttp_client): retry_options = ExponentialRetry(statuses={405}) # method not allowed retry_client, _ = await get_retry_client_and_test_app_for_test(aiohttp_client, retry_options=retry_options) async with retry_client.requests( params_list=[ RequestParams( method='POST', url='/ping', ), RequestParams( method='GET', url='/ping', ), ] ) as response: text = await response.text() assert response.status == 200 assert text == 'Ok!' await retry_client.close() async def test_change_headers(aiohttp_client): retry_options = ExponentialRetry(statuses={406}) retry_client, test_app = await get_retry_client_and_test_app_for_test(aiohttp_client, retry_options=retry_options) async with retry_client.requests( params_list=[ RequestParams( method='GET', url='/check_headers', ), RequestParams( method='GET', url='/check_headers', headers={'correct_headers': 'True'}, ), ] ) as response: text = await response.text() assert response.status == 200 assert text == 'Ok!' assert test_app.counter == 2 await retry_client.close() async def test_additional_params(aiohttp_client): # https://github.com/inyutin/aiohttp_retry/issues/79 auth = BasicAuth("username", "password") retry_client, _ = await get_retry_client_and_test_app_for_test(aiohttp_client) async with retry_client.request(hdrs.METH_GET, '/with_auth', auth=auth) as response: text = await response.text() assert response.status == 200 assert text == 'Ok!' await retry_client.close() async def test_request_headers(aiohttp_client): retry_client, test_app = await get_retry_client_and_test_app_for_test(aiohttp_client) async with retry_client.get(url='/check_headers', headers={'correct_headers': 'True'}) as response: text = await response.text() assert response.status == 200 assert text == 'Ok!' assert test_app.counter == 1 await retry_client.close() async def test_list_retry_all_failed(aiohttp_client): # there was a specific bug async def evaluate_response(response: ClientResponse) -> bool: return False retry_options = ListRetry(timeouts=[1]*3, statuses={403}, evaluate_response_callback=evaluate_response) retry_client, test_app = await get_retry_client_and_test_app_for_test(aiohttp_client) async with retry_client.get('/with_auth', retry_options=retry_options) as response: assert response.status == 403 assert test_app.counter == 3 await retry_client.close() tests/test_retry_options.py000066400000000000000000000026661432126250700165650ustar00rootroot00000000000000import random from aiohttp_retry import ( ExponentialRetry, FibonacciRetry, JitterRetry, ListRetry, RandomRetry, ) def test_exponential_retry(): retry = ExponentialRetry(attempts=10) timeouts = [retry.get_timeout(x) for x in range(10)] assert timeouts == [0.1, 0.2, 0.4, 0.8, 1.6, 3.2, 6.4, 12.8, 25.6, 30.0] def test_random_retry(): retry = RandomRetry(attempts=10, random_func=random.Random(0).random) timeouts = [round(retry.get_timeout(x), 2) for x in range(10)] assert timeouts == [2.55, 2.3, 1.32, 0.85, 1.58, 1.27, 2.37, 0.98, 1.48, 1.79] def test_list_retry(): expected = [1.2, 2.1, 3.4, 4.3, 4.5, 5.4, 5.6, 6.5, 6.7, 7.6] retry = ListRetry(expected) timeouts = [retry.get_timeout(x) for x in range(10)] assert timeouts == expected def test_fibonacci_retry(): retry = FibonacciRetry(attempts=10, multiplier=2, max_timeout=60) timeouts = [retry.get_timeout(x) for x in range(10)] assert timeouts == [4.0, 6.0, 10.0, 16.0, 26.0, 42.0, 60, 60, 60, 60] def test_jitter_retry(): random.seed(10) retry = JitterRetry(attempts=10) timeouts = [retry.get_timeout(x) for x in range(10)] assert len(timeouts) == 10 expected = [ 1.4, 0.9, 1.7, 0.9, 4.2, 5.9, 8.1, 12.9, 26.6, 30.4, ] for idx, timeout in enumerate(timeouts): assert abs(timeout - expected[idx]) < 0.1