diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 71ea09f..8762d03 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -16,7 +16,7 @@ repos: rev: v0.14.6 hooks: - id: ruff-check - # - id: ruff-format + - id: ruff-format - repo: https://github.com/astral-sh/uv-pre-commit rev: 0.9.13 hooks: @@ -26,3 +26,10 @@ repos: hooks: - id: check-github-workflows - id: check-renovate +- repo: https://github.com/adamtheturtle/doccmd-pre-commit + rev: v2025.12.8.5 + hooks: + - id: doccmd + args: ["--language", "python", "--no-pad-file", "--no-pad-groups", "--command", "ruff format", "docs/"] + additional_dependencies: + - ruff==0.14.8 diff --git a/CHANGELOG.md b/CHANGELOG.md index 80ea3e5..d94452a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ - Switch to [Zensical for building the documentation](https://zensical.org/) [#62](https://github.com/python-backoff/backoff/pull/62) (from [@edgarrmondragon](https://github.com/edgarrmondragon)) - Include changelog in the documentation [#65](https://github.com/python-backoff/backoff/pull/65) (from [@edgarrmondragon](https://github.com/edgarrmondragon)) +### Internal + +- Use Ruff to give the codebase a consistent format [#66](https://github.com/python-backoff/backoff/pull/66) (from [@edgarrmondragon](https://github.com/edgarrmondragon)) + ## [v2.3.0] - 2025-11-28 ### Changed diff --git a/README.rst b/README.rst index 1f9620d..0193055 100644 --- a/README.rst +++ b/README.rst @@ -40,8 +40,10 @@ is raised. Here's an example using exponential backoff when any .. code-block:: python - @backoff.on_exception(backoff.expo, - requests.exceptions.RequestException) + @backoff.on_exception( + backoff.expo, + requests.exceptions.RequestException, + ) def get_url(url): return requests.get(url) @@ -50,9 +52,13 @@ the same backoff behavior is desired for more than one exception type: .. code-block:: python - @backoff.on_exception(backoff.expo, - (requests.exceptions.Timeout, - requests.exceptions.ConnectionError)) + @backoff.on_exception( + backoff.expo, + ( + requests.exceptions.Timeout, + requests.exceptions.ConnectionError, + ), + ) def get_url(url): return requests.get(url) @@ -66,9 +72,11 @@ of total time in seconds that can elapse before giving up. .. code-block:: python - @backoff.on_exception(backoff.expo, - requests.exceptions.RequestException, - max_time=60) + @backoff.on_exception( + backoff.expo, + requests.exceptions.RequestException, + max_time=60, + ) def get_url(url): return requests.get(url) @@ -78,10 +86,12 @@ to make to the target function before giving up. .. code-block:: python - @backoff.on_exception(backoff.expo, - requests.exceptions.RequestException, - max_tries=8, - jitter=None) + @backoff.on_exception( + backoff.expo, + requests.exceptions.RequestException, + max_tries=8, + jitter=None, + ) def get_url(url): return requests.get(url) @@ -97,10 +107,13 @@ be retried: def fatal_code(e): return 400 <= e.response.status_code < 500 - @backoff.on_exception(backoff.expo, - requests.exceptions.RequestException, - max_time=300, - giveup=fatal_code) + + @backoff.on_exception( + backoff.expo, + requests.exceptions.RequestException, + max_time=300, + giveup=fatal_code, + ) def get_url(url): return requests.get(url) @@ -118,11 +131,14 @@ case, regardless of the logic in the `on_exception` handler. def fatal_code(e): return 400 <= e.response.status_code < 500 - @backoff.on_exception(backoff.expo, - requests.exceptions.RequestException, - max_time=300, - raise_on_giveup=False, - giveup=fatal_code) + + @backoff.on_exception( + backoff.expo, + requests.exceptions.RequestException, + max_time=300, + raise_on_giveup=False, + giveup=fatal_code, + ) def get_url(url): return requests.get(url) @@ -142,7 +158,11 @@ return value of the target function is the empty list: .. code-block:: python - @backoff.on_predicate(backoff.fibo, lambda x: x == [], max_value=13) + @backoff.on_predicate( + backoff.fibo, + lambda x: x == [], + max_value=13, + ) def poll_for_messages(queue): return queue.get() @@ -164,7 +184,11 @@ gets a non-falsey result could be defined like like this: .. code-block:: python - @backoff.on_predicate(backoff.constant, jitter=None, interval=1) + @backoff.on_predicate( + backoff.constant, + jitter=None, + interval=1, + ) def poll_for_message(queue): return queue.get() @@ -217,12 +241,16 @@ backoff behavior for different cases: .. code-block:: python @backoff.on_predicate(backoff.fibo, max_value=13) - @backoff.on_exception(backoff.expo, - requests.exceptions.HTTPError, - max_time=60) - @backoff.on_exception(backoff.expo, - requests.exceptions.Timeout, - max_time=300) + @backoff.on_exception( + backoff.expo, + requests.exceptions.HTTPError, + max_time=60, + ) + @backoff.on_exception( + backoff.expo, + requests.exceptions.Timeout, + max_time=300, + ) def poll_for_message(queue): return queue.get() @@ -245,9 +273,13 @@ runtime to obtain the value: # and that it has a dictionary-like 'config' property return app.config["BACKOFF_MAX_TIME"] - @backoff.on_exception(backoff.expo, - ValueError, - max_time=lookup_max_time) + + @backoff.on_exception( + backoff.expo, + ValueError, + max_time=lookup_max_time, + ) + def my_function(): ... Event handlers -------------- @@ -275,13 +307,18 @@ implemented like so: .. code-block:: python def backoff_hdlr(details): - print ("Backing off {wait:0.1f} seconds after {tries} tries " - "calling function {target} with args {args} and kwargs " - "{kwargs}".format(**details)) + print( + "Backing off {wait:0.1f} seconds after {tries} tries " + "calling function {target} with args {args} and kwargs " + "{kwargs}".format(**details) + ) + - @backoff.on_exception(backoff.expo, - requests.exceptions.RequestException, - on_backoff=backoff_hdlr) + @backoff.on_exception( + backoff.expo, + requests.exceptions.RequestException, + on_backoff=backoff_hdlr, + ) def get_url(url): return requests.get(url) @@ -293,9 +330,14 @@ handler functions as the value of the ``on_backoff`` keyword arg: .. code-block:: python - @backoff.on_exception(backoff.expo, - requests.exceptions.RequestException, - on_backoff=[backoff_hdlr1, backoff_hdlr2]) + @backoff.on_exception( + backoff.expo, + requests.exceptions.RequestException, + on_backoff=[ + backoff_hdlr1, + backoff_hdlr2, + ], + ) def get_url(url): return requests.get(url) @@ -326,7 +368,11 @@ asynchronous HTTP client/server library. .. code-block:: python - @backoff.on_exception(backoff.expo, aiohttp.ClientError, max_time=60) + @backoff.on_exception( + backoff.expo, + aiohttp.ClientError, + max_time=60, + ) async def get_url(url): async with aiohttp.ClientSession(raise_for_status=True) as session: async with session.get(url) as response: @@ -343,7 +389,7 @@ as: .. code-block:: python - logging.getLogger('backoff').addHandler(logging.StreamHandler()) + logging.getLogger("backoff").addHandler(logging.StreamHandler()) The default logging level is INFO, which corresponds to logging anytime a retry event occurs. If you would instead like to log @@ -351,7 +397,7 @@ only when a giveup event occurs, set the logger level to ERROR. .. code-block:: python - logging.getLogger('backoff').setLevel(logging.ERROR) + logging.getLogger("backoff").setLevel(logging.ERROR) It is also possible to specify an alternate logger with the ``logger`` keyword argument. If a string value is specified the logger will be @@ -359,25 +405,30 @@ looked up by name. .. code-block:: python - @backoff.on_exception(backoff.expo, - requests.exceptions.RequestException, - logger='my_logger') - # ... + @backoff.on_exception( + backoff.expo, + requests.exceptions.RequestException, + logger="my_logger", + ) + def my_function(): ... It is also supported to specify a Logger (or LoggerAdapter) object directly. .. code-block:: python - my_logger = logging.getLogger('my_logger') + my_logger = logging.getLogger("my_logger") my_handler = logging.StreamHandler() my_logger.addHandler(my_handler) my_logger.setLevel(logging.ERROR) - @backoff.on_exception(backoff.expo, - requests.exceptions.RequestException, - logger=my_logger) - # ... + + @backoff.on_exception( + backoff.expo, + requests.exceptions.RequestException, + logger=my_logger, + ) + def my_function(): ... Default logging can be disabled all together by specifying ``logger=None``. In this case, if desired alternative logging behavior diff --git a/backoff/__init__.py b/backoff/__init__.py index 1a5fd81..4d2bb4f 100644 --- a/backoff/__init__.py +++ b/backoff/__init__.py @@ -12,20 +12,21 @@ For examples and full documentation see the README at https://github.com/python-backoff/backoff """ + from backoff._decorator import on_exception, on_predicate from backoff._jitter import full_jitter, random_jitter from backoff._wait_gen import constant, expo, fibo, runtime, decay __all__ = [ - 'on_predicate', - 'on_exception', - 'constant', - 'expo', - 'decay', - 'fibo', - 'runtime', - 'full_jitter', - 'random_jitter', + "on_predicate", + "on_exception", + "constant", + "expo", + "decay", + "fibo", + "runtime", + "full_jitter", + "random_jitter", ] __version__ = "2.2.1" diff --git a/backoff/_async.py b/backoff/_async.py index 7c950b2..89148d0 100644 --- a/backoff/_async.py +++ b/backoff/_async.py @@ -11,9 +11,11 @@ def _ensure_coroutine(coro_or_func): if inspect.iscoroutinefunction(coro_or_func): return coro_or_func else: + @functools.wraps(coro_or_func) async def f(*args, **kwargs): return coro_or_func(*args, **kwargs) + return f @@ -21,27 +23,32 @@ def _ensure_coroutines(coros_or_funcs): return [_ensure_coroutine(f) for f in coros_or_funcs] -async def _call_handlers(handlers, - *, - target, args, kwargs, tries, elapsed, - **extra): +async def _call_handlers(handlers, *, target, args, kwargs, tries, elapsed, **extra): details = { - 'target': target, - 'args': args, - 'kwargs': kwargs, - 'tries': tries, - 'elapsed': elapsed, + "target": target, + "args": args, + "kwargs": kwargs, + "tries": tries, + "elapsed": elapsed, } details.update(extra) for handler in handlers: await handler(details) -def retry_predicate(target, wait_gen, predicate, - *, - max_tries, max_time, jitter, - on_success, on_backoff, on_giveup, - wait_gen_kwargs): +def retry_predicate( + target, + wait_gen, + predicate, + *, + max_tries, + max_time, + jitter, + on_success, + on_backoff, + on_giveup, + wait_gen_kwargs, +): on_success = _ensure_coroutines(on_success) on_backoff = _ensure_coroutines(on_backoff) on_giveup = _ensure_coroutines(on_giveup) @@ -54,7 +61,6 @@ def retry_predicate(target, wait_gen, predicate, @functools.wraps(target) async def retry(*args, **kwargs): - # update variables from outer function args max_tries_value = _maybe_call(max_tries) max_time_value = _maybe_call(max_time) @@ -75,23 +81,22 @@ async def retry(*args, **kwargs): ret = await target(*args, **kwargs) if predicate(ret): - max_tries_exceeded = (tries == max_tries_value) - max_time_exceeded = (max_time_value is not None and - elapsed >= max_time_value) + max_tries_exceeded = tries == max_tries_value + max_time_exceeded = ( + max_time_value is not None and elapsed >= max_time_value + ) if max_tries_exceeded or max_time_exceeded: await _call_handlers(on_giveup, **details, value=ret) break try: - seconds = _next_wait(wait, ret, jitter, elapsed, - max_time_value) + seconds = _next_wait(wait, ret, jitter, elapsed, max_time_value) except StopIteration: await _call_handlers(on_giveup, **details, value=ret) break - await _call_handlers(on_backoff, **details, value=ret, - wait=seconds) + await _call_handlers(on_backoff, **details, value=ret, wait=seconds) # Note: there is no convenient way to pass explicit event # loop to decorator, so here we assume that either default @@ -113,11 +118,21 @@ async def retry(*args, **kwargs): return retry -def retry_exception(target, wait_gen, exception, - *, - max_tries, max_time, jitter, giveup, - on_success, on_backoff, on_giveup, raise_on_giveup, - wait_gen_kwargs): +def retry_exception( + target, + wait_gen, + exception, + *, + max_tries, + max_time, + jitter, + giveup, + on_success, + on_backoff, + on_giveup, + raise_on_giveup, + wait_gen_kwargs, +): on_success = _ensure_coroutines(on_success) on_backoff = _ensure_coroutines(on_backoff) on_giveup = _ensure_coroutines(on_giveup) @@ -129,7 +144,6 @@ def retry_exception(target, wait_gen, exception, @functools.wraps(target) async def retry(*args, **kwargs): - max_tries_value = _maybe_call(max_tries) max_time_value = _maybe_call(max_time) @@ -151,9 +165,10 @@ async def retry(*args, **kwargs): ret = await target(*args, **kwargs) except exception as e: giveup_result = await giveup(e) - max_tries_exceeded = (tries == max_tries_value) - max_time_exceeded = (max_time_value is not None and - elapsed >= max_time_value) + max_tries_exceeded = tries == max_tries_value + max_time_exceeded = ( + max_time_value is not None and elapsed >= max_time_value + ) if giveup_result or max_tries_exceeded or max_time_exceeded: await _call_handlers(on_giveup, **details, exception=e) @@ -162,14 +177,12 @@ async def retry(*args, **kwargs): return None try: - seconds = _next_wait(wait, e, jitter, elapsed, - max_time_value) + seconds = _next_wait(wait, e, jitter, elapsed, max_time_value) except StopIteration: await _call_handlers(on_giveup, **details, exception=e) raise e - await _call_handlers(on_backoff, **details, wait=seconds, - exception=e) + await _call_handlers(on_backoff, **details, wait=seconds, exception=e) # Note: there is no convenient way to pass explicit event # loop to decorator, so here we assume that either default @@ -185,4 +198,5 @@ async def retry(*args, **kwargs): await _call_handlers(on_success, **details) return ret + return retry diff --git a/backoff/_common.py b/backoff/_common.py index 468ab32..69fb847 100644 --- a/backoff/_common.py +++ b/backoff/_common.py @@ -8,7 +8,7 @@ # Use module-specific logger with a default null handler. -_logger = logging.getLogger('backoff') +_logger = logging.getLogger("backoff") _logger.addHandler(logging.NullHandler()) # pragma: no cover _logger.setLevel(logging.INFO) @@ -34,10 +34,7 @@ def _init_wait_gen(wait_gen, wait_gen_kwargs): def _next_wait(wait, send_value, jitter, elapsed, max_time): value = wait.send(send_value) try: - if jitter is not None: - seconds = jitter(value) - else: - seconds = value + seconds = jitter(value) if jitter is not None else value except TypeError: warnings.warn( "Nullary jitter function signature is deprecated. Use " @@ -65,14 +62,20 @@ def _prepare_logger(logger): # Configure handler list with user specified handler and optionally # with a default handler bound to the specified logger. def _config_handlers( - user_handlers, *, default_handler=None, logger=None, log_level=None, + user_handlers, + *, + default_handler=None, + logger=None, + log_level=None, ): handlers = [] if logger is not None: assert log_level is not None, "Log level is not specified" # bind the specified logger to the default log handler log_handler = functools.partial( - default_handler, logger=logger, log_level=log_level, + default_handler, + logger=logger, + log_level=log_level, ) handlers.append(log_handler) @@ -81,7 +84,7 @@ def _config_handlers( # user specified handlers can either be an iterable of handlers # or a single handler. either way append them to the list. - if hasattr(user_handlers, '__iter__'): + if hasattr(user_handlers, "__iter__"): # add all handlers in the iterable handlers += list(user_handlers) else: @@ -94,27 +97,27 @@ def _config_handlers( # Default backoff handler def _log_backoff(details, logger, log_level): msg = "Backing off %s(...) for %.1fs (%s)" - log_args = [details['target'].__name__, details['wait']] + log_args = [details["target"].__name__, details["wait"]] exc_typ, exc, _ = sys.exc_info() if exc is not None: exc_fmt = traceback.format_exception_only(exc_typ, exc)[-1] log_args.append(exc_fmt.rstrip("\n")) else: - log_args.append(details['value']) + log_args.append(details["value"]) logger.log(log_level, msg, *log_args) # Default giveup handler def _log_giveup(details, logger, log_level): msg = "Giving up %s(...) after %d tries (%s)" - log_args = [details['target'].__name__, details['tries']] + log_args = [details["target"].__name__, details["tries"]] exc_typ, exc, _ = sys.exc_info() if exc is not None: exc_fmt = traceback.format_exception_only(exc_typ, exc)[-1] log_args.append(exc_fmt.rstrip("\n")) else: - log_args.append(details['value']) + log_args.append(details["value"]) logger.log(log_level, msg, *log_args) diff --git a/backoff/_decorator.py b/backoff/_decorator.py index 5aeb284..e068a58 100644 --- a/backoff/_decorator.py +++ b/backoff/_decorator.py @@ -24,19 +24,21 @@ ) -def on_predicate(wait_gen: _WaitGenerator, - predicate: _Predicate[Any] = operator.not_, - *, - max_tries: Optional[_MaybeCallable[int]] = None, - max_time: Optional[_MaybeCallable[float]] = None, - jitter: Union[_Jitterer, None] = full_jitter, - on_success: Union[_Handler, Iterable[_Handler], None] = None, - on_backoff: Union[_Handler, Iterable[_Handler], None] = None, - on_giveup: Union[_Handler, Iterable[_Handler], None] = None, - logger: _MaybeLogger = 'backoff', - backoff_log_level: int = logging.INFO, - giveup_log_level: int = logging.ERROR, - **wait_gen_kwargs: Any) -> Callable[[_CallableT], _CallableT]: +def on_predicate( + wait_gen: _WaitGenerator, + predicate: _Predicate[Any] = operator.not_, + *, + max_tries: Optional[_MaybeCallable[int]] = None, + max_time: Optional[_MaybeCallable[float]] = None, + jitter: Union[_Jitterer, None] = full_jitter, + on_success: Union[_Handler, Iterable[_Handler], None] = None, + on_backoff: Union[_Handler, Iterable[_Handler], None] = None, + on_giveup: Union[_Handler, Iterable[_Handler], None] = None, + logger: _MaybeLogger = "backoff", + backoff_log_level: int = logging.INFO, + giveup_log_level: int = logging.ERROR, + **wait_gen_kwargs: Any, +) -> Callable[[_CallableT], _CallableT]: """Returns decorator for backoff and retry triggered by predicate. Args: @@ -80,6 +82,7 @@ def on_predicate(wait_gen: _WaitGenerator, args will first be evaluated and their return values passed. This is useful for runtime configuration. """ + def decorate(target): nonlocal logger, on_success, on_backoff, on_giveup @@ -120,21 +123,23 @@ def decorate(target): return decorate -def on_exception(wait_gen: _WaitGenerator, - exception: _MaybeSequence[Type[Exception]], - *, - max_tries: Optional[_MaybeCallable[int]] = None, - max_time: Optional[_MaybeCallable[float]] = None, - jitter: Union[_Jitterer, None] = full_jitter, - giveup: _Predicate[Exception] = lambda e: False, - on_success: Union[_Handler, Iterable[_Handler], None] = None, - on_backoff: Union[_Handler, Iterable[_Handler], None] = None, - on_giveup: Union[_Handler, Iterable[_Handler], None] = None, - raise_on_giveup: bool = True, - logger: _MaybeLogger = 'backoff', - backoff_log_level: int = logging.INFO, - giveup_log_level: int = logging.ERROR, - **wait_gen_kwargs: Any) -> Callable[[_CallableT], _CallableT]: +def on_exception( + wait_gen: _WaitGenerator, + exception: _MaybeSequence[Type[Exception]], + *, + max_tries: Optional[_MaybeCallable[int]] = None, + max_time: Optional[_MaybeCallable[float]] = None, + jitter: Union[_Jitterer, None] = full_jitter, + giveup: _Predicate[Exception] = lambda e: False, + on_success: Union[_Handler, Iterable[_Handler], None] = None, + on_backoff: Union[_Handler, Iterable[_Handler], None] = None, + on_giveup: Union[_Handler, Iterable[_Handler], None] = None, + raise_on_giveup: bool = True, + logger: _MaybeLogger = "backoff", + backoff_log_level: int = logging.INFO, + giveup_log_level: int = logging.ERROR, + **wait_gen_kwargs: Any, +) -> Callable[[_CallableT], _CallableT]: """Returns decorator for backoff and retry triggered by exception. Args: @@ -180,6 +185,7 @@ def on_exception(wait_gen: _WaitGenerator, args will first be evaluated and their return values passed. This is useful for runtime configuration. """ + def decorate(target): nonlocal logger, on_success, on_backoff, on_giveup diff --git a/backoff/_sync.py b/backoff/_sync.py index d8d0046..f7058da 100644 --- a/backoff/_sync.py +++ b/backoff/_sync.py @@ -2,28 +2,35 @@ import functools import time -from backoff._common import (_init_wait_gen, _maybe_call, _next_wait) +from backoff._common import _init_wait_gen, _maybe_call, _next_wait def _call_handlers(hdlrs, target, args, kwargs, tries, elapsed, **extra): details = { - 'target': target, - 'args': args, - 'kwargs': kwargs, - 'tries': tries, - 'elapsed': elapsed, + "target": target, + "args": args, + "kwargs": kwargs, + "tries": tries, + "elapsed": elapsed, } details.update(extra) for hdlr in hdlrs: hdlr(details) -def retry_predicate(target, wait_gen, predicate, - *, - max_tries, max_time, jitter, - on_success, on_backoff, on_giveup, - wait_gen_kwargs): - +def retry_predicate( + target, + wait_gen, + predicate, + *, + max_tries, + max_time, + jitter, + on_success, + on_backoff, + on_giveup, + wait_gen_kwargs, +): @functools.wraps(target) def retry(*args, **kwargs): max_tries_value = _maybe_call(max_tries) @@ -45,23 +52,22 @@ def retry(*args, **kwargs): ret = target(*args, **kwargs) if predicate(ret): - max_tries_exceeded = (tries == max_tries_value) - max_time_exceeded = (max_time_value is not None and - elapsed >= max_time_value) + max_tries_exceeded = tries == max_tries_value + max_time_exceeded = ( + max_time_value is not None and elapsed >= max_time_value + ) if max_tries_exceeded or max_time_exceeded: _call_handlers(on_giveup, **details, value=ret) break try: - seconds = _next_wait(wait, ret, jitter, elapsed, - max_time_value) + seconds = _next_wait(wait, ret, jitter, elapsed, max_time_value) except StopIteration: _call_handlers(on_giveup, **details) break - _call_handlers(on_backoff, **details, - value=ret, wait=seconds) + _call_handlers(on_backoff, **details, value=ret, wait=seconds) time.sleep(seconds) continue @@ -74,12 +80,21 @@ def retry(*args, **kwargs): return retry -def retry_exception(target, wait_gen, exception, - *, - max_tries, max_time, jitter, giveup, - on_success, on_backoff, on_giveup, raise_on_giveup, - wait_gen_kwargs): - +def retry_exception( + target, + wait_gen, + exception, + *, + max_tries, + max_time, + jitter, + giveup, + on_success, + on_backoff, + on_giveup, + raise_on_giveup, + wait_gen_kwargs, +): @functools.wraps(target) def retry(*args, **kwargs): max_tries_value = _maybe_call(max_tries) @@ -102,9 +117,10 @@ def retry(*args, **kwargs): try: ret = target(*args, **kwargs) except exception as e: - max_tries_exceeded = (tries == max_tries_value) - max_time_exceeded = (max_time_value is not None and - elapsed >= max_time_value) + max_tries_exceeded = tries == max_tries_value + max_time_exceeded = ( + max_time_value is not None and elapsed >= max_time_value + ) if giveup(e) or max_tries_exceeded or max_time_exceeded: _call_handlers(on_giveup, **details, exception=e) @@ -113,18 +129,17 @@ def retry(*args, **kwargs): return None try: - seconds = _next_wait(wait, e, jitter, elapsed, - max_time_value) + seconds = _next_wait(wait, e, jitter, elapsed, max_time_value) except StopIteration: _call_handlers(on_giveup, **details, exception=e) raise e - _call_handlers(on_backoff, **details, wait=seconds, - exception=e) + _call_handlers(on_backoff, **details, wait=seconds, exception=e) time.sleep(seconds) else: _call_handlers(on_success, **details) return ret + return retry diff --git a/backoff/_typing.py b/backoff/_typing.py index a72960a..99d2e9e 100644 --- a/backoff/_typing.py +++ b/backoff/_typing.py @@ -1,8 +1,17 @@ # coding:utf-8 import logging import sys -from typing import (Any, Callable, Coroutine, Dict, Generator, Sequence, Tuple, - TypeVar, Union) +from typing import ( + Any, + Callable, + Coroutine, + Dict, + Generator, + Sequence, + Tuple, + TypeVar, + Union, +) if sys.version_info >= (3, 8): # pragma: no cover from typing import TypedDict @@ -11,6 +20,7 @@ try: from typing_extensions import TypedDict except ImportError: + class TypedDict(dict): def __init_subclass__(cls, **kwargs: Any) -> None: return super().__init_subclass__() @@ -32,7 +42,7 @@ class Details(_Details, total=False): T = TypeVar("T") -_CallableT = TypeVar('_CallableT', bound=Callable[..., Any]) +_CallableT = TypeVar("_CallableT", bound=Callable[..., Any]) _Handler = Union[ Callable[[Details], None], Callable[[Details], Coroutine[Any, Any, None]], diff --git a/backoff/_wait_gen.py b/backoff/_wait_gen.py index 6338c5a..14fbae3 100644 --- a/backoff/_wait_gen.py +++ b/backoff/_wait_gen.py @@ -10,7 +10,6 @@ def expo( factor: float = 1, max_value: Optional[float] = None, ) -> Generator[float, Any, None]: - """Generator for exponential decay. Args: @@ -37,7 +36,6 @@ def decay( decay_factor: float = 1, min_value: Optional[float] = None, ) -> Generator[float, Any, None]: - """Generator for exponential decay[1]: Args: diff --git a/backoff/types.py b/backoff/types.py index fc280aa..7ca670f 100644 --- a/backoff/types.py +++ b/backoff/types.py @@ -2,5 +2,5 @@ from ._typing import Details __all__ = [ - 'Details', + "Details", ] diff --git a/docs/advanced/combining-decorators.md b/docs/advanced/combining-decorators.md index a7afcc5..870936b 100644 --- a/docs/advanced/combining-decorators.md +++ b/docs/advanced/combining-decorators.md @@ -8,8 +8,8 @@ Decorators are applied from bottom to top (inside out): ```python @backoff.on_predicate(backoff.fibo, lambda x: x is None) # Applied last -@backoff.on_exception(backoff.expo, HTTPError) # Applied second -@backoff.on_exception(backoff.expo, Timeout) # Applied first +@backoff.on_exception(backoff.expo, HTTPError) # Applied second +@backoff.on_exception(backoff.expo, Timeout) # Applied first def complex_operation(): pass ``` @@ -22,13 +22,13 @@ def complex_operation(): @backoff.on_exception( backoff.expo, requests.exceptions.Timeout, - max_time=300 # Generous timeout for network issues + max_time=300, # Generous timeout for network issues ) @backoff.on_exception( backoff.expo, requests.exceptions.HTTPError, max_time=60, # Shorter timeout for HTTP errors - giveup=lambda e: 400 <= e.response.status_code < 500 + giveup=lambda e: 400 <= e.response.status_code < 500, ) def api_call(url): response = requests.get(url) @@ -43,12 +43,12 @@ def api_call(url): backoff.constant, lambda result: result.get("status") == "pending", interval=5, - max_time=600 + max_time=600, ) @backoff.on_exception( backoff.expo, requests.exceptions.RequestException, - max_time=60 + max_time=60, ) def poll_until_ready(job_id): response = requests.get(f"/api/jobs/{job_id}") @@ -63,24 +63,27 @@ Inner decorators execute first: ```python calls = [] + def track_call(func_name): def handler(details): calls.append(func_name) + return handler + @backoff.on_exception( backoff.constant, ValueError, - on_backoff=track_call('outer'), + on_backoff=track_call("outer"), max_tries=2, - interval=0.01 + interval=0.01, ) @backoff.on_exception( backoff.constant, TypeError, - on_backoff=track_call('inner'), + on_backoff=track_call("inner"), max_tries=2, - interval=0.01 + interval=0.01, ) def failing_function(error_type): raise error_type("Test") @@ -107,13 +110,13 @@ def network_operation(): @backoff.on_exception( backoff.expo, Exception, - max_time=600 # Overall 10-minute limit + max_time=600, # Overall 10-minute limit ) @backoff.on_exception( backoff.constant, Timeout, interval=1, - max_tries=3 # Quick retries for timeouts + max_tries=3, # Quick retries for timeouts ) def layered_retry(): pass diff --git a/docs/advanced/custom-strategies.md b/docs/advanced/custom-strategies.md index 1c8b0cc..9863a3f 100644 --- a/docs/advanced/custom-strategies.md +++ b/docs/advanced/custom-strategies.md @@ -14,6 +14,7 @@ def my_wait_gen(): while True: yield 5 + @backoff.on_exception(my_wait_gen, Exception) def my_function(): pass @@ -34,12 +35,13 @@ def linear_backoff(start=1, increment=1, max_value=None): yield value value += increment + @backoff.on_exception( linear_backoff, Exception, start=2, increment=3, - max_value=30 + max_value=30, ) def my_function(): pass @@ -54,14 +56,22 @@ def polynomial_backoff(base=2, exponent=2, max_value=None): """Polynomial: base^(tries^exponent)""" n = 1 while True: - value = base ** (n ** exponent) + value = base ** (n**exponent) if max_value and value > max_value: yield max_value else: yield value n += 1 -@backoff.on_exception(polynomial_backoff, Exception, base=2, exponent=1.5) + +@backoff.on_exception( + polynomial_backoff, + Exception, + base=2, + exponent=1.5, +) +def my_function(): + pass ``` ### Stepped Backoff @@ -79,11 +89,14 @@ def stepped_backoff(steps): for _ in range(max_tries): yield wait_time + @backoff.on_exception( stepped_backoff, Exception, - steps=[(3, 1), (3, 5), (None, 30)] + steps=[(3, 1), (3, 5), (None, 30)], ) +def my_function(): + pass ``` ### Random Backoff @@ -91,12 +104,21 @@ def stepped_backoff(steps): ```python import random + def random_backoff(min_wait=1, max_wait=60): """Random wait between min and max""" while True: yield random.uniform(min_wait, max_wait) -@backoff.on_exception(random_backoff, Exception, min_wait=1, max_wait=10) + +@backoff.on_exception( + random_backoff, + Exception, + min_wait=1, + max_wait=10, +) +def my_function(): + pass ``` ### Time-of-Day Aware @@ -104,6 +126,7 @@ def random_backoff(min_wait=1, max_wait=60): ```python from datetime import datetime + def business_hours_backoff(): """Shorter waits during business hours""" while True: @@ -113,5 +136,8 @@ def business_hours_backoff(): else: yield 60 # 1 minute otherwise + @backoff.on_exception(business_hours_backoff, Exception) +def my_function(): + pass ``` diff --git a/docs/advanced/runtime-config.md b/docs/advanced/runtime-config.md index 31d3e25..bf3dc02 100644 --- a/docs/advanced/runtime-config.md +++ b/docs/advanced/runtime-config.md @@ -13,15 +13,17 @@ class Config: MAX_RETRIES = 5 MAX_TIME = 60 + @backoff.on_exception( backoff.expo, Exception, max_tries=lambda: Config.MAX_RETRIES, - max_time=lambda: Config.MAX_TIME + max_time=lambda: Config.MAX_TIME, ) def configurable_function(): pass + # Change configuration at runtime Config.MAX_RETRIES = 10 ``` @@ -31,11 +33,12 @@ Config.MAX_RETRIES = 10 ```python import os + @backoff.on_exception( backoff.expo, Exception, - max_tries=lambda: int(os.getenv('RETRY_MAX_TRIES', '5')), - max_time=lambda: int(os.getenv('RETRY_MAX_TIME', '60')) + max_tries=lambda: int(os.getenv("RETRY_MAX_TRIES", "5")), + max_time=lambda: int(os.getenv("RETRY_MAX_TIME", "60")), ) def env_configured(): pass @@ -46,15 +49,17 @@ def env_configured(): ```python import json + def load_config(): - with open('config.json') as f: + with open("config.json") as f: return json.load(f) + @backoff.on_exception( backoff.expo, Exception, - max_tries=lambda: load_config()['retry']['max_tries'], - max_time=lambda: load_config()['retry']['max_time'] + max_tries=lambda: load_config()["retry"]["max_tries"], + max_time=lambda: load_config()["retry"]["max_time"], ) def file_configured(): pass @@ -64,14 +69,12 @@ def file_configured(): ```python def get_wait_gen(): - if app.config.get('fast_retry'): + if app.config.get("fast_retry"): return backoff.constant return backoff.expo -@backoff.on_exception( - lambda: get_wait_gen(), - Exception -) + +@backoff.on_exception(lambda: get_wait_gen(), Exception) def dynamic_wait(): pass ``` @@ -86,11 +89,13 @@ class RateLimiter: def get_interval(self): return 10 if self.rate_limited else 1 + rate_limiter = RateLimiter() + @backoff.on_predicate( backoff.constant, - interval=lambda: rate_limiter.get_interval() + interval=lambda: rate_limiter.get_interval(), ) def adaptive_poll(): return check_resource() diff --git a/docs/examples.md b/docs/examples.md index cc30a23..97ecaf6 100644 --- a/docs/examples.md +++ b/docs/examples.md @@ -10,9 +10,12 @@ Real-world examples of using backoff in production. import backoff import requests -@backoff.on_exception(backoff.expo, - requests.exceptions.RequestException, - max_time=60) + +@backoff.on_exception( + backoff.expo, + requests.exceptions.RequestException, + max_time=60, +) def fetch_data(url): response = requests.get(url) response.raise_for_status() @@ -27,7 +30,7 @@ def fetch_data(url): predicate=lambda r: r.status_code == 429, value=lambda r: int(r.headers.get("Retry-After", 1)), jitter=None, - max_tries=10 + max_tries=10, ) def rate_limited_api_call(endpoint): return requests.get(endpoint) @@ -40,10 +43,11 @@ def should_retry(response): # Retry on 5xx and 429, but not 4xx return response.status_code >= 500 or response.status_code == 429 + @backoff.on_predicate( backoff.expo, should_retry, - max_time=120 + max_time=120, ) def resilient_api_call(url): response = requests.get(url) @@ -60,11 +64,12 @@ def resilient_api_call(url): import sqlalchemy from sqlalchemy.exc import OperationalError, TimeoutError + @backoff.on_exception( backoff.expo, (OperationalError, TimeoutError), max_tries=5, - max_time=30 + max_time=30, ) def connect_to_database(connection_string): engine = sqlalchemy.create_engine(connection_string) @@ -77,17 +82,19 @@ def connect_to_database(connection_string): ```python from sqlalchemy.exc import DBAPIError + def is_deadlock(e): """Check if exception is a deadlock""" if isinstance(e, DBAPIError): return "deadlock" in str(e).lower() return False + @backoff.on_exception( backoff.expo, DBAPIError, giveup=lambda e: not is_deadlock(e), - max_tries=3 + max_tries=3, ) def execute_transaction(session, operation): try: @@ -107,10 +114,11 @@ def execute_transaction(session, operation): import aiohttp import backoff + @backoff.on_exception( backoff.expo, aiohttp.ClientError, - max_time=60 + max_time=60, ) async def fetch_async(url): async with aiohttp.ClientSession() as session: @@ -123,10 +131,11 @@ async def fetch_async(url): ```python import asyncpg + @backoff.on_exception( backoff.expo, asyncpg.PostgresError, - max_tries=5 + max_tries=5, ) async def query_async(pool, query): async with pool.acquire() as conn: @@ -136,11 +145,16 @@ async def query_async(pool, query): ### Multiple Async Tasks with Individual Retries ```python -@backoff.on_exception(backoff.expo, aiohttp.ClientError, max_tries=3) +@backoff.on_exception( + backoff.expo, + aiohttp.ClientError, + max_tries=3, +) async def fetch_with_retry(session, url): async with session.get(url) as response: return await response.json() + async def fetch_all(urls): async with aiohttp.ClientSession() as session: tasks = [fetch_with_retry(session, url) for url in urls] @@ -156,7 +170,7 @@ async def fetch_all(urls): backoff.constant, lambda job: job["status"] != "complete", interval=5, - max_time=600 + max_time=600, ) def wait_for_job(job_id): response = requests.get(f"/api/jobs/{job_id}") @@ -170,7 +184,7 @@ def wait_for_job(job_id): backoff.fibo, lambda result: not result, max_value=30, - max_time=300 + max_time=300, ) def wait_for_resource(resource_id): try: @@ -187,7 +201,7 @@ def wait_for_resource(resource_id): backoff.constant, lambda messages: len(messages) == 0, interval=2, - jitter=None + jitter=None, ) def poll_queue(queue_name): return message_queue.receive(queue_name, max_messages=10) @@ -201,19 +215,21 @@ def poll_queue(queue_name): import boto3 from botocore.exceptions import ClientError + def is_throttled(e): if isinstance(e, ClientError): - return e.response['Error']['Code'] in ['SlowDown', 'RequestLimitExceeded'] + return e.response["Error"]["Code"] in ["SlowDown", "RequestLimitExceeded"] return False + @backoff.on_exception( backoff.expo, ClientError, giveup=lambda e: not is_throttled(e), - max_tries=5 + max_tries=5, ) def upload_to_s3(bucket, key, data): - s3 = boto3.client('s3') + s3 = boto3.client("s3") s3.put_object(Bucket=bucket, Key=key, Body=data) ``` @@ -223,11 +239,12 @@ def upload_to_s3(bucket, key, data): @backoff.on_exception( backoff.expo, ClientError, - giveup=lambda e: e.response['Error']['Code'] != 'ProvisionedThroughputExceededException', - max_time=30 + giveup=lambda e: e.response["Error"]["Code"] + != "ProvisionedThroughputExceededException", + max_time=30, ) def write_to_dynamodb(table_name, item): - dynamodb = boto3.resource('dynamodb') + dynamodb = boto3.resource("dynamodb") table = dynamodb.Table(table_name) table.put_item(Item=item) ``` @@ -242,24 +259,26 @@ import logging logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) + def log_retry(details): logger.warning( f"Backing off {details['wait']:.1f}s after {details['tries']} tries " f"calling {details['target'].__name__}" ) + def log_giveup(details): logger.error( - f"Giving up after {details['tries']} tries " - f"and {details['elapsed']:.1f}s" + f"Giving up after {details['tries']} tries and {details['elapsed']:.1f}s" ) + @backoff.on_exception( backoff.expo, Exception, on_backoff=log_retry, on_giveup=log_giveup, - max_tries=5 + max_tries=5, ) def flaky_function(): pass @@ -268,20 +287,23 @@ def flaky_function(): ### Metrics Collection ```python -retry_metrics = {'total_retries': 0, 'giveups': 0} +retry_metrics = {"total_retries": 0, "giveups": 0} + def count_retry(details): - retry_metrics['total_retries'] += 1 + retry_metrics["total_retries"] += 1 + def count_giveup(details): - retry_metrics['giveups'] += 1 + retry_metrics["giveups"] += 1 + @backoff.on_exception( backoff.expo, Exception, on_backoff=count_retry, on_giveup=count_giveup, - max_tries=3 + max_tries=3, ) def monitored_function(): pass @@ -296,24 +318,24 @@ def monitored_function(): @backoff.on_predicate( backoff.fibo, lambda result: result is None, - max_value=13 + max_value=13, ) @backoff.on_exception( backoff.expo, requests.exceptions.HTTPError, giveup=lambda e: 400 <= e.response.status_code < 500, - max_time=60 + max_time=60, ) @backoff.on_exception( backoff.expo, requests.exceptions.Timeout, - max_tries=3 + max_tries=3, ) def comprehensive_retry(url): response = requests.get(url) response.raise_for_status() data = response.json() - return data if data.get('ready') else None + return data if data.get("ready") else None ``` ### Circuit Breaker Pattern @@ -340,17 +362,20 @@ class CircuitBreaker: if self.failure_count >= self.failure_threshold: self.opened_at = time.time() + circuit_breaker = CircuitBreaker() + def check_circuit(e): circuit_breaker.record_failure() return not circuit_breaker.should_attempt() + @backoff.on_exception( backoff.expo, requests.exceptions.RequestException, giveup=check_circuit, - max_tries=5 + max_tries=5, ) def protected_api_call(url): if not circuit_breaker.should_attempt(): @@ -372,17 +397,20 @@ class RetryConfig: def get_max_tries(self): return self.max_tries + config = RetryConfig() + @backoff.on_exception( backoff.expo, Exception, max_time=lambda: config.get_max_time(), - max_tries=lambda: config.get_max_tries() + max_tries=lambda: config.get_max_tries(), ) def configurable_retry(): pass + # Can update config at runtime config.max_time = 120 ``` @@ -396,11 +424,12 @@ config.max_time = 120 backoff.expo, requests.exceptions.RequestException, max_tries=3, - raise_on_giveup=False + raise_on_giveup=False, ) def optional_api_call(url): return requests.get(url) + # Use with fallback result = optional_api_call(primary_url) if result is None: @@ -413,17 +442,17 @@ if result is None: class RetryExhaustedError(Exception): pass + def raise_custom_error(details): - raise RetryExhaustedError( - f"Failed after {details['tries']} attempts" - ) + raise RetryExhaustedError(f"Failed after {details['tries']} attempts") + @backoff.on_exception( backoff.expo, Exception, on_giveup=raise_custom_error, max_tries=5, - raise_on_giveup=False + raise_on_giveup=False, ) def critical_operation(): pass diff --git a/docs/faq.md b/docs/faq.md index 7ba6958..d0d6bb3 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -55,7 +55,12 @@ def my_function(): Yes! The function will stop retrying when either limit is reached: ```python -@backoff.on_exception(backoff.expo, Exception, max_tries=10, max_time=300) +@backoff.on_exception( + backoff.expo, + Exception, + max_tries=10, + max_time=300, +) def my_function(): pass ``` @@ -97,14 +102,15 @@ Use the `giveup` parameter: ```python def is_permanent_error(e): - if hasattr(e, 'response'): + if hasattr(e, "response"): return 400 <= e.response.status_code < 500 return False + @backoff.on_exception( backoff.expo, requests.exceptions.RequestException, - giveup=is_permanent_error + giveup=is_permanent_error, ) def api_call(): pass @@ -119,11 +125,12 @@ By default (`raise_on_giveup=True`), the original exception is re-raised. You ca backoff.expo, Exception, max_tries=5, - raise_on_giveup=False + raise_on_giveup=False, ) def my_function(): pass + result = my_function() # Returns None if all retries fail ``` @@ -134,7 +141,11 @@ Yes, pass a tuple: ```python @backoff.on_exception( backoff.expo, - (TimeoutError, ConnectionError, requests.exceptions.RequestException) + ( + TimeoutError, + ConnectionError, + requests.exceptions.RequestException, + ), ) def my_function(): pass @@ -162,7 +173,7 @@ Use `backoff.runtime`: backoff.runtime, predicate=lambda r: r.status_code == 429, value=lambda r: int(r.headers.get("Retry-After", 1)), - jitter=None + jitter=None, ) def api_call(): return requests.get(url) @@ -190,10 +201,11 @@ Yes, you can use async functions for `on_success`, `on_backoff`, and `on_giveup` async def log_retry(details): await async_logger.log(f"Retry {details['tries']}") + @backoff.on_exception( backoff.expo, Exception, - on_backoff=log_retry + on_backoff=log_retry, ) async def my_function(): pass @@ -209,10 +221,11 @@ Use event handlers: def log_backoff(details): logger.warning(f"Retry {details['tries']} after {details['elapsed']:.1f}s") + @backoff.on_exception( backoff.expo, Exception, - on_backoff=log_backoff + on_backoff=log_backoff, ) def my_function(): pass @@ -225,8 +238,8 @@ Yes, backoff has a default logger. Enable it: ```python import logging -logging.getLogger('backoff').addHandler(logging.StreamHandler()) -logging.getLogger('backoff').setLevel(logging.INFO) +logging.getLogger("backoff").addHandler(logging.StreamHandler()) +logging.getLogger("backoff").setLevel(logging.INFO) ``` ### How do I disable all logging? @@ -308,12 +321,14 @@ class CircuitBreaker: return True return False + breaker = CircuitBreaker() + @backoff.on_exception( backoff.expo, Exception, - giveup=breaker.should_giveup + giveup=breaker.should_giveup, ) def protected_call(): pass @@ -325,12 +340,13 @@ Yes, pass callables instead of values: ```python def get_max_time(): - return app.config['RETRY_MAX_TIME'] + return app.config["RETRY_MAX_TIME"] + @backoff.on_exception( backoff.expo, Exception, - max_time=get_max_time + max_time=get_max_time, ) def my_function(): pass @@ -347,14 +363,14 @@ def test_retry_behavior(): attempts = [] def track_attempts(details): - attempts.append(details['tries']) + attempts.append(details["tries"]) @backoff.on_exception( backoff.constant, ValueError, on_backoff=track_attempts, max_tries=3, - interval=0.01 + interval=0.01, ) def failing_function(): raise ValueError("Test error") @@ -382,6 +398,8 @@ The default `full_jitter` adds randomness. To see exact wait times, disable jitt ```python @backoff.on_exception(backoff.expo, Exception, jitter=None) +def my_function(): + pass ``` ### Can I see what's happening during retries? @@ -390,8 +408,9 @@ Enable logging or use event handlers: ```python import logging + logging.basicConfig(level=logging.DEBUG) -logging.getLogger('backoff').setLevel(logging.DEBUG) +logging.getLogger("backoff").setLevel(logging.DEBUG) ``` Or: @@ -400,8 +419,10 @@ Or: @backoff.on_exception( backoff.expo, Exception, - on_backoff=lambda d: print(f"Try {d['tries']}, wait {d['wait']:.1f}s") + on_backoff=lambda d: print(f"Try {d['tries']}, wait {d['wait']:.1f}s"), ) +def my_function(): + pass ``` ## Migration and Alternatives diff --git a/docs/getting-started.md b/docs/getting-started.md index 329f6f8..5203fe6 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -25,8 +25,11 @@ Let's start with a simple example - retrying a network request: import backoff import requests -@backoff.on_exception(backoff.expo, - requests.exceptions.RequestException) + +@backoff.on_exception( + backoff.expo, + requests.exceptions.RequestException, +) def get_url(url): return requests.get(url) ``` @@ -42,10 +45,12 @@ This decorator will: In production, you'll want to limit retries: ```python -@backoff.on_exception(backoff.expo, - requests.exceptions.RequestException, - max_time=60, - max_tries=5) +@backoff.on_exception( + backoff.expo, + requests.exceptions.RequestException, + max_time=60, + max_tries=5, +) def get_url(url): return requests.get(url) ``` @@ -62,9 +67,11 @@ You can retry on multiple exception types: ```python @backoff.on_exception( backoff.expo, - (requests.exceptions.Timeout, - requests.exceptions.ConnectionError), - max_time=30 + ( + requests.exceptions.Timeout, + requests.exceptions.ConnectionError, + ), + max_time=30, ) def get_url(url): return requests.get(url) @@ -79,11 +86,12 @@ def fatal_code(e): """Don't retry on 4xx errors""" return 400 <= e.response.status_code < 500 + @backoff.on_exception( backoff.expo, requests.exceptions.RequestException, max_time=300, - giveup=fatal_code + giveup=fatal_code, ) def get_url(url): return requests.get(url) @@ -94,10 +102,12 @@ def get_url(url): For polling or checking return values: ```python -@backoff.on_predicate(backoff.constant, - lambda result: result is None, - interval=5, - max_time=300) +@backoff.on_predicate( + backoff.constant, + lambda result: result is None, + interval=5, + max_time=300, +) def check_job_status(job_id): response = requests.get(f"/jobs/{job_id}") if response.json()["status"] == "complete": @@ -113,6 +123,7 @@ Backoff provides several wait strategies: ```python @backoff.on_exception(backoff.expo, Exception) +def my_function(): ... ``` Wait times: 1s, 2s, 4s, 8s, 16s, ... @@ -121,6 +132,7 @@ Wait times: 1s, 2s, 4s, 8s, 16s, ... ```python @backoff.on_exception(backoff.fibo, Exception) +def my_function(): ... ``` Wait times: 1s, 1s, 2s, 3s, 5s, 8s, 13s, ... @@ -128,7 +140,12 @@ Wait times: 1s, 1s, 2s, 3s, 5s, 8s, 13s, ... ### Constant ```python -@backoff.on_exception(backoff.constant, Exception, interval=5) +@backoff.on_exception( + backoff.constant, + Exception, + interval=5, +) +def my_function(): ... ``` Wait times: 5s, 5s, 5s, 5s, ... @@ -141,15 +158,17 @@ Track what's happening during retries: def log_backoff(details): print(f"Backing off {details['wait']:.1f} seconds after {details['tries']} tries") + def log_success(details): print(f"Success after {details['tries']} tries") + @backoff.on_exception( backoff.expo, requests.exceptions.RequestException, on_backoff=log_backoff, on_success=log_success, - max_tries=5 + max_tries=5, ) def get_url(url): return requests.get(url) @@ -162,9 +181,12 @@ Backoff works seamlessly with async functions: ```python import aiohttp -@backoff.on_exception(backoff.expo, - aiohttp.ClientError, - max_time=60) + +@backoff.on_exception( + backoff.expo, + aiohttp.ClientError, + max_time=60, +) async def get_url(url): async with aiohttp.ClientSession() as session: async with session.get(url) as response: diff --git a/docs/index.md b/docs/index.md index 159eaed..ed64404 100644 --- a/docs/index.md +++ b/docs/index.md @@ -32,9 +32,12 @@ Basic retry on exception: import backoff import requests -@backoff.on_exception(backoff.expo, - requests.exceptions.RequestException, - max_time=60) + +@backoff.on_exception( + backoff.expo, + requests.exceptions.RequestException, + max_time=60, +) def get_url(url): return requests.get(url) ``` @@ -59,9 +62,11 @@ def call_api(): ### Database Retries ```python -@backoff.on_exception(backoff.expo, - sqlalchemy.exc.OperationalError, - max_tries=5) +@backoff.on_exception( + backoff.expo, + sqlalchemy.exc.OperationalError, + max_tries=5, +) def query_database(): return session.query(Model).all() ``` @@ -69,10 +74,12 @@ def query_database(): ### Polling for Results ```python -@backoff.on_predicate(backoff.constant, - lambda result: result is None, - interval=2, - max_time=300) +@backoff.on_predicate( + backoff.constant, + lambda result: result is None, + interval=2, + max_time=300, +) def poll_for_result(job_id): return check_job_status(job_id) ``` diff --git a/docs/user-guide/async.md b/docs/user-guide/async.md index 5323bbb..bf931bd 100644 --- a/docs/user-guide/async.md +++ b/docs/user-guide/async.md @@ -10,6 +10,7 @@ Simply decorate async functions with the same decorators: import backoff import aiohttp + @backoff.on_exception(backoff.expo, aiohttp.ClientError) async def fetch_data(url): async with aiohttp.ClientSession() as session: @@ -23,14 +24,13 @@ Event handlers can be async when used with async functions: ```python async def async_log_retry(details): - await log_service.log( - f"Retry {details['tries']} after {details['elapsed']:.1f}s" - ) + await log_service.log(f"Retry {details['tries']} after {details['elapsed']:.1f}s") + @backoff.on_exception( backoff.expo, Exception, - on_backoff=async_log_retry + on_backoff=async_log_retry, ) async def async_operation(): pass @@ -44,7 +44,7 @@ async def async_operation(): @backoff.on_exception( backoff.expo, aiohttp.ClientError, - max_time=60 + max_time=60, ) async def get_url(url): async with aiohttp.ClientSession(raise_for_status=True) as session: @@ -57,10 +57,11 @@ async def get_url(url): ```python import asyncpg + @backoff.on_exception( backoff.expo, asyncpg.PostgresError, - max_tries=5 + max_tries=5, ) async def query_database(pool, query): async with pool.acquire() as conn: @@ -72,11 +73,17 @@ async def query_database(pool, query): ```python import asyncio -@backoff.on_exception(backoff.expo, aiohttp.ClientError, max_tries=3) + +@backoff.on_exception( + backoff.expo, + aiohttp.ClientError, + max_tries=3, +) async def fetch_one(session, url): async with session.get(url) as response: return await response.json() + async def fetch_all(urls): async with aiohttp.ClientSession() as session: tasks = [fetch_one(session, url) for url in urls] @@ -90,7 +97,7 @@ async def fetch_all(urls): backoff.constant, lambda result: result["status"] != "complete", interval=5, - max_time=300 + max_time=300, ) async def poll_job_status(job_id): async with aiohttp.ClientSession() as session: @@ -106,10 +113,11 @@ Sync handlers work with async functions: def sync_log(details): print(f"Retry {details['tries']}") + @backoff.on_exception( backoff.expo, Exception, - on_backoff=sync_log # Sync handler with async function + on_backoff=sync_log, # Sync handler with async function ) async def async_function(): pass @@ -121,10 +129,11 @@ But async handlers only work with async functions: async def async_log(details): await log_to_service(details) + @backoff.on_exception( backoff.expo, Exception, - on_backoff=async_log # Must be used with async function + on_backoff=async_log, # Must be used with async function ) async def async_function(): pass @@ -140,6 +149,7 @@ import logging logger = logging.getLogger(__name__) + async def log_async_retry(details): logger.warning( f"Async retry {details['tries']}: " @@ -147,12 +157,16 @@ async def log_async_retry(details): f"elapsed={details['elapsed']:.1f}s" ) + @backoff.on_exception( backoff.expo, - (aiohttp.ClientError, asyncio.TimeoutError), + ( + aiohttp.ClientError, + asyncio.TimeoutError, + ), max_tries=5, max_time=60, - on_backoff=log_async_retry + on_backoff=log_async_retry, ) async def robust_fetch(url, timeout=10): async with aiohttp.ClientSession() as session: @@ -160,10 +174,12 @@ async def robust_fetch(url, timeout=10): response.raise_for_status() return await response.json() + # Usage async def main(): result = await robust_fetch("https://api.example.com/data") print(result) + asyncio.run(main()) ``` diff --git a/docs/user-guide/configuration.md b/docs/user-guide/configuration.md index 645b1e1..8bb23b0 100644 --- a/docs/user-guide/configuration.md +++ b/docs/user-guide/configuration.md @@ -41,7 +41,7 @@ Use both to create flexible retry policies: backoff.expo, Exception, max_tries=10, - max_time=300 + max_time=300, ) def my_function(): pass @@ -58,15 +58,17 @@ class Config: MAX_RETRIES = 5 MAX_TIME = 60 + @backoff.on_exception( backoff.expo, Exception, max_tries=lambda: Config.MAX_RETRIES, - max_time=lambda: Config.MAX_TIME + max_time=lambda: Config.MAX_TIME, ) def configurable_function(): pass + # Can change at runtime Config.MAX_RETRIES = 10 ``` @@ -76,11 +78,12 @@ Config.MAX_RETRIES = 10 ```python import os + @backoff.on_exception( backoff.expo, Exception, - max_tries=lambda: int(os.getenv('MAX_RETRIES', '5')), - max_time=lambda: int(os.getenv('MAX_TIME', '60')) + max_tries=lambda: int(os.getenv("MAX_RETRIES", "5")), + max_time=lambda: int(os.getenv("MAX_TIME", "60")), ) def env_configured(): pass @@ -96,9 +99,9 @@ Each wait strategy accepts different parameters. @backoff.on_exception( backoff.expo, Exception, - base=2, # Base wait time - factor=2, # Multiplication factor - max_value=60 # Maximum wait time + base=2, # Base wait time + factor=2, # Multiplication factor + max_value=60, # Maximum wait time ) def expo_config(): pass @@ -110,7 +113,7 @@ def expo_config(): @backoff.on_exception( backoff.fibo, Exception, - max_value=30 # Maximum wait time + max_value=30, # Maximum wait time ) def fibo_config(): pass @@ -122,7 +125,7 @@ def fibo_config(): @backoff.on_exception( backoff.constant, Exception, - interval=5 # Fixed interval in seconds + interval=5, # Fixed interval in seconds ) def constant_config(): pass @@ -134,7 +137,7 @@ def constant_config(): @backoff.on_predicate( backoff.runtime, predicate=lambda r: r.status_code == 429, - value=lambda r: int(r.headers.get("Retry-After", 1)) + value=lambda r: int(r.headers.get("Retry-After", 1)), ) def runtime_config(): return requests.get(url) @@ -148,8 +151,18 @@ Control randomization of wait times. ```python @backoff.on_exception(backoff.expo, Exception) +def my_function(): + pass + + # Same as: -@backoff.on_exception(backoff.expo, Exception, jitter=backoff.full_jitter) +@backoff.on_exception( + backoff.expo, + Exception, + jitter=backoff.full_jitter, +) +def my_function(): + pass ``` Wait time is random between 0 and calculated value. @@ -157,7 +170,13 @@ Wait time is random between 0 and calculated value. ### Random Jitter ```python -@backoff.on_exception(backoff.expo, Exception, jitter=backoff.random_jitter) +@backoff.on_exception( + backoff.expo, + Exception, + jitter=backoff.random_jitter, +) +def my_function(): + pass ``` Adds 0-1000ms to calculated value. @@ -166,6 +185,8 @@ Adds 0-1000ms to calculated value. ```python @backoff.on_exception(backoff.expo, Exception, jitter=None) +def my_function(): + pass ``` Exact wait times, no randomization. @@ -175,10 +196,16 @@ Exact wait times, no randomization. ```python import random + def custom_jitter(value): return value * random.uniform(0.8, 1.2) -@backoff.on_exception(backoff.expo, Exception, jitter=custom_jitter) + +@backoff.on_exception( + backoff.expo, + Exception, + jitter=custom_jitter, +) def my_function(): pass ``` @@ -191,10 +218,11 @@ def my_function(): def should_giveup(e): return isinstance(e, ValueError) + @backoff.on_exception( backoff.expo, Exception, - giveup=should_giveup + giveup=should_giveup, ) def my_function(): pass @@ -204,16 +232,17 @@ def my_function(): ```python def fatal_error(e): - if hasattr(e, 'response'): + if hasattr(e, "response"): status = e.response.status_code # Don't retry client errors except rate limiting return 400 <= status < 500 and status != 429 return False + @backoff.on_exception( backoff.expo, requests.exceptions.RequestException, - giveup=fatal_error + giveup=fatal_error, ) def api_call(): pass @@ -228,7 +257,7 @@ def complex_giveup(e): return True # Give up on 4xx except 429 - if hasattr(e, 'response'): + if hasattr(e, "response"): status = e.response.status_code if 400 <= status < 500 and status != 429: return True @@ -246,12 +275,13 @@ Control whether to raise exception when giving up: def raises_on_failure(): pass + # Returns None instead @backoff.on_exception( backoff.expo, Exception, max_tries=3, - raise_on_giveup=False + raise_on_giveup=False, ) def returns_none_on_failure(): pass @@ -275,7 +305,12 @@ def wait_for_truthy(): def needs_retry(result): return result.get("status") == "pending" -@backoff.on_predicate(backoff.expo, needs_retry, max_time=300) + +@backoff.on_predicate( + backoff.expo, + needs_retry, + max_time=300, +) def poll_status(): return api.get_status() ``` @@ -292,7 +327,12 @@ def should_retry(result): return True return False -@backoff.on_predicate(backoff.fibo, should_retry, max_value=60) + +@backoff.on_predicate( + backoff.fibo, + should_retry, + max_value=60, +) def complex_poll(): return get_resource() ``` @@ -307,7 +347,7 @@ def complex_poll(): requests.exceptions.RequestException, max_tries=5, max_time=60, - giveup=lambda e: 400 <= getattr(e.response, 'status_code', 500) < 500 + giveup=lambda e: 400 <= getattr(e.response, "status_code", 500) < 500, ) def api_request(): pass @@ -320,7 +360,7 @@ def api_request(): backoff.expo, sqlalchemy.exc.OperationalError, max_tries=3, - max_time=30 + max_time=30, ) def db_query(): pass @@ -334,7 +374,7 @@ def db_query(): lambda result: result["status"] != "complete", interval=5, jitter=None, - max_time=600 + max_time=600, ) def poll_job(): return check_job_status() @@ -347,7 +387,7 @@ def poll_job(): backoff.fibo, lambda result: not result.is_ready(), max_value=60, - max_time=3600 # 1 hour + max_time=3600, # 1 hour ) def wait_for_completion(): return check_operation() diff --git a/docs/user-guide/decorators.md b/docs/user-guide/decorators.md index 86cef23..02d72bb 100644 --- a/docs/user-guide/decorators.md +++ b/docs/user-guide/decorators.md @@ -12,8 +12,11 @@ The `on_exception` decorator retries a function when a specified exception is ra import backoff import requests -@backoff.on_exception(backoff.expo, - requests.exceptions.RequestException) + +@backoff.on_exception( + backoff.expo, + requests.exceptions.RequestException, +) def get_url(url): return requests.get(url) ``` @@ -39,9 +42,11 @@ Handle different exceptions with the same backoff: ```python @backoff.on_exception( backoff.expo, - (requests.exceptions.Timeout, - requests.exceptions.ConnectionError, - requests.exceptions.HTTPError) + ( + requests.exceptions.Timeout, + requests.exceptions.ConnectionError, + requests.exceptions.HTTPError, + ), ) def make_request(url): return requests.get(url) @@ -54,15 +59,16 @@ Customize when to stop retrying: ```python def is_fatal(e): """Don't retry on client errors""" - if hasattr(e, 'response') and e.response is not None: + if hasattr(e, "response") and e.response is not None: return 400 <= e.response.status_code < 500 return False + @backoff.on_exception( backoff.expo, requests.exceptions.RequestException, giveup=is_fatal, - max_time=300 + max_time=300, ) def api_call(endpoint): response = requests.get(endpoint) @@ -79,11 +85,12 @@ Return None instead of raising when all retries are exhausted: backoff.expo, requests.exceptions.RequestException, max_tries=5, - raise_on_giveup=False + raise_on_giveup=False, ) def optional_request(url): return requests.get(url) + # Returns None if all retries fail result = optional_request("https://example.com") ``` @@ -95,9 +102,11 @@ The `on_predicate` decorator retries when a condition is true about the return v ### Basic Usage ```python -@backoff.on_predicate(backoff.fibo, - lambda x: x is None, - max_value=13) +@backoff.on_predicate( + backoff.fibo, + lambda x: x is None, + max_value=13, +) def poll_for_result(job_id): result = check_job(job_id) return result if result else None @@ -120,7 +129,11 @@ def poll_for_result(job_id): When no predicate is specified, the decorator retries on falsey values: ```python -@backoff.on_predicate(backoff.constant, interval=2, max_time=60) +@backoff.on_predicate( + backoff.constant, + interval=2, + max_time=60, +) def wait_for_resource(): # Retries until a truthy value is returned return resource.get() or None @@ -134,7 +147,7 @@ Define specific conditions for retry: @backoff.on_predicate( backoff.expo, lambda result: result["status"] == "pending", - max_time=600 + max_time=600, ) def poll_job_status(job_id): return api.get_job(job_id) @@ -145,12 +158,17 @@ def poll_job_status(job_id): ```python def needs_retry(result): return ( - result is None or - result.get("status") in ["pending", "processing"] or - not result.get("ready", False) + result is None + or result.get("status") in ["pending", "processing"] + or not result.get("ready", False) ) -@backoff.on_predicate(backoff.fibo, needs_retry, max_value=60) + +@backoff.on_predicate( + backoff.fibo, + needs_retry, + max_value=60, +) def complex_poll(resource_id): return api.get_resource(resource_id) ``` @@ -160,13 +178,21 @@ def complex_poll(resource_id): Stack multiple decorators for complex retry logic: ```python -@backoff.on_predicate(backoff.fibo, lambda x: x is None, max_value=13) -@backoff.on_exception(backoff.expo, - requests.exceptions.HTTPError, - max_time=60) -@backoff.on_exception(backoff.expo, - requests.exceptions.Timeout, - max_time=300) +@backoff.on_predicate( + backoff.fibo, + lambda x: x is None, + max_value=13, +) +@backoff.on_exception( + backoff.expo, + requests.exceptions.HTTPError, + max_time=60, +) +@backoff.on_exception( + backoff.expo, + requests.exceptions.Timeout, + max_time=300, +) def robust_poll(endpoint): response = requests.get(endpoint) response.raise_for_status() @@ -186,14 +212,14 @@ Event handlers receive a dictionary with these keys: ```python { - 'target': , - 'args': , - 'kwargs': , - 'tries': , - 'elapsed': , - 'wait': , # on_backoff only - 'value': , # on_predicate only - 'exception': , # on_exception only + "target": my_function, # function reference + "args": (arg1, arg2), # positional args tuple + "kwargs": {"key": "value"}, # keyword args dict + "tries": 3, # number of tries so far + "elapsed": 1.5, # elapsed time in seconds + "wait": 2.0, # seconds to wait (on_backoff only) + "value": None, # return value (on_predicate only) + "exception": Exception(), # exception (on_exception only) } ``` @@ -201,15 +227,18 @@ Example handler: ```python def detailed_log(details): - print(f"Try {details['tries']}: " - f"elapsed={details['elapsed']:.2f}s, " - f"wait={details.get('wait', 0):.2f}s") + print( + f"Try {details['tries']}: " + f"elapsed={details['elapsed']:.2f}s, " + f"wait={details.get('wait', 0):.2f}s" + ) + @backoff.on_exception( backoff.expo, Exception, on_backoff=detailed_log, - max_tries=5 + max_tries=5, ) def my_function(): pass diff --git a/docs/user-guide/event-handlers.md b/docs/user-guide/event-handlers.md index 9f88f28..425698e 100644 --- a/docs/user-guide/event-handlers.md +++ b/docs/user-guide/event-handlers.md @@ -42,10 +42,11 @@ Called when the function completes successfully. def log_success(details): print(f"{details['target'].__name__} succeeded after {details['tries']} tries") + @backoff.on_exception( backoff.expo, Exception, - on_success=log_success + on_success=log_success, ) def my_function(): pass @@ -62,10 +63,11 @@ def log_backoff(details): f"(elapsed: {details['elapsed']:.1f}s)" ) + @backoff.on_exception( backoff.expo, Exception, - on_backoff=log_backoff + on_backoff=log_backoff, ) def my_function(): pass @@ -77,13 +79,14 @@ For `on_exception`, the exception is available: ```python def log_exception_backoff(details): - exc = details.get('exception') + exc = details.get("exception") print(f"Retrying due to: {type(exc).__name__}: {exc}") + @backoff.on_exception( backoff.expo, requests.exceptions.RequestException, - on_backoff=log_exception_backoff + on_backoff=log_exception_backoff, ) def api_call(): pass @@ -95,14 +98,15 @@ For `on_predicate`, the return value is available: ```python def log_value_backoff(details): - value = details.get('value') + value = details.get("value") print(f"Retrying because value was: {value}") + @backoff.on_predicate( backoff.constant, lambda x: x is None, on_backoff=log_value_backoff, - interval=2 + interval=2, ) def poll_resource(): pass @@ -119,11 +123,12 @@ def log_giveup(details): f"after {details['tries']} tries and {details['elapsed']:.1f}s" ) + @backoff.on_exception( backoff.expo, Exception, on_giveup=log_giveup, - max_tries=5 + max_tries=5, ) def my_function(): pass @@ -137,17 +142,24 @@ You can provide multiple handlers as a list: def log_to_console(details): print(f"Retry #{details['tries']}") + def log_to_file(details): - with open('retries.log', 'a') as f: + with open("retries.log", "a") as f: f.write(f"Retry #{details['tries']}\\n") + def send_metric(details): - metrics.increment('retry_count') + metrics.increment("retry_count") + @backoff.on_exception( backoff.expo, Exception, - on_backoff=[log_to_console, log_to_file, send_metric] + on_backoff=[ + log_to_console, + log_to_file, + send_metric, + ], ) def my_function(): pass @@ -163,19 +175,25 @@ import json logger = logging.getLogger(__name__) + def structured_log_backoff(details): - logger.warning(json.dumps({ - 'event': 'retry', - 'function': details['target'].__name__, - 'tries': details['tries'], - 'wait': details['wait'], - 'elapsed': details['elapsed'] - })) + logger.warning( + json.dumps( + { + "event": "retry", + "function": details["target"].__name__, + "tries": details["tries"], + "wait": details["wait"], + "elapsed": details["elapsed"], + } + ) + ) + @backoff.on_exception( backoff.expo, Exception, - on_backoff=structured_log_backoff + on_backoff=structured_log_backoff, ) def my_function(): pass @@ -186,17 +204,19 @@ def my_function(): ```python from prometheus_client import Counter, Histogram -retry_counter = Counter('backoff_retries_total', 'Total retries', ['function']) -retry_duration = Histogram('backoff_retry_duration_seconds', 'Retry duration') +retry_counter = Counter("backoff_retries_total", "Total retries", ["function"]) +retry_duration = Histogram("backoff_retry_duration_seconds", "Retry duration") + def record_metrics(details): - retry_counter.labels(function=details['target'].__name__).inc() - retry_duration.observe(details['elapsed']) + retry_counter.labels(function=details["target"].__name__).inc() + retry_duration.observe(details["elapsed"]) + @backoff.on_exception( backoff.expo, Exception, - on_backoff=record_metrics + on_backoff=record_metrics, ) def monitored_function(): pass @@ -207,18 +227,20 @@ def monitored_function(): ```python import sentry_sdk + def report_to_sentry(details): - if details['tries'] > 3: # Only report after 3 failures + if details["tries"] > 3: # Only report after 3 failures sentry_sdk.capture_message( f"Multiple retries for {details['target'].__name__}", - level='warning', - extra=details + level="warning", + extra=details, ) + @backoff.on_exception( backoff.expo, Exception, - on_backoff=report_to_sentry + on_backoff=report_to_sentry, ) def my_function(): pass @@ -228,17 +250,18 @@ def my_function(): ```python def alert_on_giveup(details): - if details['tries'] >= 5: + if details["tries"] >= 5: send_alert( f"Function {details['target'].__name__} failed " f"after {details['tries']} attempts" ) + @backoff.on_exception( backoff.expo, Exception, on_giveup=alert_on_giveup, - max_tries=5 + max_tries=5, ) def critical_function(): pass @@ -251,17 +274,16 @@ Event handlers can be async when used with async functions: ```python import aiohttp + async def async_log_backoff(details): async with aiohttp.ClientSession() as session: - await session.post( - 'http://log-service/events', - json=details - ) + await session.post("http://log-service/events", json=details) + @backoff.on_exception( backoff.expo, Exception, - on_backoff=async_log_backoff + on_backoff=async_log_backoff, ) async def async_function(): pass @@ -275,19 +297,21 @@ In `on_exception` handlers, you can access exception info: import sys import traceback + def detailed_exception_log(details): exc_type, exc_value, exc_tb = sys.exc_info() - tb_str = ''.join(traceback.format_tb(exc_tb)) + tb_str = "".join(traceback.format_tb(exc_tb)) logger.error( f"Retry {details['tries']} due to {exc_type.__name__}: {exc_value}\\n" f"Traceback:\\n{tb_str}" ) + @backoff.on_exception( backoff.expo, Exception, - on_backoff=detailed_exception_log + on_backoff=detailed_exception_log, ) def my_function(): pass @@ -300,18 +324,19 @@ Execute handler logic conditionally: ```python def conditional_alert(details): # Only alert after many retries - if details['tries'] >= 5: + if details["tries"] >= 5: send_alert(f"High retry count: {details['tries']}") # Only log errors, not warnings - if details.get('exception'): - if isinstance(details['exception'], CriticalError): + if details.get("exception"): + if isinstance(details["exception"], CriticalError): logger.error("Critical error during retry") + @backoff.on_exception( backoff.expo, Exception, - on_backoff=conditional_alert + on_backoff=conditional_alert, ) def my_function(): pass @@ -325,12 +350,14 @@ from datetime import datetime logger = logging.getLogger(__name__) + def log_attempt(details): logger.info( f"[{datetime.now()}] Attempt {details['tries']} " f"for {details['target'].__name__}" ) + def log_backoff(details): logger.warning( f"Backing off {details['wait']:.1f}s after {details['tries']} tries. " @@ -338,6 +365,7 @@ def log_backoff(details): f"Error: {details.get('exception', 'N/A')}" ) + def log_giveup(details): logger.error( f"Gave up on {details['target'].__name__} after " @@ -345,12 +373,14 @@ def log_giveup(details): f"Final error: {details.get('exception', 'N/A')}" ) + def log_success(details): logger.info( f"Success for {details['target'].__name__} after " f"{details['tries']} tries in {details['elapsed']:.1f}s" ) + @backoff.on_exception( backoff.expo, requests.exceptions.RequestException, @@ -358,7 +388,7 @@ def log_success(details): max_time=60, on_backoff=[log_attempt, log_backoff], on_giveup=log_giveup, - on_success=log_success + on_success=log_success, ) def comprehensive_retry(): return requests.get("https://api.example.com/data") diff --git a/docs/user-guide/logging.md b/docs/user-guide/logging.md index d001513..c2c9502 100644 --- a/docs/user-guide/logging.md +++ b/docs/user-guide/logging.md @@ -12,8 +12,8 @@ Backoff uses the `'backoff'` logger by default. It's configured with a `NullHand import logging # Enable backoff logging -logging.getLogger('backoff').addHandler(logging.StreamHandler()) -logging.getLogger('backoff').setLevel(logging.INFO) +logging.getLogger("backoff").addHandler(logging.StreamHandler()) +logging.getLogger("backoff").setLevel(logging.INFO) ``` ### Log Levels @@ -25,10 +25,10 @@ logging.getLogger('backoff').setLevel(logging.INFO) ```python # Only log when giving up -logging.getLogger('backoff').setLevel(logging.ERROR) +logging.getLogger("backoff").setLevel(logging.ERROR) # Log all retries -logging.getLogger('backoff').setLevel(logging.INFO) +logging.getLogger("backoff").setLevel(logging.INFO) ``` ## Custom Logger @@ -41,7 +41,7 @@ Specify a custom logger by name or instance. @backoff.on_exception( backoff.expo, Exception, - logger='my_custom_logger' + logger="my_custom_logger", ) def my_function(): pass @@ -52,14 +52,15 @@ def my_function(): ```python import logging -my_logger = logging.getLogger('my_app.retries') -my_logger.addHandler(logging.FileHandler('retries.log')) +my_logger = logging.getLogger("my_app.retries") +my_logger.addHandler(logging.FileHandler("retries.log")) my_logger.setLevel(logging.WARNING) + @backoff.on_exception( backoff.expo, Exception, - logger=my_logger + logger=my_logger, ) def my_function(): pass @@ -70,11 +71,7 @@ def my_function(): Pass `logger=None` to disable all default logging: ```python -@backoff.on_exception( - backoff.expo, - Exception, - logger=None -) +@backoff.on_exception(backoff.expo, Exception, logger=None) def my_function(): pass ``` @@ -85,11 +82,12 @@ Use with custom event handlers for complete control: def my_custom_log(details): print(f"Custom log: {details}") + @backoff.on_exception( backoff.expo, Exception, logger=None, - on_backoff=my_custom_log + on_backoff=my_custom_log, ) def my_function(): pass @@ -103,11 +101,11 @@ def my_function(): import logging logging.basicConfig( - format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', - level=logging.INFO + format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", + level=logging.INFO, ) -logging.getLogger('backoff').addHandler(logging.StreamHandler()) +logging.getLogger("backoff").addHandler(logging.StreamHandler()) ``` ### Structured Logging (JSON) @@ -116,20 +114,22 @@ logging.getLogger('backoff').addHandler(logging.StreamHandler()) import logging import json + class JsonFormatter(logging.Formatter): def format(self, record): log_data = { - 'timestamp': self.formatTime(record), - 'level': record.levelname, - 'logger': record.name, - 'message': record.getMessage() + "timestamp": self.formatTime(record), + "level": record.levelname, + "logger": record.name, + "message": record.getMessage(), } return json.dumps(log_data) + handler = logging.StreamHandler() handler.setFormatter(JsonFormatter()) -backoff_logger = logging.getLogger('backoff') +backoff_logger = logging.getLogger("backoff") backoff_logger.addHandler(handler) backoff_logger.setLevel(logging.INFO) ``` @@ -141,14 +141,14 @@ Send logs to multiple destinations: ```python import logging -backoff_logger = logging.getLogger('backoff') +backoff_logger = logging.getLogger("backoff") # Console handler console_handler = logging.StreamHandler() console_handler.setLevel(logging.WARNING) # File handler -file_handler = logging.FileHandler('backoff.log') +file_handler = logging.FileHandler("backoff.log") file_handler.setLevel(logging.INFO) # Add both handlers @@ -162,23 +162,25 @@ backoff_logger.setLevel(logging.INFO) Use different loggers for different functions: ```python -critical_logger = logging.getLogger('critical_ops') -routine_logger = logging.getLogger('routine_ops') +critical_logger = logging.getLogger("critical_ops") +routine_logger = logging.getLogger("routine_ops") + @backoff.on_exception( backoff.expo, Exception, logger=critical_logger, - max_tries=10 + max_tries=10, ) def critical_operation(): pass + @backoff.on_exception( backoff.expo, Exception, logger=routine_logger, - max_tries=3 + max_tries=3, ) def routine_operation(): pass @@ -191,39 +193,36 @@ import logging from logging.handlers import RotatingFileHandler # Create custom logger -logger = logging.getLogger('myapp.backoff') +logger = logging.getLogger("myapp.backoff") logger.setLevel(logging.INFO) # Console handler with WARNING level console_handler = logging.StreamHandler() console_handler.setLevel(logging.WARNING) -console_format = logging.Formatter( - '%(levelname)s: %(message)s' -) +console_format = logging.Formatter("%(levelname)s: %(message)s") console_handler.setFormatter(console_format) # File handler with INFO level and rotation file_handler = RotatingFileHandler( - 'backoff.log', - maxBytes=10*1024*1024, # 10MB - backupCount=5 + "backoff.log", + maxBytes=10 * 1024 * 1024, # 10MB + backupCount=5, ) file_handler.setLevel(logging.INFO) -file_format = logging.Formatter( - '%(asctime)s - %(name)s - %(levelname)s - %(message)s' -) +file_format = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s") file_handler.setFormatter(file_format) # Add handlers logger.addHandler(console_handler) logger.addHandler(file_handler) + # Use in decorator @backoff.on_exception( backoff.expo, Exception, logger=logger, - max_tries=5 + max_tries=5, ) def my_function(): pass diff --git a/docs/user-guide/wait-strategies.md b/docs/user-guide/wait-strategies.md index 3783b71..3ca28cc 100644 --- a/docs/user-guide/wait-strategies.md +++ b/docs/user-guide/wait-strategies.md @@ -9,6 +9,7 @@ Exponential backoff doubles the wait time after each retry. ```python import backoff + @backoff.on_exception(backoff.expo, Exception) def my_function(): pass @@ -29,12 +30,14 @@ def my_function(): @backoff.on_exception( backoff.expo, Exception, - base=2, # Start at 2 seconds - factor=3, # Triple each time - max_value=60 # Cap at 60 seconds + base=2, # Start at 2 seconds + factor=3, # Triple each time + max_value=60, # Cap at 60 seconds ) def custom_expo(): pass + + # Wait sequence: 2s, 6s, 18s, 54s, 60s, 60s, ... ``` @@ -67,10 +70,12 @@ def my_function(): @backoff.on_exception( backoff.fibo, Exception, - max_value=30 # Cap at 30 seconds + max_value=30, # Cap at 30 seconds ) def fibo_with_cap(): pass + + # Wait sequence: 1s, 1s, 2s, 3s, 5s, 8s, 13s, 21s, 30s, 30s, ... ``` @@ -88,7 +93,7 @@ Fixed wait time between all retries. @backoff.on_exception( backoff.constant, Exception, - interval=5 # Always wait 5 seconds + interval=5, # Always wait 5 seconds ) def my_function(): pass @@ -108,7 +113,7 @@ def my_function(): backoff.constant, interval=10, jitter=None, # Disable jitter for exact intervals - max_time=300 + max_time=300, ) def poll_every_10_seconds(): pass @@ -128,7 +133,7 @@ Dynamic wait time based on function return value or exception. @backoff.on_predicate( backoff.runtime, predicate=lambda r: r.status_code == 429, - value=lambda r: int(r.headers.get("Retry-After", 1)) + value=lambda r: int(r.headers.get("Retry-After", 1)), ) def respect_retry_after(): return requests.get(url) @@ -151,11 +156,12 @@ def get_retry_after(response): return int(retry_after) return 1 # Default + @backoff.on_predicate( backoff.runtime, predicate=lambda r: r.status_code == 429, value=get_retry_after, - jitter=None + jitter=None, ) def api_call(): return requests.get(api_url) @@ -169,10 +175,11 @@ class RetryableError(Exception): super().__init__(message) self.wait_seconds = wait_seconds + @backoff.on_exception( backoff.runtime, RetryableError, - value=lambda e: e.wait_seconds + value=lambda e: e.wait_seconds, ) def custom_retry(): raise RetryableError("Try again", wait_seconds=30) @@ -194,8 +201,18 @@ Uses AWS's Full Jitter algorithm - wait time is random between 0 and the calcula ```python @backoff.on_exception(backoff.expo, Exception) +def my_function(): + pass + + # Equivalent to: -@backoff.on_exception(backoff.expo, Exception, jitter=backoff.full_jitter) +@backoff.on_exception( + backoff.expo, + Exception, + jitter=backoff.full_jitter, +) +def my_function(): + pass ``` For exponential backoff: actual wait is random between 0 and 2^n seconds. @@ -205,7 +222,13 @@ For exponential backoff: actual wait is random between 0 and 2^n seconds. Adds random milliseconds (0-1000ms) to the calculated wait time. ```python -@backoff.on_exception(backoff.expo, Exception, jitter=backoff.random_jitter) +@backoff.on_exception( + backoff.expo, + Exception, + jitter=backoff.random_jitter, +) +def my_function(): + pass ``` ### Custom Jitter @@ -213,12 +236,18 @@ Adds random milliseconds (0-1000ms) to the calculated wait time. ```python import random + def custom_jitter(value): """Add 10-50% randomness""" jitter_amount = value * random.uniform(0.1, 0.5) return value + jitter_amount -@backoff.on_exception(backoff.expo, Exception, jitter=custom_jitter) + +@backoff.on_exception( + backoff.expo, + Exception, + jitter=custom_jitter, +) def my_function(): pass ``` @@ -227,6 +256,8 @@ def my_function(): ```python @backoff.on_exception(backoff.expo, Exception, jitter=None) +def my_function(): + pass ``` ## Comparison diff --git a/pyproject.toml b/pyproject.toml index 59dceb3..6e8b25d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -167,6 +167,15 @@ package = [ [tool.ruff] line-length = 88 +[tool.ruff.format] +docstring-code-format = true +docstring-code-line-length = 20 + +[tool.ruff.lint] +extend-select = [ + "SIM", # flake8-simplify +] + [tool.coverage.report] fail_under = 100 show_missing = true @@ -179,11 +188,6 @@ source = [ "backoff", ] -[tool.ruff.lint] -extend-select = [ - "COM", # flake8-commas -] - [build-system] requires = ["hatchling"] build-backend = "hatchling.build" diff --git a/tests/common.py b/tests/common.py index 56c2a8d..d9a1a77 100644 --- a/tests/common.py +++ b/tests/common.py @@ -10,9 +10,9 @@ def _log_hdlrs(): def log_hdlr(event, details): log[event].append(details) - log_success = functools.partial(log_hdlr, 'success') - log_backoff = functools.partial(log_hdlr, 'backoff') - log_giveup = functools.partial(log_hdlr, 'giveup') + log_success = functools.partial(log_hdlr, "success") + log_backoff = functools.partial(log_hdlr, "backoff") + log_giveup = functools.partial(log_hdlr, "giveup") return log, log_success, log_backoff, log_giveup diff --git a/tests/test_backoff.py b/tests/test_backoff.py index d68cc4e..6d0aaaa 100644 --- a/tests/test_backoff.py +++ b/tests/test_backoff.py @@ -15,33 +15,33 @@ def test_on_predicate(monkeypatch): - monkeypatch.setattr('time.sleep', lambda x: None) + monkeypatch.setattr("time.sleep", lambda x: None) @backoff.on_predicate(backoff.expo) def return_true(log, n): - val = (len(log) == n - 1) + val = len(log) == n - 1 log.append(val) return val log = [] ret = return_true(log, 3) assert ret is True - assert 3 == len(log) + assert len(log) == 3 def test_on_predicate_max_tries(monkeypatch): - monkeypatch.setattr('time.sleep', lambda x: None) + monkeypatch.setattr("time.sleep", lambda x: None) @backoff.on_predicate(backoff.expo, jitter=None, max_tries=3) def return_true(log, n): - val = (len(log) == n) + val = len(log) == n log.append(val) return val log = [] ret = return_true(log, 10) assert ret is False - assert 3 == len(log) + assert len(log) == 3 def test_on_predicate_max_time(monkeypatch): @@ -55,17 +55,16 @@ def test_on_predicate_max_time(monkeypatch): def monotonic(): return nows.pop() - monkeypatch.setattr('time.sleep', lambda x: None) - monkeypatch.setattr('time.monotonic', monotonic) + monkeypatch.setattr("time.sleep", lambda x: None) + monkeypatch.setattr("time.monotonic", monotonic) def giveup(details): - assert details['tries'] == 3 - assert details['elapsed'] == 10.000005 + assert details["tries"] == 3 + assert details["elapsed"] == 10.000005 - @backoff.on_predicate(backoff.expo, jitter=None, max_time=10, - on_giveup=giveup) + @backoff.on_predicate(backoff.expo, jitter=None, max_time=10, on_giveup=giveup) def return_true(log, n): - val = (len(log) == n) + val = len(log) == n log.append(val) return val @@ -86,20 +85,21 @@ def test_on_predicate_max_time_callable(monkeypatch): def monotonic(): return nows.pop() - monkeypatch.setattr('time.sleep', lambda x: None) - monkeypatch.setattr('time.monotonic', monotonic) + monkeypatch.setattr("time.sleep", lambda x: None) + monkeypatch.setattr("time.monotonic", monotonic) def giveup(details): - assert details['tries'] == 3 - assert details['elapsed'] == 10.000005 + assert details["tries"] == 3 + assert details["elapsed"] == 10.000005 def lookup_max_time(): return 10 - @backoff.on_predicate(backoff.expo, jitter=None, max_time=lookup_max_time, - on_giveup=giveup) + @backoff.on_predicate( + backoff.expo, jitter=None, max_time=lookup_max_time, on_giveup=giveup + ) def return_true(log, n): - val = (len(log) == n) + val = len(log) == n log.append(val) return val @@ -110,7 +110,7 @@ def return_true(log, n): def test_on_exception(monkeypatch): - monkeypatch.setattr('time.sleep', lambda x: None) + monkeypatch.setattr("time.sleep", lambda x: None) @backoff.on_exception(backoff.expo, KeyError) def keyerror_then_true(log, n): @@ -122,11 +122,11 @@ def keyerror_then_true(log, n): log = [] assert keyerror_then_true(log, 3) is True - assert 3 == len(log) + assert len(log) == 3 def test_on_exception_tuple(monkeypatch): - monkeypatch.setattr('time.sleep', lambda x: None) + monkeypatch.setattr("time.sleep", lambda x: None) @backoff.on_exception(backoff.expo, (KeyError, ValueError)) def keyerror_valueerror_then_true(log): @@ -141,13 +141,13 @@ def keyerror_valueerror_then_true(log): log = [] assert keyerror_valueerror_then_true(log) is True - assert 2 == len(log) + assert len(log) == 2 assert isinstance(log[0], KeyError) assert isinstance(log[1], ValueError) def test_on_exception_max_tries(monkeypatch): - monkeypatch.setattr('time.sleep', lambda x: None) + monkeypatch.setattr("time.sleep", lambda x: None) @backoff.on_exception(backoff.expo, KeyError, jitter=None, max_tries=3) def keyerror_then_true(log, n, foo=None): @@ -161,14 +161,13 @@ def keyerror_then_true(log, n, foo=None): with pytest.raises(KeyError): keyerror_then_true(log, 10, foo="bar") - assert 3 == len(log) + assert len(log) == 3 def test_on_exception_max_tries_callable(monkeypatch): - monkeypatch.setattr('time.sleep', lambda x: None) + monkeypatch.setattr("time.sleep", lambda x: None) - @backoff.on_exception(backoff.expo, KeyError, jitter=None, - max_tries=lambda: 3) + @backoff.on_exception(backoff.expo, KeyError, jitter=None, max_tries=lambda: 3) def keyerror_then_true(log, n, foo=None): if len(log) == n: return True @@ -180,11 +179,11 @@ def keyerror_then_true(log, n, foo=None): with pytest.raises(KeyError): keyerror_then_true(log, 10, foo="bar") - assert 3 == len(log) + assert len(log) == 3 def test_on_exception_constant_iterable(monkeypatch): - monkeypatch.setattr('time.sleep', lambda x: None) + monkeypatch.setattr("time.sleep", lambda x: None) backoffs = [] giveups = [] @@ -211,7 +210,6 @@ def on_success(details: Details): successes.append(details) - @backoff.on_exception( backoff.constant, KeyError, @@ -221,7 +219,7 @@ def on_success(details: Details): on_success=on_success, ) def endless_exceptions(): - raise KeyError('foo') + raise KeyError("foo") with pytest.raises(KeyError): endless_exceptions() @@ -232,17 +230,19 @@ def endless_exceptions(): def test_on_exception_success_random_jitter(monkeypatch): - monkeypatch.setattr('time.sleep', lambda x: None) + monkeypatch.setattr("time.sleep", lambda x: None) backoffs, giveups, successes = [], [], [] - @backoff.on_exception(backoff.expo, - Exception, - on_success=successes.append, - on_backoff=backoffs.append, - on_giveup=giveups.append, - jitter=backoff.random_jitter, - factor=0.5) + @backoff.on_exception( + backoff.expo, + Exception, + on_success=successes.append, + on_backoff=backoffs.append, + on_giveup=giveups.append, + jitter=backoff.random_jitter, + factor=0.5, + ) @_save_target def succeeder(*args, **kwargs): # succeed after we've backed off twice @@ -258,21 +258,23 @@ def succeeder(*args, **kwargs): for i in range(2): details = backoffs[i] - assert details['wait'] >= 0.5 * 2 ** i + assert details["wait"] >= 0.5 * 2**i def test_on_exception_success_full_jitter(monkeypatch): - monkeypatch.setattr('time.sleep', lambda x: None) + monkeypatch.setattr("time.sleep", lambda x: None) backoffs, giveups, successes = [], [], [] - @backoff.on_exception(backoff.expo, - Exception, - on_success=successes.append, - on_backoff=backoffs.append, - on_giveup=giveups.append, - jitter=backoff.full_jitter, - factor=0.5) + @backoff.on_exception( + backoff.expo, + Exception, + on_success=successes.append, + on_backoff=backoffs.append, + on_giveup=giveups.append, + jitter=backoff.full_jitter, + factor=0.5, + ) @_save_target def succeeder(*args, **kwargs): # succeed after we've backed off twice @@ -288,19 +290,21 @@ def succeeder(*args, **kwargs): for i in range(2): details = backoffs[i] - assert details['wait'] <= 0.5 * 2 ** i + assert details["wait"] <= 0.5 * 2**i def test_on_exception_success(): backoffs, giveups, successes = [], [], [] - @backoff.on_exception(backoff.constant, - Exception, - on_success=successes.append, - on_backoff=backoffs.append, - on_giveup=giveups.append, - jitter=None, - interval=0) + @backoff.on_exception( + backoff.constant, + Exception, + on_success=successes.append, + on_backoff=backoffs.append, + on_giveup=giveups.append, + jitter=None, + interval=0, + ) @_save_target def succeeder(*args, **kwargs): # succeed after we've backed off twice @@ -316,38 +320,44 @@ def succeeder(*args, **kwargs): for i in range(2): details = backoffs[i] - elapsed = details.pop('elapsed') - exception = details.pop('exception') + elapsed = details.pop("elapsed") + exception = details.pop("exception") assert isinstance(elapsed, float) assert isinstance(exception, ValueError) - assert details == {'args': (1, 2, 3), - 'kwargs': {'foo': 1, 'bar': 2}, - 'target': succeeder._target, - 'tries': i + 1, - 'wait': 0} + assert details == { + "args": (1, 2, 3), + "kwargs": {"foo": 1, "bar": 2}, + "target": succeeder._target, + "tries": i + 1, + "wait": 0, + } details = successes[0] - elapsed = details.pop('elapsed') + elapsed = details.pop("elapsed") assert isinstance(elapsed, float) - assert details == {'args': (1, 2, 3), - 'kwargs': {'foo': 1, 'bar': 2}, - 'target': succeeder._target, - 'tries': 3} + assert details == { + "args": (1, 2, 3), + "kwargs": {"foo": 1, "bar": 2}, + "target": succeeder._target, + "tries": 3, + } -@pytest.mark.parametrize('raise_on_giveup', [True, False]) +@pytest.mark.parametrize("raise_on_giveup", [True, False]) def test_on_exception_giveup(raise_on_giveup): backoffs, giveups, successes = [], [], [] - @backoff.on_exception(backoff.constant, - ValueError, - on_success=successes.append, - on_backoff=backoffs.append, - on_giveup=giveups.append, - max_tries=3, - jitter=None, - raise_on_giveup=raise_on_giveup, - interval=0) + @backoff.on_exception( + backoff.constant, + ValueError, + on_success=successes.append, + on_backoff=backoffs.append, + on_giveup=giveups.append, + max_tries=3, + jitter=None, + raise_on_giveup=raise_on_giveup, + interval=0, + ) @_save_target def exceptor(*args, **kwargs): raise ValueError("catch me") @@ -364,27 +374,27 @@ def exceptor(*args, **kwargs): assert len(giveups) == 1 details = giveups[0] - elapsed = details.pop('elapsed') - exception = details.pop('exception') + elapsed = details.pop("elapsed") + exception = details.pop("exception") assert isinstance(elapsed, float) assert isinstance(exception, ValueError) - assert details == {'args': (1, 2, 3), - 'kwargs': {'foo': 1, 'bar': 2}, - 'target': exceptor._target, - 'tries': 3} + assert details == { + "args": (1, 2, 3), + "kwargs": {"foo": 1, "bar": 2}, + "target": exceptor._target, + "tries": 3, + } def test_on_exception_giveup_predicate(monkeypatch): - monkeypatch.setattr('time.sleep', lambda x: None) + monkeypatch.setattr("time.sleep", lambda x: None) def on_baz(e): return str(e) == "baz" vals = ["baz", "bar", "foo"] - @backoff.on_exception(backoff.constant, - ValueError, - giveup=on_baz) + @backoff.on_exception(backoff.constant, ValueError, giveup=on_baz) def foo_bar_baz(): raise ValueError(vals.pop()) @@ -397,12 +407,14 @@ def foo_bar_baz(): def test_on_predicate_success(): backoffs, giveups, successes = [], [], [] - @backoff.on_predicate(backoff.constant, - on_success=successes.append, - on_backoff=backoffs.append, - on_giveup=giveups.append, - jitter=None, - interval=0) + @backoff.on_predicate( + backoff.constant, + on_success=successes.append, + on_backoff=backoffs.append, + on_giveup=giveups.append, + jitter=None, + interval=0, + ) @_save_target def success(*args, **kwargs): # succeed after we've backed off twice @@ -418,35 +430,41 @@ def success(*args, **kwargs): for i in range(2): details = backoffs[i] - elapsed = details.pop('elapsed') + elapsed = details.pop("elapsed") assert isinstance(elapsed, float) - assert details == {'args': (1, 2, 3), - 'kwargs': {'foo': 1, 'bar': 2}, - 'target': success._target, - 'tries': i + 1, - 'value': False, - 'wait': 0} + assert details == { + "args": (1, 2, 3), + "kwargs": {"foo": 1, "bar": 2}, + "target": success._target, + "tries": i + 1, + "value": False, + "wait": 0, + } details = successes[0] - elapsed = details.pop('elapsed') + elapsed = details.pop("elapsed") assert isinstance(elapsed, float) - assert details == {'args': (1, 2, 3), - 'kwargs': {'foo': 1, 'bar': 2}, - 'target': success._target, - 'tries': 3, - 'value': True} + assert details == { + "args": (1, 2, 3), + "kwargs": {"foo": 1, "bar": 2}, + "target": success._target, + "tries": 3, + "value": True, + } def test_on_predicate_giveup(): backoffs, giveups, successes = [], [], [] - @backoff.on_predicate(backoff.constant, - on_success=successes.append, - on_backoff=backoffs.append, - on_giveup=giveups.append, - max_tries=3, - jitter=None, - interval=0) + @backoff.on_predicate( + backoff.constant, + on_success=successes.append, + on_backoff=backoffs.append, + on_giveup=giveups.append, + max_tries=3, + jitter=None, + interval=0, + ) @_save_target def emptiness(*args, **kwargs): pass @@ -459,13 +477,15 @@ def emptiness(*args, **kwargs): assert len(giveups) == 1 details = giveups[0] - elapsed = details.pop('elapsed') + elapsed = details.pop("elapsed") assert isinstance(elapsed, float) - assert details == {'args': (1, 2, 3), - 'kwargs': {'foo': 1, 'bar': 2}, - 'target': emptiness._target, - 'tries': 3, - 'value': None} + assert details == { + "args": (1, 2, 3), + "kwargs": {"foo": 1, "bar": 2}, + "target": emptiness._target, + "tries": 3, + "value": None, + } def test_on_predicate_iterable_handlers(): @@ -477,13 +497,15 @@ def __init__(self): loggers = [Logger() for _ in range(3)] - @backoff.on_predicate(backoff.constant, - on_backoff=(l.backoffs.append for l in loggers), # noqa: E741 - on_giveup=(l.giveups.append for l in loggers), # noqa: E741 - on_success=(l.successes.append for l in loggers), # noqa: E741 - max_tries=3, - jitter=None, - interval=0) + @backoff.on_predicate( + backoff.constant, + on_backoff=(l.backoffs.append for l in loggers), # noqa: E741 + on_giveup=(l.giveups.append for l in loggers), # noqa: E741 + on_success=(l.successes.append for l in loggers), # noqa: E741 + max_tries=3, + jitter=None, + interval=0, + ) @_save_target def emptiness(*args, **kwargs): pass @@ -491,36 +513,39 @@ def emptiness(*args, **kwargs): emptiness(1, 2, 3, foo=1, bar=2) for logger in loggers: - assert len(logger.successes) == 0 assert len(logger.backoffs) == 2 assert len(logger.giveups) == 1 details = dict(logger.giveups[0]) - elapsed = details.pop('elapsed') + elapsed = details.pop("elapsed") assert isinstance(elapsed, float) - assert details == {'args': (1, 2, 3), - 'kwargs': {'foo': 1, 'bar': 2}, - 'target': emptiness._target, - 'tries': 3, - 'value': None} + assert details == { + "args": (1, 2, 3), + "kwargs": {"foo": 1, "bar": 2}, + "target": emptiness._target, + "tries": 3, + "value": None, + } # To maintain backward compatibility, # on_predicate should support 0-argument jitter function. def test_on_exception_success_0_arg_jitter(monkeypatch): - monkeypatch.setattr('time.sleep', lambda x: None) - monkeypatch.setattr('random.random', lambda: 0) + monkeypatch.setattr("time.sleep", lambda x: None) + monkeypatch.setattr("random.random", lambda: 0) backoffs, giveups, successes = [], [], [] - @backoff.on_exception(backoff.constant, - Exception, - on_success=successes.append, - on_backoff=backoffs.append, - on_giveup=giveups.append, - jitter=random.random, - interval=0) + @backoff.on_exception( + backoff.constant, + Exception, + on_success=successes.append, + on_backoff=backoffs.append, + on_giveup=giveups.append, + jitter=random.random, + interval=0, + ) @_save_target def succeeder(*args, **kwargs): # succeed after we've backed off twice @@ -537,39 +562,45 @@ def succeeder(*args, **kwargs): for i in range(2): details = backoffs[i] - elapsed = details.pop('elapsed') - exception = details.pop('exception') + elapsed = details.pop("elapsed") + exception = details.pop("exception") assert isinstance(elapsed, float) assert isinstance(exception, ValueError) - assert details == {'args': (1, 2, 3), - 'kwargs': {'foo': 1, 'bar': 2}, - 'target': succeeder._target, - 'tries': i + 1, - 'wait': 0} + assert details == { + "args": (1, 2, 3), + "kwargs": {"foo": 1, "bar": 2}, + "target": succeeder._target, + "tries": i + 1, + "wait": 0, + } details = successes[0] - elapsed = details.pop('elapsed') + elapsed = details.pop("elapsed") assert isinstance(elapsed, float) - assert details == {'args': (1, 2, 3), - 'kwargs': {'foo': 1, 'bar': 2}, - 'target': succeeder._target, - 'tries': 3} + assert details == { + "args": (1, 2, 3), + "kwargs": {"foo": 1, "bar": 2}, + "target": succeeder._target, + "tries": 3, + } # To maintain backward compatibility, # on_predicate should support 0-argument jitter function. def test_on_predicate_success_0_arg_jitter(monkeypatch): - monkeypatch.setattr('time.sleep', lambda x: None) - monkeypatch.setattr('random.random', lambda: 0) + monkeypatch.setattr("time.sleep", lambda x: None) + monkeypatch.setattr("random.random", lambda: 0) backoffs, giveups, successes = [], [], [] - @backoff.on_predicate(backoff.constant, - on_success=successes.append, - on_backoff=backoffs.append, - on_giveup=giveups.append, - jitter=random.random, - interval=0) + @backoff.on_predicate( + backoff.constant, + on_success=successes.append, + on_backoff=backoffs.append, + on_giveup=giveups.append, + jitter=random.random, + interval=0, + ) @_save_target def success(*args, **kwargs): # succeed after we've backed off twice @@ -585,27 +616,31 @@ def success(*args, **kwargs): for i in range(2): details = backoffs[i] - elapsed = details.pop('elapsed') + elapsed = details.pop("elapsed") assert isinstance(elapsed, float) - assert details == {'args': (1, 2, 3), - 'kwargs': {'foo': 1, 'bar': 2}, - 'target': success._target, - 'tries': i + 1, - 'value': False, - 'wait': 0} + assert details == { + "args": (1, 2, 3), + "kwargs": {"foo": 1, "bar": 2}, + "target": success._target, + "tries": i + 1, + "value": False, + "wait": 0, + } details = successes[0] - elapsed = details.pop('elapsed') + elapsed = details.pop("elapsed") assert isinstance(elapsed, float) - assert details == {'args': (1, 2, 3), - 'kwargs': {'foo': 1, 'bar': 2}, - 'target': success._target, - 'tries': 3, - 'value': True} + assert details == { + "args": (1, 2, 3), + "kwargs": {"foo": 1, "bar": 2}, + "target": success._target, + "tries": 3, + "value": True, + } def test_on_exception_callable_max_tries(monkeypatch): - monkeypatch.setattr('time.sleep', lambda x: None) + monkeypatch.setattr("time.sleep", lambda x: None) log = [] @@ -621,7 +656,7 @@ def exceptor(): def test_on_exception_callable_max_tries_reads_every_time(monkeypatch): - monkeypatch.setattr('time.sleep', lambda x: None) + monkeypatch.setattr("time.sleep", lambda x: None) lookups = [] @@ -629,9 +664,7 @@ def lookup_max_tries(): lookups.append(True) return 3 - @backoff.on_exception(backoff.constant, - ValueError, - max_tries=lookup_max_tries) + @backoff.on_exception(backoff.constant, ValueError, max_tries=lookup_max_tries) def exceptor(): raise ValueError() @@ -645,7 +678,6 @@ def exceptor(): def test_on_exception_callable_gen_kwargs(): - def lookup_foo(): return "foo" @@ -656,11 +688,7 @@ def wait_gen(foo=None, bar=None): while True: yield 0 - @backoff.on_exception(wait_gen, - ValueError, - max_tries=2, - foo=lookup_foo, - bar="bar") + @backoff.on_exception(wait_gen, ValueError, max_tries=2, foo=lookup_foo, bar="bar") def exceptor(): raise ValueError("aah") @@ -669,38 +697,39 @@ def exceptor(): def test_on_predicate_in_thread(monkeypatch): - monkeypatch.setattr('time.sleep', lambda x: None) + monkeypatch.setattr("time.sleep", lambda x: None) result = [] def check(): try: + @backoff.on_predicate(backoff.expo) def return_true(log, n): - val = (len(log) == n - 1) + val = len(log) == n - 1 log.append(val) return val log = [] ret = return_true(log, 3) assert ret is True - assert 3 == len(log) + assert len(log) == 3 except Exception as ex: result.append(ex) else: - result.append('success') + result.append("success") t = threading.Thread(target=check) t.start() t.join() assert len(result) == 1 - assert result[0] == 'success' + assert result[0] == "success" def test_on_predicate_constant_iterable(monkeypatch): - monkeypatch.setattr('time.sleep', lambda x: None) + monkeypatch.setattr("time.sleep", lambda x: None) waits = [1, 2, 3, 6, 9] backoffs = [] @@ -722,19 +751,20 @@ def falsey(): assert len(backoffs) == len(waits) for i, wait in enumerate(waits): - assert backoffs[i]['wait'] == wait + assert backoffs[i]["wait"] == wait assert len(giveups) == 1 assert len(successes) == 0 def test_on_exception_in_thread(monkeypatch): - monkeypatch.setattr('time.sleep', lambda x: None) + monkeypatch.setattr("time.sleep", lambda x: None) result = [] def check(): try: + @backoff.on_exception(backoff.expo, KeyError) def keyerror_then_true(log, n): if len(log) == n: @@ -745,25 +775,25 @@ def keyerror_then_true(log, n): log = [] assert keyerror_then_true(log, 3) is True - assert 3 == len(log) + assert len(log) == 3 except Exception as ex: result.append(ex) else: - result.append('success') + result.append("success") t = threading.Thread(target=check) t.start() t.join() assert len(result) == 1 - assert result[0] == 'success' + assert result[0] == "success" def test_on_exception_logger_default(monkeypatch, caplog): - monkeypatch.setattr('time.sleep', lambda x: None) + monkeypatch.setattr("time.sleep", lambda x: None) - logger = logging.getLogger('backoff') + logger = logging.getLogger("backoff") handler = logging.StreamHandler(sys.stdout) logger.addHandler(handler) @@ -771,19 +801,18 @@ def test_on_exception_logger_default(monkeypatch, caplog): def key_error(): raise KeyError() - with caplog.at_level(logging.INFO): - with pytest.raises(KeyError): - key_error() + with caplog.at_level(logging.INFO), pytest.raises(KeyError): + key_error() assert len(caplog.records) == 3 # 2 backoffs and 1 giveup for record in caplog.records: - assert record.name == 'backoff' + assert record.name == "backoff" def test_on_exception_logger_none(monkeypatch, caplog): - monkeypatch.setattr('time.sleep', lambda x: None) + monkeypatch.setattr("time.sleep", lambda x: None) - logger = logging.getLogger('backoff') + logger = logging.getLogger("backoff") handler = logging.StreamHandler(sys.stdout) logger.addHandler(handler) @@ -791,17 +820,16 @@ def test_on_exception_logger_none(monkeypatch, caplog): def key_error(): raise KeyError() - with caplog.at_level(logging.INFO): - with pytest.raises(KeyError): - key_error() + with caplog.at_level(logging.INFO), pytest.raises(KeyError): + key_error() assert not caplog.records def test_on_exception_logger_user(monkeypatch, caplog): - monkeypatch.setattr('time.sleep', lambda x: None) + monkeypatch.setattr("time.sleep", lambda x: None) - logger = logging.getLogger('my-logger') + logger = logging.getLogger("my-logger") handler = logging.StreamHandler(sys.stdout) logger.addHandler(handler) @@ -809,38 +837,37 @@ def test_on_exception_logger_user(monkeypatch, caplog): def key_error(): raise KeyError() - with caplog.at_level(logging.INFO): - with pytest.raises(KeyError): - key_error() + with caplog.at_level(logging.INFO), pytest.raises(KeyError): + key_error() assert len(caplog.records) == 3 # 2 backoffs and 1 giveup for record in caplog.records: - assert record.name == 'my-logger' + assert record.name == "my-logger" def test_on_exception_logger_user_str(monkeypatch, caplog): - monkeypatch.setattr('time.sleep', lambda x: None) + monkeypatch.setattr("time.sleep", lambda x: None) - logger = logging.getLogger('my-logger') + logger = logging.getLogger("my-logger") handler = logging.StreamHandler(sys.stdout) logger.addHandler(handler) - @backoff.on_exception(backoff.expo, KeyError, max_tries=3, - logger='my-logger') + @backoff.on_exception(backoff.expo, KeyError, max_tries=3, logger="my-logger") def key_error(): raise KeyError() - with caplog.at_level(logging.INFO): - with pytest.raises(KeyError): - key_error() + with caplog.at_level(logging.INFO), pytest.raises(KeyError): + key_error() assert len(caplog.records) == 3 # 2 backoffs and 1 giveup for record in caplog.records: - assert record.name == 'my-logger' + assert record.name == "my-logger" def _on_exception_factory( - backoff_log_level, giveup_log_level, max_tries, + backoff_log_level, + giveup_log_level, + max_tries, ): @backoff.on_exception( backoff.expo, @@ -860,7 +887,9 @@ def func(): def _on_predicate_factory( - backoff_log_level, giveup_log_level, max_tries, + backoff_log_level, + giveup_log_level, + max_tries, ): @backoff.on_predicate( backoff.expo, @@ -892,16 +921,19 @@ def func(): ), ) def test_event_log_levels( - caplog, func_factory, backoff_log_level, giveup_log_level, + caplog, + func_factory, + backoff_log_level, + giveup_log_level, ): max_tries = 3 func = func_factory(backoff_log_level, giveup_log_level, max_tries) - with unittest.mock.patch('time.sleep', return_value=None): - with caplog.at_level( - min(backoff_log_level, giveup_log_level), logger="backoff", - ): - func() + with unittest.mock.patch("time.sleep", return_value=None), caplog.at_level( + min(backoff_log_level, giveup_log_level), + logger="backoff", + ): + func() backoff_re = re.compile("backing off", re.IGNORECASE) giveup_re = re.compile("giving up", re.IGNORECASE) diff --git a/tests/test_backoff_async.py b/tests/test_backoff_async.py index dbdb0fb..2199e97 100644 --- a/tests/test_backoff_async.py +++ b/tests/test_backoff_async.py @@ -15,55 +15,55 @@ async def _await_none(x): @pytest.mark.asyncio async def test_on_predicate(monkeypatch): - monkeypatch.setattr('asyncio.sleep', _await_none) + monkeypatch.setattr("asyncio.sleep", _await_none) @backoff.on_predicate(backoff.expo) async def return_true(log, n): - val = (len(log) == n - 1) + val = len(log) == n - 1 log.append(val) return val log = [] ret = await return_true(log, 3) assert ret is True - assert 3 == len(log) + assert len(log) == 3 @pytest.mark.asyncio async def test_on_predicate_max_tries(monkeypatch): - monkeypatch.setattr('asyncio.sleep', _await_none) + monkeypatch.setattr("asyncio.sleep", _await_none) @backoff.on_predicate(backoff.expo, jitter=None, max_tries=3) async def return_true(log, n): - val = (len(log) == n) + val = len(log) == n log.append(val) return val log = [] ret = await return_true(log, 10) assert ret is False - assert 3 == len(log) + assert len(log) == 3 @pytest.mark.asyncio async def test_on_predicate_max_tries_callable(monkeypatch): - monkeypatch.setattr('asyncio.sleep', _await_none) + monkeypatch.setattr("asyncio.sleep", _await_none) @backoff.on_predicate(backoff.expo, jitter=None, max_tries=lambda: 3) async def return_true(log, n): - val = (len(log) == n) + val = len(log) == n log.append(val) return val log = [] ret = await return_true(log, 10) assert ret is False - assert 3 == len(log) + assert len(log) == 3 @pytest.mark.asyncio async def test_on_exception(monkeypatch): - monkeypatch.setattr('asyncio.sleep', _await_none) + monkeypatch.setattr("asyncio.sleep", _await_none) @backoff.on_exception(backoff.expo, KeyError) async def keyerror_then_true(log, n): @@ -75,12 +75,12 @@ async def keyerror_then_true(log, n): log = [] assert (await keyerror_then_true(log, 3)) is True - assert 3 == len(log) + assert len(log) == 3 @pytest.mark.asyncio async def test_on_exception_tuple(monkeypatch): - monkeypatch.setattr('asyncio.sleep', _await_none) + monkeypatch.setattr("asyncio.sleep", _await_none) @backoff.on_exception(backoff.expo, (KeyError, ValueError)) async def keyerror_valueerror_then_true(log): @@ -95,14 +95,14 @@ async def keyerror_valueerror_then_true(log): log = [] assert (await keyerror_valueerror_then_true(log)) is True - assert 2 == len(log) + assert len(log) == 2 assert isinstance(log[0], KeyError) assert isinstance(log[1], ValueError) @pytest.mark.asyncio async def test_on_exception_max_tries(monkeypatch): - monkeypatch.setattr('asyncio.sleep', _await_none) + monkeypatch.setattr("asyncio.sleep", _await_none) @backoff.on_exception(backoff.expo, KeyError, jitter=None, max_tries=3) async def keyerror_then_true(log, n, foo=None): @@ -116,15 +116,14 @@ async def keyerror_then_true(log, n, foo=None): with pytest.raises(KeyError): await keyerror_then_true(log, 10, foo="bar") - assert 3 == len(log) + assert len(log) == 3 @pytest.mark.asyncio async def test_on_exception_max_tries_callable(monkeypatch): - monkeypatch.setattr('asyncio.sleep', _await_none) + monkeypatch.setattr("asyncio.sleep", _await_none) - @backoff.on_exception(backoff.expo, KeyError, jitter=None, - max_tries=lambda: 3) + @backoff.on_exception(backoff.expo, KeyError, jitter=None, max_tries=lambda: 3) async def keyerror_then_true(log, n, foo=None): if len(log) == n: return True @@ -136,12 +135,12 @@ async def keyerror_then_true(log, n, foo=None): with pytest.raises(KeyError): await keyerror_then_true(log, 10, foo="bar") - assert 3 == len(log) + assert len(log) == 3 @pytest.mark.asyncio async def test_on_exception_constant_iterable(monkeypatch): - monkeypatch.setattr('asyncio.sleep', _await_none) + monkeypatch.setattr("asyncio.sleep", _await_none) backoffs = [] giveups = [] @@ -156,7 +155,7 @@ async def test_on_exception_constant_iterable(monkeypatch): on_success=successes.append, ) async def endless_exceptions(): - raise KeyError('foo') + raise KeyError("foo") with pytest.raises(KeyError): await endless_exceptions() @@ -168,125 +167,137 @@ async def endless_exceptions(): @pytest.mark.asyncio async def test_on_exception_success_random_jitter(monkeypatch): - monkeypatch.setattr('asyncio.sleep', _await_none) + monkeypatch.setattr("asyncio.sleep", _await_none) log, log_success, log_backoff, log_giveup = _log_hdlrs() - @backoff.on_exception(backoff.expo, - Exception, - on_success=log_success, - on_backoff=log_backoff, - on_giveup=log_giveup, - jitter=backoff.random_jitter, - factor=0.5) + @backoff.on_exception( + backoff.expo, + Exception, + on_success=log_success, + on_backoff=log_backoff, + on_giveup=log_giveup, + jitter=backoff.random_jitter, + factor=0.5, + ) @_save_target async def succeeder(*args, **kwargs): # succeed after we've backed off twice - if len(log['backoff']) < 2: + if len(log["backoff"]) < 2: raise ValueError("catch me") await succeeder(1, 2, 3, foo=1, bar=2) # we try 3 times, backing off twice before succeeding - assert len(log['success']) == 1 - assert len(log['backoff']) == 2 - assert len(log['giveup']) == 0 + assert len(log["success"]) == 1 + assert len(log["backoff"]) == 2 + assert len(log["giveup"]) == 0 for i in range(2): - details = log['backoff'][i] - assert details['wait'] >= 0.5 * 2 ** i + details = log["backoff"][i] + assert details["wait"] >= 0.5 * 2**i @pytest.mark.asyncio async def test_on_exception_success_full_jitter(monkeypatch): - monkeypatch.setattr('asyncio.sleep', _await_none) + monkeypatch.setattr("asyncio.sleep", _await_none) log, log_success, log_backoff, log_giveup = _log_hdlrs() - @backoff.on_exception(backoff.expo, - Exception, - on_success=log_success, - on_backoff=log_backoff, - on_giveup=log_giveup, - jitter=backoff.full_jitter, - factor=0.5) + @backoff.on_exception( + backoff.expo, + Exception, + on_success=log_success, + on_backoff=log_backoff, + on_giveup=log_giveup, + jitter=backoff.full_jitter, + factor=0.5, + ) @_save_target async def succeeder(*args, **kwargs): # succeed after we've backed off twice - if len(log['backoff']) < 2: + if len(log["backoff"]) < 2: raise ValueError("catch me") await succeeder(1, 2, 3, foo=1, bar=2) # we try 3 times, backing off twice before succeeding - assert len(log['success']) == 1 - assert len(log['backoff']) == 2 - assert len(log['giveup']) == 0 + assert len(log["success"]) == 1 + assert len(log["backoff"]) == 2 + assert len(log["giveup"]) == 0 for i in range(2): - details = log['backoff'][i] - assert details['wait'] <= 0.5 * 2 ** i + details = log["backoff"][i] + assert details["wait"] <= 0.5 * 2**i @pytest.mark.asyncio async def test_on_exception_success(): log, log_success, log_backoff, log_giveup = _log_hdlrs() - @backoff.on_exception(backoff.constant, - Exception, - on_success=log_success, - on_backoff=log_backoff, - on_giveup=log_giveup, - jitter=None, - interval=0) + @backoff.on_exception( + backoff.constant, + Exception, + on_success=log_success, + on_backoff=log_backoff, + on_giveup=log_giveup, + jitter=None, + interval=0, + ) @_save_target async def succeeder(*args, **kwargs): # succeed after we've backed off twice - if len(log['backoff']) < 2: + if len(log["backoff"]) < 2: raise ValueError("catch me") await succeeder(1, 2, 3, foo=1, bar=2) # we try 3 times, backing off twice before succeeding - assert len(log['success']) == 1 - assert len(log['backoff']) == 2 - assert len(log['giveup']) == 0 + assert len(log["success"]) == 1 + assert len(log["backoff"]) == 2 + assert len(log["giveup"]) == 0 for i in range(2): - details = log['backoff'][i] - elapsed = details.pop('elapsed') - exception = details.pop('exception') + details = log["backoff"][i] + elapsed = details.pop("elapsed") + exception = details.pop("exception") assert isinstance(elapsed, float) assert isinstance(exception, ValueError) - assert details == {'args': (1, 2, 3), - 'kwargs': {'foo': 1, 'bar': 2}, - 'target': succeeder._target, - 'tries': i + 1, - 'wait': 0} - - details = log['success'][0] - elapsed = details.pop('elapsed') + assert details == { + "args": (1, 2, 3), + "kwargs": {"foo": 1, "bar": 2}, + "target": succeeder._target, + "tries": i + 1, + "wait": 0, + } + + details = log["success"][0] + elapsed = details.pop("elapsed") assert isinstance(elapsed, float) - assert details == {'args': (1, 2, 3), - 'kwargs': {'foo': 1, 'bar': 2}, - 'target': succeeder._target, - 'tries': 3} + assert details == { + "args": (1, 2, 3), + "kwargs": {"foo": 1, "bar": 2}, + "target": succeeder._target, + "tries": 3, + } @pytest.mark.asyncio -@pytest.mark.parametrize('raise_on_giveup', [True, False]) +@pytest.mark.parametrize("raise_on_giveup", [True, False]) async def test_on_exception_giveup(raise_on_giveup): log, log_success, log_backoff, log_giveup = _log_hdlrs() - @backoff.on_exception(backoff.constant, - ValueError, - on_success=log_success, - on_backoff=log_backoff, - on_giveup=log_giveup, - raise_on_giveup=raise_on_giveup, - max_tries=3, - jitter=None, - interval=0) + @backoff.on_exception( + backoff.constant, + ValueError, + on_success=log_success, + on_backoff=log_backoff, + on_giveup=log_giveup, + raise_on_giveup=raise_on_giveup, + max_tries=3, + jitter=None, + interval=0, + ) @_save_target async def exceptor(*args, **kwargs): raise ValueError("catch me") @@ -298,33 +309,33 @@ async def exceptor(*args, **kwargs): await exceptor(1, 2, 3, foo=1, bar=2) # we try 3 times, backing off twice and giving up once - assert len(log['success']) == 0 - assert len(log['backoff']) == 2 - assert len(log['giveup']) == 1 + assert len(log["success"]) == 0 + assert len(log["backoff"]) == 2 + assert len(log["giveup"]) == 1 - details = log['giveup'][0] - elapsed = details.pop('elapsed') - exception = details.pop('exception') + details = log["giveup"][0] + elapsed = details.pop("elapsed") + exception = details.pop("exception") assert isinstance(elapsed, float) assert isinstance(exception, ValueError) - assert details == {'args': (1, 2, 3), - 'kwargs': {'foo': 1, 'bar': 2}, - 'target': exceptor._target, - 'tries': 3} + assert details == { + "args": (1, 2, 3), + "kwargs": {"foo": 1, "bar": 2}, + "target": exceptor._target, + "tries": 3, + } @pytest.mark.asyncio async def test_on_exception_giveup_predicate(monkeypatch): - monkeypatch.setattr('asyncio.sleep', _await_none) + monkeypatch.setattr("asyncio.sleep", _await_none) def on_baz(e): return str(e) == "baz" vals = ["baz", "bar", "foo"] - @backoff.on_exception(backoff.constant, - ValueError, - giveup=on_baz) + @backoff.on_exception(backoff.constant, ValueError, giveup=on_baz) async def foo_bar_baz(): raise ValueError(vals.pop()) @@ -336,16 +347,14 @@ async def foo_bar_baz(): @pytest.mark.asyncio async def test_on_exception_giveup_coro(monkeypatch): - monkeypatch.setattr('asyncio.sleep', _await_none) + monkeypatch.setattr("asyncio.sleep", _await_none) async def on_baz(e): return str(e) == "baz" vals = ["baz", "bar", "foo"] - @backoff.on_exception(backoff.constant, - ValueError, - giveup=on_baz) + @backoff.on_exception(backoff.constant, ValueError, giveup=on_baz) async def foo_bar_baz(): raise ValueError(vals.pop()) @@ -359,56 +368,64 @@ async def foo_bar_baz(): async def test_on_predicate_success(): log, log_success, log_backoff, log_giveup = _log_hdlrs() - @backoff.on_predicate(backoff.constant, - on_success=log_success, - on_backoff=log_backoff, - on_giveup=log_giveup, - jitter=None, - interval=0) + @backoff.on_predicate( + backoff.constant, + on_success=log_success, + on_backoff=log_backoff, + on_giveup=log_giveup, + jitter=None, + interval=0, + ) @_save_target async def success(*args, **kwargs): # succeed after we've backed off twice - return len(log['backoff']) == 2 + return len(log["backoff"]) == 2 await success(1, 2, 3, foo=1, bar=2) # we try 3 times, backing off twice before succeeding - assert len(log['success']) == 1 - assert len(log['backoff']) == 2 - assert len(log['giveup']) == 0 + assert len(log["success"]) == 1 + assert len(log["backoff"]) == 2 + assert len(log["giveup"]) == 0 for i in range(2): - details = log['backoff'][i] - elapsed = details.pop('elapsed') + details = log["backoff"][i] + elapsed = details.pop("elapsed") assert isinstance(elapsed, float) - assert details == {'args': (1, 2, 3), - 'kwargs': {'foo': 1, 'bar': 2}, - 'target': success._target, - 'tries': i + 1, - 'value': False, - 'wait': 0} - - details = log['success'][0] - elapsed = details.pop('elapsed') + assert details == { + "args": (1, 2, 3), + "kwargs": {"foo": 1, "bar": 2}, + "target": success._target, + "tries": i + 1, + "value": False, + "wait": 0, + } + + details = log["success"][0] + elapsed = details.pop("elapsed") assert isinstance(elapsed, float) - assert details == {'args': (1, 2, 3), - 'kwargs': {'foo': 1, 'bar': 2}, - 'target': success._target, - 'tries': 3, - 'value': True} + assert details == { + "args": (1, 2, 3), + "kwargs": {"foo": 1, "bar": 2}, + "target": success._target, + "tries": 3, + "value": True, + } @pytest.mark.asyncio async def test_on_predicate_giveup(): log, log_success, log_backoff, log_giveup = _log_hdlrs() - @backoff.on_predicate(backoff.constant, - on_success=log_success, - on_backoff=log_backoff, - on_giveup=log_giveup, - max_tries=3, - jitter=None, - interval=0) + @backoff.on_predicate( + backoff.constant, + on_success=log_success, + on_backoff=log_backoff, + on_giveup=log_giveup, + max_tries=3, + jitter=None, + interval=0, + ) @_save_target async def emptiness(*args, **kwargs): pass @@ -416,31 +433,35 @@ async def emptiness(*args, **kwargs): await emptiness(1, 2, 3, foo=1, bar=2) # we try 3 times, backing off twice and giving up once - assert len(log['success']) == 0 - assert len(log['backoff']) == 2 - assert len(log['giveup']) == 1 + assert len(log["success"]) == 0 + assert len(log["backoff"]) == 2 + assert len(log["giveup"]) == 1 - details = log['giveup'][0] - elapsed = details.pop('elapsed') + details = log["giveup"][0] + elapsed = details.pop("elapsed") assert isinstance(elapsed, float) - assert details == {'args': (1, 2, 3), - 'kwargs': {'foo': 1, 'bar': 2}, - 'target': emptiness._target, - 'tries': 3, - 'value': None} + assert details == { + "args": (1, 2, 3), + "kwargs": {"foo": 1, "bar": 2}, + "target": emptiness._target, + "tries": 3, + "value": None, + } @pytest.mark.asyncio async def test_on_predicate_iterable_handlers(): hdlrs = [_log_hdlrs() for _ in range(3)] - @backoff.on_predicate(backoff.constant, - on_success=(h[1] for h in hdlrs), - on_backoff=(h[2] for h in hdlrs), - on_giveup=(h[3] for h in hdlrs), - max_tries=3, - jitter=None, - interval=0) + @backoff.on_predicate( + backoff.constant, + on_success=(h[1] for h in hdlrs), + on_backoff=(h[2] for h in hdlrs), + on_giveup=(h[3] for h in hdlrs), + max_tries=3, + jitter=None, + interval=0, + ) @_save_target async def emptiness(*args, **kwargs): pass @@ -448,23 +469,25 @@ async def emptiness(*args, **kwargs): await emptiness(1, 2, 3, foo=1, bar=2) for i in range(3): - assert len(hdlrs[i][0]['success']) == 0 - assert len(hdlrs[i][0]['backoff']) == 2 - assert len(hdlrs[i][0]['giveup']) == 1 + assert len(hdlrs[i][0]["success"]) == 0 + assert len(hdlrs[i][0]["backoff"]) == 2 + assert len(hdlrs[i][0]["giveup"]) == 1 - details = dict(hdlrs[i][0]['giveup'][0]) - elapsed = details.pop('elapsed') + details = dict(hdlrs[i][0]["giveup"][0]) + elapsed = details.pop("elapsed") assert isinstance(elapsed, float) - assert details == {'args': (1, 2, 3), - 'kwargs': {'foo': 1, 'bar': 2}, - 'target': emptiness._target, - 'tries': 3, - 'value': None} + assert details == { + "args": (1, 2, 3), + "kwargs": {"foo": 1, "bar": 2}, + "target": emptiness._target, + "tries": 3, + "value": None, + } @pytest.mark.asyncio async def test_on_predicate_constant_iterable(monkeypatch): - monkeypatch.setattr('asyncio.sleep', _await_none) + monkeypatch.setattr("asyncio.sleep", _await_none) waits = [1, 2, 3, 6, 9] backoffs = [] @@ -486,7 +509,7 @@ async def falsey(): assert len(backoffs) == len(waits) for i, wait in enumerate(waits): - assert backoffs[i]['wait'] == wait + assert backoffs[i]["wait"] == wait assert len(giveups) == 1 assert len(successes) == 0 @@ -496,114 +519,124 @@ async def falsey(): # on_predicate should support 0-argument jitter function. @pytest.mark.asyncio async def test_on_exception_success_0_arg_jitter(monkeypatch): - monkeypatch.setattr('asyncio.sleep', _await_none) - monkeypatch.setattr('random.random', lambda: 0) + monkeypatch.setattr("asyncio.sleep", _await_none) + monkeypatch.setattr("random.random", lambda: 0) log, log_success, log_backoff, log_giveup = _log_hdlrs() - @backoff.on_exception(backoff.constant, - Exception, - on_success=log_success, - on_backoff=log_backoff, - on_giveup=log_giveup, - jitter=random.random, - interval=0) + @backoff.on_exception( + backoff.constant, + Exception, + on_success=log_success, + on_backoff=log_backoff, + on_giveup=log_giveup, + jitter=random.random, + interval=0, + ) @_save_target async def succeeder(*args, **kwargs): # succeed after we've backed off twice - if len(log['backoff']) < 2: + if len(log["backoff"]) < 2: raise ValueError("catch me") with pytest.deprecated_call(): await succeeder(1, 2, 3, foo=1, bar=2) # we try 3 times, backing off twice before succeeding - assert len(log['success']) == 1 - assert len(log['backoff']) == 2 - assert len(log['giveup']) == 0 + assert len(log["success"]) == 1 + assert len(log["backoff"]) == 2 + assert len(log["giveup"]) == 0 for i in range(2): - details = log['backoff'][i] - elapsed = details.pop('elapsed') - exception = details.pop('exception') + details = log["backoff"][i] + elapsed = details.pop("elapsed") + exception = details.pop("exception") assert isinstance(elapsed, float) assert isinstance(exception, ValueError) - assert details == {'args': (1, 2, 3), - 'kwargs': {'foo': 1, 'bar': 2}, - 'target': succeeder._target, - 'tries': i + 1, - 'wait': 0} - - details = log['success'][0] - elapsed = details.pop('elapsed') + assert details == { + "args": (1, 2, 3), + "kwargs": {"foo": 1, "bar": 2}, + "target": succeeder._target, + "tries": i + 1, + "wait": 0, + } + + details = log["success"][0] + elapsed = details.pop("elapsed") assert isinstance(elapsed, float) - assert details == {'args': (1, 2, 3), - 'kwargs': {'foo': 1, 'bar': 2}, - 'target': succeeder._target, - 'tries': 3} + assert details == { + "args": (1, 2, 3), + "kwargs": {"foo": 1, "bar": 2}, + "target": succeeder._target, + "tries": 3, + } # To maintain backward compatibility, # on_predicate should support 0-argument jitter function. @pytest.mark.asyncio async def test_on_predicate_success_0_arg_jitter(monkeypatch): - monkeypatch.setattr('asyncio.sleep', _await_none) - monkeypatch.setattr('random.random', lambda: 0) + monkeypatch.setattr("asyncio.sleep", _await_none) + monkeypatch.setattr("random.random", lambda: 0) log, log_success, log_backoff, log_giveup = _log_hdlrs() - @backoff.on_predicate(backoff.constant, - on_success=log_success, - on_backoff=log_backoff, - on_giveup=log_giveup, - jitter=random.random, - interval=0) + @backoff.on_predicate( + backoff.constant, + on_success=log_success, + on_backoff=log_backoff, + on_giveup=log_giveup, + jitter=random.random, + interval=0, + ) @_save_target async def success(*args, **kwargs): # succeed after we've backed off twice - return len(log['backoff']) == 2 + return len(log["backoff"]) == 2 with pytest.deprecated_call(): await success(1, 2, 3, foo=1, bar=2) # we try 3 times, backing off twice before succeeding - assert len(log['success']) == 1 - assert len(log['backoff']) == 2 - assert len(log['giveup']) == 0 + assert len(log["success"]) == 1 + assert len(log["backoff"]) == 2 + assert len(log["giveup"]) == 0 for i in range(2): - details = log['backoff'][i] - elapsed = details.pop('elapsed') + details = log["backoff"][i] + elapsed = details.pop("elapsed") assert isinstance(elapsed, float) - assert details == {'args': (1, 2, 3), - 'kwargs': {'foo': 1, 'bar': 2}, - 'target': success._target, - 'tries': i + 1, - 'value': False, - 'wait': 0} - - details = log['success'][0] - elapsed = details.pop('elapsed') + assert details == { + "args": (1, 2, 3), + "kwargs": {"foo": 1, "bar": 2}, + "target": success._target, + "tries": i + 1, + "value": False, + "wait": 0, + } + + details = log["success"][0] + elapsed = details.pop("elapsed") assert isinstance(elapsed, float) - assert details == {'args': (1, 2, 3), - 'kwargs': {'foo': 1, 'bar': 2}, - 'target': success._target, - 'tries': 3, - 'value': True} + assert details == { + "args": (1, 2, 3), + "kwargs": {"foo": 1, "bar": 2}, + "target": success._target, + "tries": 3, + "value": True, + } @pytest.mark.asyncio async def test_on_exception_callable_max_tries(monkeypatch): - monkeypatch.setattr('asyncio.sleep', _await_none) + monkeypatch.setattr("asyncio.sleep", _await_none) def lookup_max_tries(): return 3 log = [] - @backoff.on_exception(backoff.constant, - ValueError, - max_tries=lookup_max_tries) + @backoff.on_exception(backoff.constant, ValueError, max_tries=lookup_max_tries) async def exceptor(): log.append(True) raise ValueError() @@ -616,7 +649,7 @@ async def exceptor(): @pytest.mark.asyncio async def test_on_exception_callable_max_tries_reads_every_time(monkeypatch): - monkeypatch.setattr('asyncio.sleep', _await_none) + monkeypatch.setattr("asyncio.sleep", _await_none) lookups = [] @@ -624,9 +657,7 @@ def lookup_max_tries(): lookups.append(True) return 3 - @backoff.on_exception(backoff.constant, - ValueError, - max_tries=lookup_max_tries) + @backoff.on_exception(backoff.constant, ValueError, max_tries=lookup_max_tries) async def exceptor(): raise ValueError() @@ -641,7 +672,6 @@ async def exceptor(): @pytest.mark.asyncio async def test_on_exception_callable_gen_kwargs(): - def lookup_foo(): return "foo" @@ -652,11 +682,7 @@ def wait_gen(foo=None, bar=None): while True: yield 0 - @backoff.on_exception(wait_gen, - ValueError, - max_tries=2, - foo=lookup_foo, - bar="bar") + @backoff.on_exception(wait_gen, ValueError, max_tries=2, foo=lookup_foo, bar="bar") async def exceptor(): raise ValueError("aah") @@ -685,4 +711,4 @@ async def coro(): task.cancel() - assert (await task) + assert await task diff --git a/tests/test_integration.py b/tests/test_integration.py index 2dcade8..9a59cb1 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -14,7 +14,6 @@ @responses.activate def test_on_predicate_runtime(monkeypatch): - log = [] def sleep(seconds): @@ -46,7 +45,6 @@ def get_url(): @responses.activate def test_on_exception_runtime(monkeypatch): - log = [] def sleep(seconds): diff --git a/tests/test_typing.py b/tests/test_typing.py index ea3b27c..24a20b5 100644 --- a/tests/test_typing.py +++ b/tests/test_typing.py @@ -4,6 +4,7 @@ # No pyunit tests are defined here yet, but the following decorator calls will # be analyzed by mypy which would have caught a bug the last release. + @backoff.on_exception( backoff.expo, ValueError, diff --git a/tests/test_wait_gen.py b/tests/test_wait_gen.py index bebe58b..94f4b38 100644 --- a/tests/test_wait_gen.py +++ b/tests/test_wait_gen.py @@ -7,14 +7,14 @@ def test_decay(): gen = backoff.decay() gen.send(None) for i in range(10): - assert math.e ** -i == next(gen) + assert math.e**-i == next(gen) def test_decay_init100(): gen = backoff.decay(initial_value=100) gen.send(None) for i in range(10): - assert 100 * math.e ** -i == next(gen) + assert 100 * math.e**-i == next(gen) def test_decay_init100_decay3(): @@ -35,32 +35,32 @@ def test_expo(): gen = backoff.expo() gen.send(None) for i in range(9): - assert 2 ** i == next(gen) + assert 2**i == next(gen) def test_expo_base3(): gen = backoff.expo(base=3) gen.send(None) for i in range(9): - assert 3 ** i == next(gen) + assert 3**i == next(gen) def test_expo_factor3(): gen = backoff.expo(factor=3) gen.send(None) for i in range(9): - assert 3 * 2 ** i == next(gen) + assert 3 * 2**i == next(gen) def test_expo_base3_factor5(): gen = backoff.expo(base=3, factor=5) gen.send(None) for i in range(9): - assert 5 * 3 ** i == next(gen) + assert 5 * 3**i == next(gen) def test_expo_max_value(): - gen = backoff.expo(max_value=2 ** 4) + gen = backoff.expo(max_value=2**4) gen.send(None) expected = [1, 2, 4, 8, 16, 16, 16] for expect in expected: @@ -68,7 +68,7 @@ def test_expo_max_value(): def test_expo_max_value_factor(): - gen = backoff.expo(factor=3, max_value=2 ** 4) + gen = backoff.expo(factor=3, max_value=2**4) gen.send(None) expected = [3 * 1, 3 * 2, 3 * 4, 16, 16, 16, 16] for expect in expected: @@ -95,7 +95,7 @@ def test_constant(): gen = backoff.constant(interval=3) gen.send(None) for i in range(9): - assert 3 == next(gen) + assert next(gen) == 3 def test_runtime():