diff --git a/rhodecode/apps/repository/views/repo_pull_requests.py b/rhodecode/apps/repository/views/repo_pull_requests.py --- a/rhodecode/apps/repository/views/repo_pull_requests.py +++ b/rhodecode/apps/repository/views/repo_pull_requests.py @@ -39,7 +39,7 @@ from rhodecode.lib.ext_json import json from rhodecode.lib.auth import ( LoginRequired, HasRepoPermissionAny, HasRepoPermissionAnyDecorator, NotAnonymous, CSRFRequired) -from rhodecode.lib.utils2 import str2bool, safe_str, safe_unicode, safe_int, aslist +from rhodecode.lib.utils2 import str2bool, safe_str, safe_unicode, safe_int, aslist, retry from rhodecode.lib.vcs.backends.base import ( EmptyCommit, UpdateFailureReason, unicode_to_reference) from rhodecode.lib.vcs.exceptions import ( @@ -1353,9 +1353,13 @@ class RepoPullRequestsView(RepoAppView, def _update_commits(self, c, pull_request): _ = self.request.translate + @retry(exception=Exception, n_tries=3) + def commits_update(): + return PullRequestModel().update_commits( + pull_request, self._rhodecode_db_user) + with pull_request.set_state(PullRequest.STATE_UPDATING): - resp = PullRequestModel().update_commits( - pull_request, self._rhodecode_db_user) + resp = commits_update() # retry x3 if resp.executed: diff --git a/rhodecode/lib/utils2.py b/rhodecode/lib/utils2.py --- a/rhodecode/lib/utils2.py +++ b/rhodecode/lib/utils2.py @@ -35,7 +35,7 @@ import urllib import urlobject import uuid import getpass -from functools import update_wrapper, partial +from functools import update_wrapper, partial, wraps import pygments.lexers import sqlalchemy @@ -1038,16 +1038,17 @@ class CachedProperty(object): """ Lazy Attributes. With option to invalidate the cache by running a method - class Foo(): + >>> class Foo(object): + ... + ... @CachedProperty + ... def heavy_func(self): + ... return 'super-calculation' + ... + ... foo = Foo() + ... foo.heavy_func() # first computation + ... foo.heavy_func() # fetch from cache + ... foo._invalidate_prop_cache('heavy_func') - @CachedProperty - def heavy_func(): - return 'super-calculation' - - foo = Foo() - foo.heavy_func() # first computions - foo.heavy_func() # fetch from cache - foo._invalidate_prop_cache('heavy_func') # at this point calling foo.heavy_func() will be re-computed """ @@ -1072,3 +1073,76 @@ class CachedProperty(object): def _invalidate_prop_cache(self, inst, name): inst.__dict__.pop(name, None) + + +def retry(func=None, exception=Exception, n_tries=5, delay=5, backoff=1, logger=True): + """ + Retry decorator with exponential backoff. + + Parameters + ---------- + func : typing.Callable, optional + Callable on which the decorator is applied, by default None + exception : Exception or tuple of Exceptions, optional + Exception(s) that invoke retry, by default Exception + n_tries : int, optional + Number of tries before giving up, by default 5 + delay : int, optional + Initial delay between retries in seconds, by default 5 + backoff : int, optional + Backoff multiplier e.g. value of 2 will double the delay, by default 1 + logger : bool, optional + Option to log or print, by default False + + Returns + ------- + typing.Callable + Decorated callable that calls itself when exception(s) occur. + + Examples + -------- + >>> import random + >>> @retry(exception=Exception, n_tries=3) + ... def test_random(text): + ... x = random.random() + ... if x < 0.5: + ... raise Exception("Fail") + ... else: + ... print("Success: ", text) + >>> test_random("It works!") + """ + + if func is None: + return partial( + retry, + exception=exception, + n_tries=n_tries, + delay=delay, + backoff=backoff, + logger=logger, + ) + + @wraps(func) + def wrapper(*args, **kwargs): + _n_tries, n_delay = n_tries, delay + log = logging.getLogger('rhodecode.retry') + + while _n_tries > 1: + try: + return func(*args, **kwargs) + except exception as e: + e_details = repr(e) + msg = "Exception on calling func {func}: {e}, " \ + "Retrying in {n_delay} seconds..."\ + .format(func=func, e=e_details, n_delay=n_delay) + if logger: + log.warning(msg) + else: + print(msg) + time.sleep(n_delay) + _n_tries -= 1 + n_delay *= backoff + + return func(*args, **kwargs) + + return wrapper diff --git a/rhodecode/model/db.py b/rhodecode/model/db.py --- a/rhodecode/model/db.py +++ b/rhodecode/model/db.py @@ -4062,7 +4062,7 @@ class _SetState(object): return self def __exit__(self, exc_type, exc_val, exc_tb): - if exc_val is not None: + if exc_val is not None or exc_type is not None: log.error(traceback.format_exc(exc_tb)) return None