pull-requests: added retry mechanism for updating pull requests.

rhodecode/apps/repository/views/repo_pull_requests.py

@@ -39,7 +39,7 b' from rhodecode.lib.ext_json import json'
 from rhodecode.lib.auth import (
     LoginRequired, HasRepoPermissionAny, HasRepoPermissionAnyDecorator,
     NotAnonymous, CSRFRequired)
-from rhodecode.lib.utils2 import str2bool, safe_str, safe_unicode, safe_int, aslist
+from rhodecode.lib.utils2 import str2bool, safe_str, safe_unicode, safe_int, aslist, retry
 from rhodecode.lib.vcs.backends.base import (
     EmptyCommit, UpdateFailureReason, unicode_to_reference)
 from rhodecode.lib.vcs.exceptions import (
@@ -1353,9 +1353,13 b' class RepoPullRequestsView(RepoAppView, '
     def _update_commits(self, c, pull_request):
         _ = self.request.translate
 
+        @retry(exception=Exception, n_tries=3)
+        def commits_update():
+            return PullRequestModel().update_commits(
+                pull_request, self._rhodecode_db_user)
+
         with pull_request.set_state(PullRequest.STATE_UPDATING):
-            resp = PullRequestModel().update_commits(
-                pull_request, self._rhodecode_db_user)
+            resp = commits_update()  # retry x3
 
         if resp.executed:
 

rhodecode/lib/utils2.py

@@ -35,7 +35,7 b' import urllib'
 import urlobject
 import uuid
 import getpass
-from functools import update_wrapper, partial
+from functools import update_wrapper, partial, wraps
 
 import pygments.lexers
 import sqlalchemy
@@ -1038,16 +1038,17 b' class CachedProperty(object):'
     """
     Lazy Attributes. With option to invalidate the cache by running a method
 
-    class Foo():
+    >>> class Foo(object):
+    ...
+    ...    @CachedProperty
+    ...    def heavy_func(self):
+    ...        return 'super-calculation'
+    ...
+    ... foo = Foo()
+    ... foo.heavy_func() # first computation
+    ... foo.heavy_func() # fetch from cache
+    ... foo._invalidate_prop_cache('heavy_func')
 
-        @CachedProperty
-        def heavy_func():
-            return 'super-calculation'
-
-    foo = Foo()
-    foo.heavy_func() # first computions
-    foo.heavy_func() # fetch from cache
-    foo._invalidate_prop_cache('heavy_func')
     # at this point calling foo.heavy_func() will be re-computed
     """
 
@@ -1072,3 +1073,76 b' class CachedProperty(object):'
 
     def _invalidate_prop_cache(self, inst, name):
         inst.__dict__.pop(name, None)
+
+
+def retry(func=None, exception=Exception, n_tries=5, delay=5, backoff=1, logger=True):
+    """
+    Retry decorator with exponential backoff.
+
+    Parameters
+    ----------
+    func : typing.Callable, optional
+        Callable on which the decorator is applied, by default None
+    exception : Exception or tuple of Exceptions, optional
+        Exception(s) that invoke retry, by default Exception
+    n_tries : int, optional
+        Number of tries before giving up, by default 5
+    delay : int, optional
+        Initial delay between retries in seconds, by default 5
+    backoff : int, optional
+        Backoff multiplier e.g. value of 2 will double the delay, by default 1
+    logger : bool, optional
+        Option to log or print, by default False
+
+    Returns
+    -------
+    typing.Callable
+        Decorated callable that calls itself when exception(s) occur.
+
+    Examples
+    --------
+    >>> import random
+    >>> @retry(exception=Exception, n_tries=3)
+    ... def test_random(text):
+    ...    x = random.random()
+    ...    if x < 0.5:
+    ...        raise Exception("Fail")
+    ...    else:
+    ...        print("Success: ", text)
+    >>> test_random("It works!")
+    """
+
+    if func is None:
+        return partial(
+            retry,
+            exception=exception,
+            n_tries=n_tries,
+            delay=delay,
+            backoff=backoff,
+            logger=logger,
+        )
+
+    @wraps(func)
+    def wrapper(*args, **kwargs):
+        _n_tries, n_delay = n_tries, delay
+        log = logging.getLogger('rhodecode.retry')
+
+        while _n_tries > 1:
+            try:
+                return func(*args, **kwargs)
+            except exception as e:
+                e_details = repr(e)
+                msg = "Exception on calling func {func}: {e}, " \
+                      "Retrying in {n_delay} seconds..."\
+                    .format(func=func, e=e_details, n_delay=n_delay)
+                if logger:
+                    log.warning(msg)
+                else:
+                    print(msg)
+                time.sleep(n_delay)
+                _n_tries -= 1
+                n_delay *= backoff
+
+        return func(*args, **kwargs)
+
+    return wrapper

rhodecode/model/db.py

@@ -4062,7 +4062,7 b' class _SetState(object):'
         return self
 
     def __exit__(self, exc_type, exc_val, exc_tb):
-        if exc_val is not None:
+        if exc_val is not None or exc_type is not None:
             log.error(traceback.format_exc(exc_tb))
             return None