remote-call: added retries for connection-reset-by-peer and optimize logging leaking too much info

rhodecode/lib/vcs/client_http.py

@@ -54,23 +54,26 b' EXCEPTIONS_MAP = {'
 }
 
 
-def _remote_call(url, payload, exceptions_map, session):
-    try:
-        headers = {
-            'X-RC-Method': payload.get('method'),
-            'X-RC-Repo-Name': payload.get('_repo_name')
-        }
-        response = session.post(url, data=msgpack.packb(payload), headers=headers)
-    except pycurl.error as e:
-        msg = '{}. \npycurl traceback: {}'.format(e, traceback.format_exc())
-        raise exceptions.HttpVCSCommunicationError(msg)
-    except Exception as e:
-        message = getattr(e, 'message', '')
-        if 'Failed to connect' in message:
-            # gevent doesn't return proper pycurl errors
-            raise exceptions.HttpVCSCommunicationError(e)
-        else:
-            raise
+def _remote_call(url, payload, exceptions_map, session, retries=3):
+
+    for attempt in range(retries):
+        try:
+            response = session.post(url, data=msgpack.packb(payload))
+        except pycurl.error as e:
+            error_code, error_message = e.args
+            if error_code == pycurl.E_RECV_ERROR:
+                log.warning(f'Received a "Connection reset by peer" error. '
+                            f'Retrying... ({attempt + 1}/{retries})')
+                continue  # Retry if connection reset error.
+            msg = '{}. \npycurl traceback: {}'.format(e, traceback.format_exc())
+            raise exceptions.HttpVCSCommunicationError(msg)
+        except Exception as e:
+            message = getattr(e, 'message', '')
+            if 'Failed to connect' in message:
+                # gevent doesn't return proper pycurl errors
+                raise exceptions.HttpVCSCommunicationError(e)
+            else:
+                raise
 
     if response.status_code >= 400:
         log.error('Call to %s returned non 200 HTTP code: %s',
@@ -80,7 +83,7 b' def _remote_call(url, payload, exception'
     try:
         response = msgpack.unpackb(response.content, raw=False)
     except Exception:
-        log.exception('Failed to decode response %r', response.content)
+        log.exception('Failed to decode response from msgpack')
         raise
 
     error = response.get('error')
@@ -112,6 +115,7 b' def _streaming_remote_call(url, payload,'
         }
         response = session.post(url, data=msgpack.packb(payload), headers=headers)
     except pycurl.error as e:
+        error_code, error_message = e.args
         msg = '{}. \npycurl traceback: {}'.format(e, traceback.format_exc())
         raise exceptions.HttpVCSCommunicationError(msg)
     except Exception as e:
@@ -296,8 +300,9 b' class RemoteRepo(object):'
             namespace=self._cache_namespace, condition=cache_on and cache_key)
         def remote_call(_cache_key):
             if self._call_with_logging:
-                log.debug('Calling %s@%s with args:%.10240r. wire_context: %s cache_on: %s',
-                          url, name, args, context_uid, cache_on)
+                args_repr = f'ARG: {str(args):.256}|KW: {str(kwargs):.256}'
+                log.debug('Calling %s@%s with args:%r. wire_context: %s cache_on: %s',
+                          url, name, args_repr, context_uid, cache_on)
             return _remote_call(url, payload, EXCEPTIONS_MAP, self._session)
 
         result = remote_call(cache_key)
@@ -318,8 +323,9 b' class RemoteRepo(object):'
         # Cache is a problem because this is a stream
         def streaming_remote_call(_cache_key):
             if self._call_with_logging:
-                log.debug('Calling %s@%s with args:%.10240r. wire_context: %s cache_on: %s',
-                          url, name, args, context_uid, cache_on)
+                args_repr = f'ARG: {str(args):.256}|KW: {str(kwargs):.256}'
+                log.debug('Calling %s@%s with args:%r. wire_context: %s cache_on: %s',
+                          url, name, args_repr, context_uid, cache_on)
             return _streaming_remote_call(url, payload, EXCEPTIONS_MAP, self._session, self.CHUNK_SIZE)
 
         result = streaming_remote_call(cache_key)