diff --git a/rhodecode/lib/vcs/client_http.py b/rhodecode/lib/vcs/client_http.py --- a/rhodecode/lib/vcs/client_http.py +++ b/rhodecode/lib/vcs/client_http.py @@ -54,23 +54,26 @@ EXCEPTIONS_MAP = { } -def _remote_call(url, payload, exceptions_map, session): - try: - headers = { - 'X-RC-Method': payload.get('method'), - 'X-RC-Repo-Name': payload.get('_repo_name') - } - response = session.post(url, data=msgpack.packb(payload), headers=headers) - except pycurl.error as e: - msg = '{}. \npycurl traceback: {}'.format(e, traceback.format_exc()) - raise exceptions.HttpVCSCommunicationError(msg) - except Exception as e: - message = getattr(e, 'message', '') - if 'Failed to connect' in message: - # gevent doesn't return proper pycurl errors - raise exceptions.HttpVCSCommunicationError(e) - else: - raise +def _remote_call(url, payload, exceptions_map, session, retries=3): + + for attempt in range(retries): + try: + response = session.post(url, data=msgpack.packb(payload)) + except pycurl.error as e: + error_code, error_message = e.args + if error_code == pycurl.E_RECV_ERROR: + log.warning(f'Received a "Connection reset by peer" error. ' + f'Retrying... ({attempt + 1}/{retries})') + continue # Retry if connection reset error. + msg = '{}. \npycurl traceback: {}'.format(e, traceback.format_exc()) + raise exceptions.HttpVCSCommunicationError(msg) + except Exception as e: + message = getattr(e, 'message', '') + if 'Failed to connect' in message: + # gevent doesn't return proper pycurl errors + raise exceptions.HttpVCSCommunicationError(e) + else: + raise if response.status_code >= 400: log.error('Call to %s returned non 200 HTTP code: %s', @@ -80,7 +83,7 @@ def _remote_call(url, payload, exception try: response = msgpack.unpackb(response.content, raw=False) except Exception: - log.exception('Failed to decode response %r', response.content) + log.exception('Failed to decode response from msgpack') raise error = response.get('error') @@ -112,6 +115,7 @@ def _streaming_remote_call(url, payload, } response = session.post(url, data=msgpack.packb(payload), headers=headers) except pycurl.error as e: + error_code, error_message = e.args msg = '{}. \npycurl traceback: {}'.format(e, traceback.format_exc()) raise exceptions.HttpVCSCommunicationError(msg) except Exception as e: @@ -296,8 +300,9 @@ class RemoteRepo(object): namespace=self._cache_namespace, condition=cache_on and cache_key) def remote_call(_cache_key): if self._call_with_logging: - log.debug('Calling %s@%s with args:%.10240r. wire_context: %s cache_on: %s', - url, name, args, context_uid, cache_on) + args_repr = f'ARG: {str(args):.256}|KW: {str(kwargs):.256}' + log.debug('Calling %s@%s with args:%r. wire_context: %s cache_on: %s', + url, name, args_repr, context_uid, cache_on) return _remote_call(url, payload, EXCEPTIONS_MAP, self._session) result = remote_call(cache_key) @@ -318,8 +323,9 @@ class RemoteRepo(object): # Cache is a problem because this is a stream def streaming_remote_call(_cache_key): if self._call_with_logging: - log.debug('Calling %s@%s with args:%.10240r. wire_context: %s cache_on: %s', - url, name, args, context_uid, cache_on) + args_repr = f'ARG: {str(args):.256}|KW: {str(kwargs):.256}' + log.debug('Calling %s@%s with args:%r. wire_context: %s cache_on: %s', + url, name, args_repr, context_uid, cache_on) return _streaming_remote_call(url, payload, EXCEPTIONS_MAP, self._session, self.CHUNK_SIZE) result = streaming_remote_call(cache_key)