diff --git a/vcsserver/http_main.py b/vcsserver/http_main.py --- a/vcsserver/http_main.py +++ b/vcsserver/http_main.py @@ -16,6 +16,7 @@ # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA import os +import sys import base64 import locale import logging @@ -36,6 +37,7 @@ from vcsserver.git_lfs.app import GIT_LF from vcsserver.echo_stub import remote_wsgi as remote_wsgi_stub from vcsserver.echo_stub.echo_app import EchoApp from vcsserver.exceptions import HTTPRepoLocked +from vcsserver.lib.exc_tracking import store_exception from vcsserver.server import VcsServer try: @@ -289,7 +291,21 @@ class HTTPApplication(object): try: resp = getattr(remote, method)(*args, **kwargs) except Exception as e: - tb_info = traceback.format_exc() + exc_info = list(sys.exc_info()) + exc_type, exc_value, exc_traceback = exc_info + + org_exc = getattr(e, '_org_exc', None) + org_exc_name = None + if org_exc: + org_exc_name = org_exc.__class__.__name__ + # replace our "faked" exception with our org + exc_info[0] = org_exc.__class__ + exc_info[1] = org_exc + + store_exception(id(exc_info), exc_info) + + tb_info = ''.join( + traceback.format_exception(exc_type, exc_value, exc_traceback)) type_ = e.__class__.__name__ if type_ not in self.ALLOWED_EXCEPTIONS: @@ -300,6 +316,7 @@ class HTTPApplication(object): 'error': { 'message': e.message, 'traceback': tb_info, + 'org_exc': org_exc_name, 'type': type_ } } @@ -492,9 +509,14 @@ class HTTPApplication(object): status_code = request.headers.get('X-RC-Locked-Status-Code') return HTTPRepoLocked( title=exception.message, status_code=status_code) + + exc_info = request.exc_info + store_exception(id(exc_info), exc_info) + traceback_info = 'unavailable' if request.exc_info: - traceback_info = traceback.format_exc(request.exc_info[2]) + exc_type, exc_value, exc_tb = request.exc_info + traceback_info = ''.join(traceback.format_exception(exc_type, exc_value, exc_tb)) log.error( 'error occurred handling this request for path: %s, \n tb: %s', diff --git a/vcsserver/lib/exc_tracking.py b/vcsserver/lib/exc_tracking.py new file mode 100644 --- /dev/null +++ b/vcsserver/lib/exc_tracking.py @@ -0,0 +1,146 @@ +# -*- coding: utf-8 -*- + +# RhodeCode VCSServer provides access to different vcs backends via network. +# Copyright (C) 2014-2018 RhodeCode GmbH +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + + +import os +import time +import datetime +import msgpack +import logging +import traceback +import tempfile + + +log = logging.getLogger(__name__) + +# NOTE: Any changes should be synced with exc_tracking at rhodecode.lib.exc_tracking +global_prefix = 'vcsserver' + + +def exc_serialize(exc_id, tb, exc_type): + + data = { + 'version': 'v1', + 'exc_id': exc_id, + 'exc_utc_date': datetime.datetime.utcnow().isoformat(), + 'exc_timestamp': repr(time.time()), + 'exc_message': tb, + 'exc_type': exc_type, + } + return msgpack.packb(data), data + + +def exc_unserialize(tb): + return msgpack.unpackb(tb) + + +def get_exc_store(): + """ + Get and create exception store if it's not existing + """ + exc_store_dir = 'rc_exception_store_v1' + # fallback + _exc_store_path = os.path.join(tempfile.gettempdir(), exc_store_dir) + + exc_store_dir = '' # TODO: need a persistent cross instance store here + if exc_store_dir: + _exc_store_path = os.path.join(exc_store_dir, exc_store_dir) + + _exc_store_path = os.path.abspath(_exc_store_path) + if not os.path.isdir(_exc_store_path): + os.makedirs(_exc_store_path) + log.debug('Initializing exceptions store at %s', _exc_store_path) + return _exc_store_path + + +def _store_exception(exc_id, exc_info, prefix): + exc_type, exc_value, exc_traceback = exc_info + tb = ''.join(traceback.format_exception( + exc_type, exc_value, exc_traceback, None)) + + exc_type_name = exc_type.__name__ + exc_store_path = get_exc_store() + exc_data, org_data = exc_serialize(exc_id, tb, exc_type_name) + exc_pref_id = '{}_{}_{}'.format(exc_id, prefix, org_data['exc_timestamp']) + if not os.path.isdir(exc_store_path): + os.makedirs(exc_store_path) + stored_exc_path = os.path.join(exc_store_path, exc_pref_id) + with open(stored_exc_path, 'wb') as f: + f.write(exc_data) + log.debug('Stored generated exception %s as: %s', exc_id, stored_exc_path) + + +def store_exception(exc_id, exc_info, prefix=global_prefix): + try: + _store_exception(exc_id=exc_id, exc_info=exc_info, prefix=prefix) + except Exception: + log.exception('Failed to store exception `%s` information', exc_id) + # there's no way this can fail, it will crash server badly if it does. + pass + + +def _find_exc_file(exc_id, prefix=global_prefix): + exc_store_path = get_exc_store() + if prefix: + exc_id = '{}_{}'.format(exc_id, prefix) + else: + # search without a prefix + exc_id = '{}'.format(exc_id) + + # we need to search the store for such start pattern as above + for fname in os.listdir(exc_store_path): + if fname.startswith(exc_id): + exc_id = os.path.join(exc_store_path, fname) + break + continue + else: + exc_id = None + + return exc_id + + +def _read_exception(exc_id, prefix): + exc_id_file_path = _find_exc_file(exc_id=exc_id, prefix=prefix) + if exc_id_file_path: + with open(exc_id_file_path, 'rb') as f: + return exc_unserialize(f.read()) + else: + log.debug('Exception File `%s` not found', exc_id_file_path) + return None + + +def read_exception(exc_id, prefix=global_prefix): + try: + return _read_exception(exc_id=exc_id, prefix=prefix) + except Exception: + log.exception('Failed to read exception `%s` information', exc_id) + # there's no way this can fail, it will crash server badly if it does. + return None + + +def delete_exception(exc_id, prefix=global_prefix): + try: + exc_id_file_path = _find_exc_file(exc_id, prefix=prefix) + if exc_id_file_path: + os.remove(exc_id_file_path) + + except Exception: + log.exception('Failed to remove exception `%s` information', exc_id) + # there's no way this can fail, it will crash server badly if it does. + pass