# HG changeset patch # User Marcin Kuzminski # Date 2018-07-23 13:56:09 # Node ID 5eed39e5099a18dfacdecd6110cd5e93e27c99f5 # Parent c4bda4e1b5457ec63d432e72530f01b874ebec0c exceptions: added new exception tracking capability. - store exceptions - expose super admin interface to browse latest errors diff --git a/rhodecode/apps/admin/__init__.py b/rhodecode/apps/admin/__init__.py --- a/rhodecode/apps/admin/__init__.py +++ b/rhodecode/apps/admin/__init__.py @@ -60,6 +60,19 @@ def admin_routes(config): pattern='/settings/system/updates') config.add_route( + name='admin_settings_exception_tracker', + pattern='/settings/exceptions') + config.add_route( + name='admin_settings_exception_tracker_delete_all', + pattern='/settings/exceptions/delete') + config.add_route( + name='admin_settings_exception_tracker_show', + pattern='/settings/exceptions/{exception_id}') + config.add_route( + name='admin_settings_exception_tracker_delete', + pattern='/settings/exceptions/{exception_id}/delete') + + config.add_route( name='admin_settings_sessions', pattern='/settings/sessions') config.add_route( diff --git a/rhodecode/apps/admin/navigation.py b/rhodecode/apps/admin/navigation.py --- a/rhodecode/apps/admin/navigation.py +++ b/rhodecode/apps/admin/navigation.py @@ -89,6 +89,9 @@ class NavigationRegistry(object): 'global_integrations_home'), NavEntry('system', _('System Info'), 'admin_settings_system'), + NavEntry('exceptions', _('Exceptions Tracker'), + 'admin_settings_exception_tracker', + active_list=['exceptions', 'exceptions_browse']), NavEntry('process_management', _('Processes'), 'admin_settings_process_management'), NavEntry('sessions', _('User Sessions'), diff --git a/rhodecode/apps/admin/views/exception_tracker.py b/rhodecode/apps/admin/views/exception_tracker.py new file mode 100644 --- /dev/null +++ b/rhodecode/apps/admin/views/exception_tracker.py @@ -0,0 +1,153 @@ +# -*- coding: utf-8 -*- + +# Copyright (C) 2018-2018 RhodeCode GmbH +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License, version 3 +# (only), as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +# This program is dual-licensed. If you wish to learn more about the +# RhodeCode Enterprise Edition, including its added features, Support services, +# and proprietary license terms, please see https://rhodecode.com/licenses/ +import os +import logging + +from pyramid.httpexceptions import HTTPFound +from pyramid.view import view_config + +from rhodecode.apps._base import BaseAppView +from rhodecode.apps.admin.navigation import navigation_list +from rhodecode.lib import helpers as h +from rhodecode.lib.auth import ( + LoginRequired, HasPermissionAllDecorator, CSRFRequired) +from rhodecode.lib.utils2 import time_to_utcdatetime +from rhodecode.lib import exc_tracking + +log = logging.getLogger(__name__) + + +class ExceptionsTrackerView(BaseAppView): + def load_default_context(self): + c = self._get_local_tmpl_context() + c.navlist = navigation_list(self.request) + return c + + def count_all_exceptions(self): + exc_store_path = exc_tracking.get_exc_store() + count = 0 + for fname in os.listdir(exc_store_path): + parts = fname.split('_', 2) + if not len(parts) == 3: + continue + count +=1 + return count + + def get_all_exceptions(self, read_metadata=False, limit=None): + exc_store_path = exc_tracking.get_exc_store() + exception_list = [] + + def key_sorter(val): + try: + return val.split('_')[-1] + except Exception: + return 0 + count = 0 + for fname in reversed(sorted(os.listdir(exc_store_path), key=key_sorter)): + + parts = fname.split('_', 2) + if not len(parts) == 3: + continue + + exc_id, app_type, exc_timestamp = parts + + exc = {'exc_id': exc_id, 'app_type': app_type, 'exc_type': 'unknown', + 'exc_utc_date': '', 'exc_timestamp': exc_timestamp} + + if read_metadata: + full_path = os.path.join(exc_store_path, fname) + # we can read our metadata + with open(full_path, 'rb') as f: + exc_metadata = exc_tracking.exc_unserialize(f.read()) + exc.update(exc_metadata) + + # convert our timestamp to a date obj, for nicer representation + exc['exc_utc_date'] = time_to_utcdatetime(exc['exc_timestamp']) + exception_list.append(exc) + + count += 1 + if limit and count >= limit: + break + return exception_list + + @LoginRequired() + @HasPermissionAllDecorator('hg.admin') + @view_config( + route_name='admin_settings_exception_tracker', request_method='GET', + renderer='rhodecode:templates/admin/settings/settings.mako') + def browse_exceptions(self): + _ = self.request.translate + c = self.load_default_context() + c.active = 'exceptions_browse' + c.limit = self.request.GET.get('limit', 50) + c.exception_list = self.get_all_exceptions(read_metadata=True, limit=c.limit) + c.exception_list_count = self.count_all_exceptions() + c.exception_store_dir = exc_tracking.get_exc_store() + return self._get_template_context(c) + + @LoginRequired() + @HasPermissionAllDecorator('hg.admin') + @view_config( + route_name='admin_settings_exception_tracker_show', request_method='GET', + renderer='rhodecode:templates/admin/settings/settings.mako') + def exception_show(self): + _ = self.request.translate + c = self.load_default_context() + + c.active = 'exceptions' + c.exception_id = self.request.matchdict['exception_id'] + c.traceback = exc_tracking.read_exception(c.exception_id, prefix=None) + return self._get_template_context(c) + + @LoginRequired() + @HasPermissionAllDecorator('hg.admin') + @CSRFRequired() + @view_config( + route_name='admin_settings_exception_tracker_delete_all', request_method='POST', + renderer='rhodecode:templates/admin/settings/settings.mako') + def exception_delete_all(self): + _ = self.request.translate + c = self.load_default_context() + + c.active = 'exceptions' + all_exc = self.get_all_exceptions() + exc_count = len(all_exc) + for exc in all_exc: + exc_tracking.delete_exception(exc['exc_id'], prefix=None) + + h.flash(_('Removed {} Exceptions').format(exc_count), category='success') + raise HTTPFound(h.route_path('admin_settings_exception_tracker')) + + @LoginRequired() + @HasPermissionAllDecorator('hg.admin') + @CSRFRequired() + @view_config( + route_name='admin_settings_exception_tracker_delete', request_method='POST', + renderer='rhodecode:templates/admin/settings/settings.mako') + def exception_delete(self): + _ = self.request.translate + c = self.load_default_context() + + c.active = 'exceptions' + c.exception_id = self.request.matchdict['exception_id'] + exc_tracking.delete_exception(c.exception_id, prefix=None) + + h.flash(_('Removed Exception {}').format(c.exception_id), category='success') + raise HTTPFound(h.route_path('admin_settings_exception_tracker')) diff --git a/rhodecode/config/middleware.py b/rhodecode/config/middleware.py --- a/rhodecode/config/middleware.py +++ b/rhodecode/config/middleware.py @@ -19,6 +19,7 @@ # and proprietary license terms, please see https://rhodecode.com/licenses/ import os +import sys import logging import traceback import collections @@ -48,6 +49,7 @@ from rhodecode.lib.middleware.https_fixu from rhodecode.lib.celerylib.loader import configure_celery from rhodecode.lib.plugins.utils import register_rhodecode_plugin from rhodecode.lib.utils2 import aslist as rhodecode_aslist, AttributeDict +from rhodecode.lib.exc_tracking import store_exception from rhodecode.subscribers import ( scan_repositories_if_enabled, write_js_routes_if_enabled, write_metadata_if_needed, inject_app_settings) @@ -172,7 +174,17 @@ def error_handler(exception, request): c.causes = base_response.causes c.messages = helpers.flash.pop_messages(request=request) - c.traceback = traceback.format_exc() + + exc_info = sys.exc_info() + c.exception_id = id(exc_info) + c.show_exception_id = isinstance(base_response, VCSServerUnavailable) \ + or base_response.status_code > 499 + c.exception_id_url = request.route_url( + 'admin_settings_exception_tracker_show', exception_id=c.exception_id) + + if c.show_exception_id: + store_exception(c.exception_id, exc_info) + response = render_to_response( '/errors/error_document.mako', {'c': c, 'h': helpers}, request=request, response=base_response) diff --git a/rhodecode/lib/exc_tracking.py b/rhodecode/lib/exc_tracking.py new file mode 100644 --- /dev/null +++ b/rhodecode/lib/exc_tracking.py @@ -0,0 +1,146 @@ +# -*- coding: utf-8 -*- + +# Copyright (C) 2010-2018 RhodeCode GmbH +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License, version 3 +# (only), as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +# This program is dual-licensed. If you wish to learn more about the +# RhodeCode Enterprise Edition, including its added features, Support services, +# and proprietary license terms, please see https://rhodecode.com/licenses/ + +import os +import time +import datetime +import msgpack +import logging +import traceback +import tempfile + + +log = logging.getLogger(__name__) + +# NOTE: Any changes should be synced with exc_tracking at vcsserver.lib.exc_tracking +global_prefix = 'rhodecode' + + +def exc_serialize(exc_id, tb, exc_type): + + data = { + 'version': 'v1', + 'exc_id': exc_id, + 'exc_utc_date': datetime.datetime.utcnow().isoformat(), + 'exc_timestamp': repr(time.time()), + 'exc_message': tb, + 'exc_type': exc_type, + } + return msgpack.packb(data), data + + +def exc_unserialize(tb): + return msgpack.unpackb(tb) + + +def get_exc_store(): + """ + Get and create exception store if it's not existing + """ + exc_store_dir = 'rc_exception_store_v1' + # fallback + _exc_store_path = os.path.join(tempfile.gettempdir(), exc_store_dir) + + exc_store_dir = '' # TODO: need a persistent cross instance store here + if exc_store_dir: + _exc_store_path = os.path.join(exc_store_dir, exc_store_dir) + + _exc_store_path = os.path.abspath(_exc_store_path) + if not os.path.isdir(_exc_store_path): + os.makedirs(_exc_store_path) + log.debug('Initializing exceptions store at %s', _exc_store_path) + return _exc_store_path + + +def _store_exception(exc_id, exc_info, prefix): + exc_type, exc_value, exc_traceback = exc_info + tb = ''.join(traceback.format_exception( + exc_type, exc_value, exc_traceback, None)) + + exc_type_name = exc_type.__name__ + exc_store_path = get_exc_store() + exc_data, org_data = exc_serialize(exc_id, tb, exc_type_name) + exc_pref_id = '{}_{}_{}'.format(exc_id, prefix, org_data['exc_timestamp']) + if not os.path.isdir(exc_store_path): + os.makedirs(exc_store_path) + stored_exc_path = os.path.join(exc_store_path, exc_pref_id) + with open(stored_exc_path, 'wb') as f: + f.write(exc_data) + log.debug('Stored generated exception %s as: %s', exc_id, stored_exc_path) + + +def store_exception(exc_id, exc_info, prefix=global_prefix): + try: + _store_exception(exc_id=exc_id, exc_info=exc_info, prefix=prefix) + except Exception: + log.exception('Failed to store exception `%s` information', exc_id) + # there's no way this can fail, it will crash server badly if it does. + pass + + +def _find_exc_file(exc_id, prefix=global_prefix): + exc_store_path = get_exc_store() + if prefix: + exc_id = '{}_{}'.format(exc_id, prefix) + else: + # search without a prefix + exc_id = '{}'.format(exc_id) + + # we need to search the store for such start pattern as above + for fname in os.listdir(exc_store_path): + if fname.startswith(exc_id): + exc_id = os.path.join(exc_store_path, fname) + break + continue + else: + exc_id = None + + return exc_id + + +def _read_exception(exc_id, prefix): + exc_id_file_path = _find_exc_file(exc_id=exc_id, prefix=prefix) + if exc_id_file_path: + with open(exc_id_file_path, 'rb') as f: + return exc_unserialize(f.read()) + else: + log.debug('Exception File `%s` not found', exc_id_file_path) + return None + + +def read_exception(exc_id, prefix=global_prefix): + try: + return _read_exception(exc_id=exc_id, prefix=prefix) + except Exception: + log.exception('Failed to read exception `%s` information', exc_id) + # there's no way this can fail, it will crash server badly if it does. + return None + + +def delete_exception(exc_id, prefix=global_prefix): + try: + exc_id_file_path = _find_exc_file(exc_id, prefix=prefix) + if exc_id_file_path: + os.remove(exc_id_file_path) + + except Exception: + log.exception('Failed to remove exception `%s` information', exc_id) + # there's no way this can fail, it will crash server badly if it does. + pass diff --git a/rhodecode/public/css/main-content.less b/rhodecode/public/css/main-content.less --- a/rhodecode/public/css/main-content.less +++ b/rhodecode/public/css/main-content.less @@ -77,6 +77,13 @@ min-width: 100%; } +.main-content-auto-width { + .main-content; + width: auto; + min-width: 100%; + max-width: inherit; +} + .field { clear: left; margin-bottom: @padding; diff --git a/rhodecode/templates/admin/settings/settings.mako b/rhodecode/templates/admin/settings/settings.mako --- a/rhodecode/templates/admin/settings/settings.mako +++ b/rhodecode/templates/admin/settings/settings.mako @@ -44,7 +44,7 @@ -
+
${self.main_content()}
diff --git a/rhodecode/templates/admin/settings/settings_exceptions.mako b/rhodecode/templates/admin/settings/settings_exceptions.mako new file mode 100644 --- /dev/null +++ b/rhodecode/templates/admin/settings/settings_exceptions.mako @@ -0,0 +1,38 @@ +
+
+

${_('Exceptions Tracker - Exception ID')}: ${c.exception_id}

+
+
+ % if c.traceback: + +

${_('Exception `{}` generated on UTC date: {}').format(c.traceback.get('exc_type', 'NO_TYPE'), c.traceback.get('exc_utc_date', 'NO_DATE'))}

+ 
${c.traceback.get('exc_message', 'NO_MESSAGE')}
+ + % else: + ${_('Unable to Read Exception. It might be removed or non-existing.')} + % endif +
+
+ + +% if c.traceback: +
+
+

${_('Delete this Exception')}

+
+
+ ${h.secure_form(h.route_path('admin_settings_exception_tracker_delete', exception_id=c.exception_id), request=request)} +
+ +
+ +
+ + ${h.end_form()} +
+
+% endif diff --git a/rhodecode/templates/admin/settings/settings_exceptions_browse.mako b/rhodecode/templates/admin/settings/settings_exceptions_browse.mako new file mode 100644 --- /dev/null +++ b/rhodecode/templates/admin/settings/settings_exceptions_browse.mako @@ -0,0 +1,52 @@ +
+
+

${_('Exceptions Tracker ')}

+
+
+ % if c.exception_list_count == 1: + ${_('There is {} stored exception.').format(c.exception_list_count)} + % else: + ${_('There are {} stored exceptions.').format(c.exception_list_count)} + % endif + ${_('Store directory')}: ${c.exception_store_dir} + + ${h.secure_form(h.route_path('admin_settings_exception_tracker_delete_all'), request=request)} +
+ +
+ +
+ + ${h.end_form()} + +
+
+ + +
+
+

${_('Exceptions Tracker - Showing the last {} Exceptions').format(c.limit)}

+
+
+ + + + + + + + % for tb in c.exception_list: + + + + + + + % endfor +
Exception IDDateApp TypeExc Type
${tb['exc_id']}${h.format_date(tb['exc_utc_date'])}${tb['app_type']}${tb['exc_type']}
+
+
diff --git a/rhodecode/templates/errors/error_document.mako b/rhodecode/templates/errors/error_document.mako --- a/rhodecode/templates/errors/error_document.mako +++ b/rhodecode/templates/errors/error_document.mako @@ -60,9 +60,17 @@

Support

-

For support, go to ${_('Support')}. +

For help and support, go to the ${_('Support Page')}. It may be useful to include your log file; see the log file locations here.

+ % if c.show_exception_id: +

+ Super Admin can see detailed traceback information from this exception by checking the below Exception ID. + Please include the below link for further details of this exception. + + Exception ID: ${c.exception_id} +

+ % endif

Documentation