##// END OF EJS Templates
exceptions: added new exception tracking capability....
marcink -
r2907:5eed39e5 default
parent child
Show More
@@ -0,0 +1,153
1 # -*- coding: utf-8 -*-
2
3 # Copyright (C) 2018-2018 RhodeCode GmbH
4 #
5 # This program is free software: you can redistribute it and/or modify
6 # it under the terms of the GNU Affero General Public License, version 3
7 # (only), as published by the Free Software Foundation.
8 #
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
13 #
14 # You should have received a copy of the GNU Affero General Public License
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
16 #
17 # This program is dual-licensed. If you wish to learn more about the
18 # RhodeCode Enterprise Edition, including its added features, Support services,
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
20 import os
21 import logging
22
23 from pyramid.httpexceptions import HTTPFound
24 from pyramid.view import view_config
25
26 from rhodecode.apps._base import BaseAppView
27 from rhodecode.apps.admin.navigation import navigation_list
28 from rhodecode.lib import helpers as h
29 from rhodecode.lib.auth import (
30 LoginRequired, HasPermissionAllDecorator, CSRFRequired)
31 from rhodecode.lib.utils2 import time_to_utcdatetime
32 from rhodecode.lib import exc_tracking
33
34 log = logging.getLogger(__name__)
35
36
37 class ExceptionsTrackerView(BaseAppView):
38 def load_default_context(self):
39 c = self._get_local_tmpl_context()
40 c.navlist = navigation_list(self.request)
41 return c
42
43 def count_all_exceptions(self):
44 exc_store_path = exc_tracking.get_exc_store()
45 count = 0
46 for fname in os.listdir(exc_store_path):
47 parts = fname.split('_', 2)
48 if not len(parts) == 3:
49 continue
50 count +=1
51 return count
52
53 def get_all_exceptions(self, read_metadata=False, limit=None):
54 exc_store_path = exc_tracking.get_exc_store()
55 exception_list = []
56
57 def key_sorter(val):
58 try:
59 return val.split('_')[-1]
60 except Exception:
61 return 0
62 count = 0
63 for fname in reversed(sorted(os.listdir(exc_store_path), key=key_sorter)):
64
65 parts = fname.split('_', 2)
66 if not len(parts) == 3:
67 continue
68
69 exc_id, app_type, exc_timestamp = parts
70
71 exc = {'exc_id': exc_id, 'app_type': app_type, 'exc_type': 'unknown',
72 'exc_utc_date': '', 'exc_timestamp': exc_timestamp}
73
74 if read_metadata:
75 full_path = os.path.join(exc_store_path, fname)
76 # we can read our metadata
77 with open(full_path, 'rb') as f:
78 exc_metadata = exc_tracking.exc_unserialize(f.read())
79 exc.update(exc_metadata)
80
81 # convert our timestamp to a date obj, for nicer representation
82 exc['exc_utc_date'] = time_to_utcdatetime(exc['exc_timestamp'])
83 exception_list.append(exc)
84
85 count += 1
86 if limit and count >= limit:
87 break
88 return exception_list
89
90 @LoginRequired()
91 @HasPermissionAllDecorator('hg.admin')
92 @view_config(
93 route_name='admin_settings_exception_tracker', request_method='GET',
94 renderer='rhodecode:templates/admin/settings/settings.mako')
95 def browse_exceptions(self):
96 _ = self.request.translate
97 c = self.load_default_context()
98 c.active = 'exceptions_browse'
99 c.limit = self.request.GET.get('limit', 50)
100 c.exception_list = self.get_all_exceptions(read_metadata=True, limit=c.limit)
101 c.exception_list_count = self.count_all_exceptions()
102 c.exception_store_dir = exc_tracking.get_exc_store()
103 return self._get_template_context(c)
104
105 @LoginRequired()
106 @HasPermissionAllDecorator('hg.admin')
107 @view_config(
108 route_name='admin_settings_exception_tracker_show', request_method='GET',
109 renderer='rhodecode:templates/admin/settings/settings.mako')
110 def exception_show(self):
111 _ = self.request.translate
112 c = self.load_default_context()
113
114 c.active = 'exceptions'
115 c.exception_id = self.request.matchdict['exception_id']
116 c.traceback = exc_tracking.read_exception(c.exception_id, prefix=None)
117 return self._get_template_context(c)
118
119 @LoginRequired()
120 @HasPermissionAllDecorator('hg.admin')
121 @CSRFRequired()
122 @view_config(
123 route_name='admin_settings_exception_tracker_delete_all', request_method='POST',
124 renderer='rhodecode:templates/admin/settings/settings.mako')
125 def exception_delete_all(self):
126 _ = self.request.translate
127 c = self.load_default_context()
128
129 c.active = 'exceptions'
130 all_exc = self.get_all_exceptions()
131 exc_count = len(all_exc)
132 for exc in all_exc:
133 exc_tracking.delete_exception(exc['exc_id'], prefix=None)
134
135 h.flash(_('Removed {} Exceptions').format(exc_count), category='success')
136 raise HTTPFound(h.route_path('admin_settings_exception_tracker'))
137
138 @LoginRequired()
139 @HasPermissionAllDecorator('hg.admin')
140 @CSRFRequired()
141 @view_config(
142 route_name='admin_settings_exception_tracker_delete', request_method='POST',
143 renderer='rhodecode:templates/admin/settings/settings.mako')
144 def exception_delete(self):
145 _ = self.request.translate
146 c = self.load_default_context()
147
148 c.active = 'exceptions'
149 c.exception_id = self.request.matchdict['exception_id']
150 exc_tracking.delete_exception(c.exception_id, prefix=None)
151
152 h.flash(_('Removed Exception {}').format(c.exception_id), category='success')
153 raise HTTPFound(h.route_path('admin_settings_exception_tracker'))
@@ -0,0 +1,146
1 # -*- coding: utf-8 -*-
2
3 # Copyright (C) 2010-2018 RhodeCode GmbH
4 #
5 # This program is free software: you can redistribute it and/or modify
6 # it under the terms of the GNU Affero General Public License, version 3
7 # (only), as published by the Free Software Foundation.
8 #
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
13 #
14 # You should have received a copy of the GNU Affero General Public License
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
16 #
17 # This program is dual-licensed. If you wish to learn more about the
18 # RhodeCode Enterprise Edition, including its added features, Support services,
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
20
21 import os
22 import time
23 import datetime
24 import msgpack
25 import logging
26 import traceback
27 import tempfile
28
29
30 log = logging.getLogger(__name__)
31
32 # NOTE: Any changes should be synced with exc_tracking at vcsserver.lib.exc_tracking
33 global_prefix = 'rhodecode'
34
35
36 def exc_serialize(exc_id, tb, exc_type):
37
38 data = {
39 'version': 'v1',
40 'exc_id': exc_id,
41 'exc_utc_date': datetime.datetime.utcnow().isoformat(),
42 'exc_timestamp': repr(time.time()),
43 'exc_message': tb,
44 'exc_type': exc_type,
45 }
46 return msgpack.packb(data), data
47
48
49 def exc_unserialize(tb):
50 return msgpack.unpackb(tb)
51
52
53 def get_exc_store():
54 """
55 Get and create exception store if it's not existing
56 """
57 exc_store_dir = 'rc_exception_store_v1'
58 # fallback
59 _exc_store_path = os.path.join(tempfile.gettempdir(), exc_store_dir)
60
61 exc_store_dir = '' # TODO: need a persistent cross instance store here
62 if exc_store_dir:
63 _exc_store_path = os.path.join(exc_store_dir, exc_store_dir)
64
65 _exc_store_path = os.path.abspath(_exc_store_path)
66 if not os.path.isdir(_exc_store_path):
67 os.makedirs(_exc_store_path)
68 log.debug('Initializing exceptions store at %s', _exc_store_path)
69 return _exc_store_path
70
71
72 def _store_exception(exc_id, exc_info, prefix):
73 exc_type, exc_value, exc_traceback = exc_info
74 tb = ''.join(traceback.format_exception(
75 exc_type, exc_value, exc_traceback, None))
76
77 exc_type_name = exc_type.__name__
78 exc_store_path = get_exc_store()
79 exc_data, org_data = exc_serialize(exc_id, tb, exc_type_name)
80 exc_pref_id = '{}_{}_{}'.format(exc_id, prefix, org_data['exc_timestamp'])
81 if not os.path.isdir(exc_store_path):
82 os.makedirs(exc_store_path)
83 stored_exc_path = os.path.join(exc_store_path, exc_pref_id)
84 with open(stored_exc_path, 'wb') as f:
85 f.write(exc_data)
86 log.debug('Stored generated exception %s as: %s', exc_id, stored_exc_path)
87
88
89 def store_exception(exc_id, exc_info, prefix=global_prefix):
90 try:
91 _store_exception(exc_id=exc_id, exc_info=exc_info, prefix=prefix)
92 except Exception:
93 log.exception('Failed to store exception `%s` information', exc_id)
94 # there's no way this can fail, it will crash server badly if it does.
95 pass
96
97
98 def _find_exc_file(exc_id, prefix=global_prefix):
99 exc_store_path = get_exc_store()
100 if prefix:
101 exc_id = '{}_{}'.format(exc_id, prefix)
102 else:
103 # search without a prefix
104 exc_id = '{}'.format(exc_id)
105
106 # we need to search the store for such start pattern as above
107 for fname in os.listdir(exc_store_path):
108 if fname.startswith(exc_id):
109 exc_id = os.path.join(exc_store_path, fname)
110 break
111 continue
112 else:
113 exc_id = None
114
115 return exc_id
116
117
118 def _read_exception(exc_id, prefix):
119 exc_id_file_path = _find_exc_file(exc_id=exc_id, prefix=prefix)
120 if exc_id_file_path:
121 with open(exc_id_file_path, 'rb') as f:
122 return exc_unserialize(f.read())
123 else:
124 log.debug('Exception File `%s` not found', exc_id_file_path)
125 return None
126
127
128 def read_exception(exc_id, prefix=global_prefix):
129 try:
130 return _read_exception(exc_id=exc_id, prefix=prefix)
131 except Exception:
132 log.exception('Failed to read exception `%s` information', exc_id)
133 # there's no way this can fail, it will crash server badly if it does.
134 return None
135
136
137 def delete_exception(exc_id, prefix=global_prefix):
138 try:
139 exc_id_file_path = _find_exc_file(exc_id, prefix=prefix)
140 if exc_id_file_path:
141 os.remove(exc_id_file_path)
142
143 except Exception:
144 log.exception('Failed to remove exception `%s` information', exc_id)
145 # there's no way this can fail, it will crash server badly if it does.
146 pass
@@ -0,0 +1,38
1 <div class="panel panel-default">
2 <div class="panel-heading">
3 <h3 class="panel-title">${_('Exceptions Tracker - Exception ID')}: ${c.exception_id}</h3>
4 </div>
5 <div class="panel-body">
6 % if c.traceback:
7
8 <h4>${_('Exception `{}` generated on UTC date: {}').format(c.traceback.get('exc_type', 'NO_TYPE'), c.traceback.get('exc_utc_date', 'NO_DATE'))}</h4>
9 <pre>${c.traceback.get('exc_message', 'NO_MESSAGE')}</pre>
10
11 % else:
12 ${_('Unable to Read Exception. It might be removed or non-existing.')}
13 % endif
14 </div>
15 </div>
16
17
18 % if c.traceback:
19 <div class="panel panel-danger">
20 <div class="panel-heading" id="advanced-delete">
21 <h3 class="panel-title">${_('Delete this Exception')}</h3>
22 </div>
23 <div class="panel-body">
24 ${h.secure_form(h.route_path('admin_settings_exception_tracker_delete', exception_id=c.exception_id), request=request)}
25 <div style="margin: 0 0 20px 0" class="fake-space"></div>
26
27 <div class="field">
28 <button class="btn btn-small btn-danger" type="submit"
29 onclick="return confirm('${_('Confirm to delete this exception')}');">
30 <i class="icon-remove-sign"></i>
31 ${_('Delete This Exception')}
32 </button>
33 </div>
34
35 ${h.end_form()}
36 </div>
37 </div>
38 % endif
@@ -0,0 +1,52
1 <div class="panel panel-default">
2 <div class="panel-heading">
3 <h3 class="panel-title">${_('Exceptions Tracker ')}</h3>
4 </div>
5 <div class="panel-body">
6 % if c.exception_list_count == 1:
7 ${_('There is {} stored exception.').format(c.exception_list_count)}
8 % else:
9 ${_('There are {} stored exceptions.').format(c.exception_list_count)}
10 % endif
11 ${_('Store directory')}: ${c.exception_store_dir}
12
13 ${h.secure_form(h.route_path('admin_settings_exception_tracker_delete_all'), request=request)}
14 <div style="margin: 0 0 20px 0" class="fake-space"></div>
15
16 <div class="field">
17 <button class="btn btn-small btn-danger" type="submit"
18 onclick="return confirm('${_('Confirm to delete all exceptions')}');">
19 <i class="icon-remove-sign"></i>
20 ${_('Delete All')}
21 </button>
22 </div>
23
24 ${h.end_form()}
25
26 </div>
27 </div>
28
29
30 <div class="panel panel-default">
31 <div class="panel-heading">
32 <h3 class="panel-title">${_('Exceptions Tracker - Showing the last {} Exceptions').format(c.limit)}</h3>
33 </div>
34 <div class="panel-body">
35 <table class="rctable">
36 <tr>
37 <th>Exception ID</th>
38 <th>Date</th>
39 <th>App Type</th>
40 <th>Exc Type</th>
41 </tr>
42 % for tb in c.exception_list:
43 <tr>
44 <td><a href="${h.route_path('admin_settings_exception_tracker_show', exception_id=tb['exc_id'])}"><code>${tb['exc_id']}</code></a></td>
45 <td>${h.format_date(tb['exc_utc_date'])}</td>
46 <td>${tb['app_type']}</td>
47 <td>${tb['exc_type']}</td>
48 </tr>
49 % endfor
50 </table>
51 </div>
52 </div>
@@ -60,6 +60,19 def admin_routes(config):
60 pattern='/settings/system/updates')
60 pattern='/settings/system/updates')
61
61
62 config.add_route(
62 config.add_route(
63 name='admin_settings_exception_tracker',
64 pattern='/settings/exceptions')
65 config.add_route(
66 name='admin_settings_exception_tracker_delete_all',
67 pattern='/settings/exceptions/delete')
68 config.add_route(
69 name='admin_settings_exception_tracker_show',
70 pattern='/settings/exceptions/{exception_id}')
71 config.add_route(
72 name='admin_settings_exception_tracker_delete',
73 pattern='/settings/exceptions/{exception_id}/delete')
74
75 config.add_route(
63 name='admin_settings_sessions',
76 name='admin_settings_sessions',
64 pattern='/settings/sessions')
77 pattern='/settings/sessions')
65 config.add_route(
78 config.add_route(
@@ -89,6 +89,9 class NavigationRegistry(object):
89 'global_integrations_home'),
89 'global_integrations_home'),
90 NavEntry('system', _('System Info'),
90 NavEntry('system', _('System Info'),
91 'admin_settings_system'),
91 'admin_settings_system'),
92 NavEntry('exceptions', _('Exceptions Tracker'),
93 'admin_settings_exception_tracker',
94 active_list=['exceptions', 'exceptions_browse']),
92 NavEntry('process_management', _('Processes'),
95 NavEntry('process_management', _('Processes'),
93 'admin_settings_process_management'),
96 'admin_settings_process_management'),
94 NavEntry('sessions', _('User Sessions'),
97 NavEntry('sessions', _('User Sessions'),
@@ -19,6 +19,7
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
20
20
21 import os
21 import os
22 import sys
22 import logging
23 import logging
23 import traceback
24 import traceback
24 import collections
25 import collections
@@ -48,6 +49,7 from rhodecode.lib.middleware.https_fixu
48 from rhodecode.lib.celerylib.loader import configure_celery
49 from rhodecode.lib.celerylib.loader import configure_celery
49 from rhodecode.lib.plugins.utils import register_rhodecode_plugin
50 from rhodecode.lib.plugins.utils import register_rhodecode_plugin
50 from rhodecode.lib.utils2 import aslist as rhodecode_aslist, AttributeDict
51 from rhodecode.lib.utils2 import aslist as rhodecode_aslist, AttributeDict
52 from rhodecode.lib.exc_tracking import store_exception
51 from rhodecode.subscribers import (
53 from rhodecode.subscribers import (
52 scan_repositories_if_enabled, write_js_routes_if_enabled,
54 scan_repositories_if_enabled, write_js_routes_if_enabled,
53 write_metadata_if_needed, inject_app_settings)
55 write_metadata_if_needed, inject_app_settings)
@@ -172,7 +174,17 def error_handler(exception, request):
172 c.causes = base_response.causes
174 c.causes = base_response.causes
173
175
174 c.messages = helpers.flash.pop_messages(request=request)
176 c.messages = helpers.flash.pop_messages(request=request)
175 c.traceback = traceback.format_exc()
177
178 exc_info = sys.exc_info()
179 c.exception_id = id(exc_info)
180 c.show_exception_id = isinstance(base_response, VCSServerUnavailable) \
181 or base_response.status_code > 499
182 c.exception_id_url = request.route_url(
183 'admin_settings_exception_tracker_show', exception_id=c.exception_id)
184
185 if c.show_exception_id:
186 store_exception(c.exception_id, exc_info)
187
176 response = render_to_response(
188 response = render_to_response(
177 '/errors/error_document.mako', {'c': c, 'h': helpers}, request=request,
189 '/errors/error_document.mako', {'c': c, 'h': helpers}, request=request,
178 response=base_response)
190 response=base_response)
@@ -77,6 +77,13
77 min-width: 100%;
77 min-width: 100%;
78 }
78 }
79
79
80 .main-content-auto-width {
81 .main-content;
82 width: auto;
83 min-width: 100%;
84 max-width: inherit;
85 }
86
80 .field {
87 .field {
81 clear: left;
88 clear: left;
82 margin-bottom: @padding;
89 margin-bottom: @padding;
@@ -44,7 +44,7
44 </ul>
44 </ul>
45 </div>
45 </div>
46
46
47 <div class="main-content-full-width">
47 <div class="main-content-auto-width">
48 ${self.main_content()}
48 ${self.main_content()}
49 </div>
49 </div>
50 </div>
50 </div>
@@ -60,9 +60,17
60 </div>
60 </div>
61 <div class="inner-column">
61 <div class="inner-column">
62 <h4>Support</h4>
62 <h4>Support</h4>
63 <p>For support, go to <a href="${c.visual.rhodecode_support_url}" target="_blank">${_('Support')}</a>.
63 <p>For help and support, go to the <a href="${c.visual.rhodecode_support_url}" target="_blank">${_('Support Page')}</a>.
64 It may be useful to include your log file; see the log file locations <a href="${h.route_url('enterprise_log_file_locations')}">here</a>.
64 It may be useful to include your log file; see the log file locations <a href="${h.route_url('enterprise_log_file_locations')}">here</a>.
65 </p>
65 </p>
66 % if c.show_exception_id:
67 <p>
68 Super Admin can see detailed traceback information from this exception by checking the below Exception ID.
69 Please include the below link for further details of this exception.
70
71 Exception ID: <a href="${c.exception_id_url}">${c.exception_id}</a>
72 </p>
73 % endif
66 </div>
74 </div>
67 <div class="inner-column">
75 <div class="inner-column">
68 <h4>Documentation</h4>
76 <h4>Documentation</h4>
General Comments 0
You need to be logged in to leave comments. Login now