##// END OF EJS Templates
exception_tracker: store request info if available.
marcink -
r4301:9712e26b default
parent child Browse files
Show More
@@ -1,222 +1,231 b''
1 # -*- coding: utf-8 -*-
1 # -*- coding: utf-8 -*-
2
2
3 # Copyright (C) 2010-2019 RhodeCode GmbH
3 # Copyright (C) 2010-2019 RhodeCode GmbH
4 #
4 #
5 # This program is free software: you can redistribute it and/or modify
5 # This program is free software: you can redistribute it and/or modify
6 # it under the terms of the GNU Affero General Public License, version 3
6 # it under the terms of the GNU Affero General Public License, version 3
7 # (only), as published by the Free Software Foundation.
7 # (only), as published by the Free Software Foundation.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU Affero General Public License
14 # You should have received a copy of the GNU Affero General Public License
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
16 #
16 #
17 # This program is dual-licensed. If you wish to learn more about the
17 # This program is dual-licensed. If you wish to learn more about the
18 # RhodeCode Enterprise Edition, including its added features, Support services,
18 # RhodeCode Enterprise Edition, including its added features, Support services,
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
20
20
21 import os
21 import os
22 import time
22 import time
23 import datetime
23 import datetime
24 import msgpack
24 import msgpack
25 import logging
25 import logging
26 import traceback
26 import traceback
27 import tempfile
27 import tempfile
28 import glob
28 import glob
29
29
30 log = logging.getLogger(__name__)
30 log = logging.getLogger(__name__)
31
31
32 # NOTE: Any changes should be synced with exc_tracking at vcsserver.lib.exc_tracking
32 # NOTE: Any changes should be synced with exc_tracking at vcsserver.lib.exc_tracking
33 global_prefix = 'rhodecode'
33 global_prefix = 'rhodecode'
34 exc_store_dir_name = 'rc_exception_store_v1'
34 exc_store_dir_name = 'rc_exception_store_v1'
35
35
36
36
37 def exc_serialize(exc_id, tb, exc_type):
37 def exc_serialize(exc_id, tb, exc_type, extra_data=None):
38
38
39 data = {
39 data = {
40 'version': 'v1',
40 'version': 'v1',
41 'exc_id': exc_id,
41 'exc_id': exc_id,
42 'exc_utc_date': datetime.datetime.utcnow().isoformat(),
42 'exc_utc_date': datetime.datetime.utcnow().isoformat(),
43 'exc_timestamp': repr(time.time()),
43 'exc_timestamp': repr(time.time()),
44 'exc_message': tb,
44 'exc_message': tb,
45 'exc_type': exc_type,
45 'exc_type': exc_type,
46 }
46 }
47 if extra_data:
48 data.update(extra_data)
47 return msgpack.packb(data), data
49 return msgpack.packb(data), data
48
50
49
51
50 def exc_unserialize(tb):
52 def exc_unserialize(tb):
51 return msgpack.unpackb(tb)
53 return msgpack.unpackb(tb)
52
54
53 _exc_store = None
55 _exc_store = None
54
56
55
57
56 def get_exc_store():
58 def get_exc_store():
57 """
59 """
58 Get and create exception store if it's not existing
60 Get and create exception store if it's not existing
59 """
61 """
60 global _exc_store
62 global _exc_store
61 import rhodecode as app
63 import rhodecode as app
62
64
63 if _exc_store is not None:
65 if _exc_store is not None:
64 # quick global cache
66 # quick global cache
65 return _exc_store
67 return _exc_store
66
68
67 exc_store_dir = app.CONFIG.get('exception_tracker.store_path', '') or tempfile.gettempdir()
69 exc_store_dir = app.CONFIG.get('exception_tracker.store_path', '') or tempfile.gettempdir()
68 _exc_store_path = os.path.join(exc_store_dir, exc_store_dir_name)
70 _exc_store_path = os.path.join(exc_store_dir, exc_store_dir_name)
69
71
70 _exc_store_path = os.path.abspath(_exc_store_path)
72 _exc_store_path = os.path.abspath(_exc_store_path)
71 if not os.path.isdir(_exc_store_path):
73 if not os.path.isdir(_exc_store_path):
72 os.makedirs(_exc_store_path)
74 os.makedirs(_exc_store_path)
73 log.debug('Initializing exceptions store at %s', _exc_store_path)
75 log.debug('Initializing exceptions store at %s', _exc_store_path)
74 _exc_store = _exc_store_path
76 _exc_store = _exc_store_path
75
77
76 return _exc_store_path
78 return _exc_store_path
77
79
78
80
79 def _store_exception(exc_id, exc_type_name, exc_traceback, prefix, send_email=None):
81 def _store_exception(exc_id, exc_type_name, exc_traceback, prefix, send_email=None):
80 """
82 """
81 Low level function to store exception in the exception tracker
83 Low level function to store exception in the exception tracker
82 """
84 """
85 from pyramid.threadlocal import get_current_request
83 import rhodecode as app
86 import rhodecode as app
87 request = get_current_request()
88 extra_data = {}
89 # NOTE(marcink): store request information into exc_data
90 if request:
91 extra_data['client_address'] = getattr(request, 'client_addr', '')
92 extra_data['user_agent'] = getattr(request, 'user_agent', '')
93 extra_data['method'] = getattr(request, 'method', '')
94 extra_data['url'] = getattr(request, 'url', '')
84
95
85 exc_store_path = get_exc_store()
96 exc_store_path = get_exc_store()
86 exc_data, org_data = exc_serialize(exc_id, exc_traceback, exc_type_name)
97 exc_data, org_data = exc_serialize(exc_id, exc_traceback, exc_type_name, extra_data=extra_data)
98
87 exc_pref_id = '{}_{}_{}'.format(exc_id, prefix, org_data['exc_timestamp'])
99 exc_pref_id = '{}_{}_{}'.format(exc_id, prefix, org_data['exc_timestamp'])
88 if not os.path.isdir(exc_store_path):
100 if not os.path.isdir(exc_store_path):
89 os.makedirs(exc_store_path)
101 os.makedirs(exc_store_path)
90 stored_exc_path = os.path.join(exc_store_path, exc_pref_id)
102 stored_exc_path = os.path.join(exc_store_path, exc_pref_id)
91 with open(stored_exc_path, 'wb') as f:
103 with open(stored_exc_path, 'wb') as f:
92 f.write(exc_data)
104 f.write(exc_data)
93 log.debug('Stored generated exception %s as: %s', exc_id, stored_exc_path)
105 log.debug('Stored generated exception %s as: %s', exc_id, stored_exc_path)
94
106
95 if send_email is None:
107 if send_email is None:
96 # NOTE(marcink): read app config unless we specify explicitly
108 # NOTE(marcink): read app config unless we specify explicitly
97 send_email = app.CONFIG.get('exception_tracker.send_email', False)
109 send_email = app.CONFIG.get('exception_tracker.send_email', False)
98
110
99 mail_server = app.CONFIG.get('smtp_server') or None
111 mail_server = app.CONFIG.get('smtp_server') or None
100 send_email = send_email and mail_server
112 send_email = send_email and mail_server
101 if send_email:
113 if send_email:
102 try:
114 try:
103 send_exc_email(exc_id, exc_type_name)
115 send_exc_email(request, exc_id, exc_type_name)
104 except Exception:
116 except Exception:
105 log.exception('Failed to send exception email')
117 log.exception('Failed to send exception email')
106 pass
118 pass
107
119
108
120
109 def send_exc_email(exc_id, exc_type_name):
121 def send_exc_email(request, exc_id, exc_type_name):
110 import rhodecode as app
122 import rhodecode as app
111 from pyramid.threadlocal import get_current_request
112 from rhodecode.apps._base import TemplateArgs
123 from rhodecode.apps._base import TemplateArgs
113 from rhodecode.lib.utils2 import aslist
124 from rhodecode.lib.utils2 import aslist
114 from rhodecode.lib.celerylib import run_task, tasks
125 from rhodecode.lib.celerylib import run_task, tasks
115 from rhodecode.lib.base import attach_context_attributes
126 from rhodecode.lib.base import attach_context_attributes
116 from rhodecode.model.notification import EmailNotificationModel
127 from rhodecode.model.notification import EmailNotificationModel
117
128
118 request = get_current_request()
119
120 recipients = aslist(app.CONFIG.get('exception_tracker.send_email_recipients', ''))
129 recipients = aslist(app.CONFIG.get('exception_tracker.send_email_recipients', ''))
121 log.debug('Sending Email exception to: `%s`', recipients or 'all super admins')
130 log.debug('Sending Email exception to: `%s`', recipients or 'all super admins')
122
131
123 # NOTE(marcink): needed for email template rendering
132 # NOTE(marcink): needed for email template rendering
124 user_id = None
133 user_id = None
125 if request:
134 if request:
126 user_id = request.user.user_id
135 user_id = request.user.user_id
127 attach_context_attributes(TemplateArgs(), request, user_id=user_id, is_api=True)
136 attach_context_attributes(TemplateArgs(), request, user_id=user_id, is_api=True)
128
137
129 email_kwargs = {
138 email_kwargs = {
130 'email_prefix': app.CONFIG.get('exception_tracker.email_prefix', '') or '[RHODECODE ERROR]',
139 'email_prefix': app.CONFIG.get('exception_tracker.email_prefix', '') or '[RHODECODE ERROR]',
131 'exc_url': request.route_url('admin_settings_exception_tracker_show', exception_id=exc_id),
140 'exc_url': request.route_url('admin_settings_exception_tracker_show', exception_id=exc_id),
132 'exc_id': exc_id,
141 'exc_id': exc_id,
133 'exc_type_name': exc_type_name,
142 'exc_type_name': exc_type_name,
134 'exc_traceback': read_exception(exc_id, prefix=None),
143 'exc_traceback': read_exception(exc_id, prefix=None),
135 }
144 }
136
145
137 (subject, headers, email_body,
146 (subject, headers, email_body,
138 email_body_plaintext) = EmailNotificationModel().render_email(
147 email_body_plaintext) = EmailNotificationModel().render_email(
139 EmailNotificationModel.TYPE_EMAIL_EXCEPTION, **email_kwargs)
148 EmailNotificationModel.TYPE_EMAIL_EXCEPTION, **email_kwargs)
140
149
141 run_task(tasks.send_email, recipients, subject,
150 run_task(tasks.send_email, recipients, subject,
142 email_body_plaintext, email_body)
151 email_body_plaintext, email_body)
143
152
144
153
145 def _prepare_exception(exc_info):
154 def _prepare_exception(exc_info):
146 exc_type, exc_value, exc_traceback = exc_info
155 exc_type, exc_value, exc_traceback = exc_info
147 exc_type_name = exc_type.__name__
156 exc_type_name = exc_type.__name__
148
157
149 tb = ''.join(traceback.format_exception(
158 tb = ''.join(traceback.format_exception(
150 exc_type, exc_value, exc_traceback, None))
159 exc_type, exc_value, exc_traceback, None))
151
160
152 return exc_type_name, tb
161 return exc_type_name, tb
153
162
154
163
155 def store_exception(exc_id, exc_info, prefix=global_prefix):
164 def store_exception(exc_id, exc_info, prefix=global_prefix):
156 """
165 """
157 Example usage::
166 Example usage::
158
167
159 exc_info = sys.exc_info()
168 exc_info = sys.exc_info()
160 store_exception(id(exc_info), exc_info)
169 store_exception(id(exc_info), exc_info)
161 """
170 """
162
171
163 try:
172 try:
164 exc_type_name, exc_traceback = _prepare_exception(exc_info)
173 exc_type_name, exc_traceback = _prepare_exception(exc_info)
165 _store_exception(exc_id=exc_id, exc_type_name=exc_type_name,
174 _store_exception(exc_id=exc_id, exc_type_name=exc_type_name,
166 exc_traceback=exc_traceback, prefix=prefix)
175 exc_traceback=exc_traceback, prefix=prefix)
167 return exc_id, exc_type_name
176 return exc_id, exc_type_name
168 except Exception:
177 except Exception:
169 log.exception('Failed to store exception `%s` information', exc_id)
178 log.exception('Failed to store exception `%s` information', exc_id)
170 # there's no way this can fail, it will crash server badly if it does.
179 # there's no way this can fail, it will crash server badly if it does.
171 pass
180 pass
172
181
173
182
174 def _find_exc_file(exc_id, prefix=global_prefix):
183 def _find_exc_file(exc_id, prefix=global_prefix):
175 exc_store_path = get_exc_store()
184 exc_store_path = get_exc_store()
176 if prefix:
185 if prefix:
177 exc_id = '{}_{}'.format(exc_id, prefix)
186 exc_id = '{}_{}'.format(exc_id, prefix)
178 else:
187 else:
179 # search without a prefix
188 # search without a prefix
180 exc_id = '{}'.format(exc_id)
189 exc_id = '{}'.format(exc_id)
181
190
182 found_exc_id = None
191 found_exc_id = None
183 matches = glob.glob(os.path.join(exc_store_path, exc_id) + '*')
192 matches = glob.glob(os.path.join(exc_store_path, exc_id) + '*')
184 if matches:
193 if matches:
185 found_exc_id = matches[0]
194 found_exc_id = matches[0]
186
195
187 return found_exc_id
196 return found_exc_id
188
197
189
198
190 def _read_exception(exc_id, prefix):
199 def _read_exception(exc_id, prefix):
191 exc_id_file_path = _find_exc_file(exc_id=exc_id, prefix=prefix)
200 exc_id_file_path = _find_exc_file(exc_id=exc_id, prefix=prefix)
192 if exc_id_file_path:
201 if exc_id_file_path:
193 with open(exc_id_file_path, 'rb') as f:
202 with open(exc_id_file_path, 'rb') as f:
194 return exc_unserialize(f.read())
203 return exc_unserialize(f.read())
195 else:
204 else:
196 log.debug('Exception File `%s` not found', exc_id_file_path)
205 log.debug('Exception File `%s` not found', exc_id_file_path)
197 return None
206 return None
198
207
199
208
200 def read_exception(exc_id, prefix=global_prefix):
209 def read_exception(exc_id, prefix=global_prefix):
201 try:
210 try:
202 return _read_exception(exc_id=exc_id, prefix=prefix)
211 return _read_exception(exc_id=exc_id, prefix=prefix)
203 except Exception:
212 except Exception:
204 log.exception('Failed to read exception `%s` information', exc_id)
213 log.exception('Failed to read exception `%s` information', exc_id)
205 # there's no way this can fail, it will crash server badly if it does.
214 # there's no way this can fail, it will crash server badly if it does.
206 return None
215 return None
207
216
208
217
209 def delete_exception(exc_id, prefix=global_prefix):
218 def delete_exception(exc_id, prefix=global_prefix):
210 try:
219 try:
211 exc_id_file_path = _find_exc_file(exc_id, prefix=prefix)
220 exc_id_file_path = _find_exc_file(exc_id, prefix=prefix)
212 if exc_id_file_path:
221 if exc_id_file_path:
213 os.remove(exc_id_file_path)
222 os.remove(exc_id_file_path)
214
223
215 except Exception:
224 except Exception:
216 log.exception('Failed to remove exception `%s` information', exc_id)
225 log.exception('Failed to remove exception `%s` information', exc_id)
217 # there's no way this can fail, it will crash server badly if it does.
226 # there's no way this can fail, it will crash server badly if it does.
218 pass
227 pass
219
228
220
229
221 def generate_id():
230 def generate_id():
222 return id(object())
231 return id(object())
@@ -1,38 +1,47 b''
1 <div class="panel panel-default">
1 <div class="panel panel-default">
2 <div class="panel-heading">
2 <div class="panel-heading">
3 <h3 class="panel-title">${_('Exceptions Tracker - Exception ID')}: ${c.exception_id}</h3>
3 <h3 class="panel-title">${_('Exceptions Tracker - Exception ID')}: ${c.exception_id}</h3>
4 </div>
4 </div>
5 <div class="panel-body">
5 <div class="panel-body">
6 % if c.traceback:
6 % if c.traceback:
7
7
8 <h4>${_('Exception `{}` generated on UTC date: {}').format(c.traceback.get('exc_type', 'NO_TYPE'), c.traceback.get('exc_utc_date', 'NO_DATE'))}</h4>
8 <h4>${_('Exception `{}` generated on UTC date: {}').format(c.traceback.get('exc_type', 'NO_TYPE'), c.traceback.get('exc_utc_date', 'NO_DATE'))}</h4>
9 % if c.traceback.get('url'):
10 Request:
11 <code>${c.traceback.get('method')} ${c.traceback.get('url')}</code><br/>
12 <code>${c.traceback.get('client_address')} ${c.traceback.get('user_agent')}</code>
13 <br/>
14 <br/>
15 % endif
16
9 <pre>${c.traceback.get('exc_message', 'NO_MESSAGE')}</pre>
17 <pre>${c.traceback.get('exc_message', 'NO_MESSAGE')}</pre>
10
18
19
11 % else:
20 % else:
12 ${_('Unable to Read Exception. It might be removed or non-existing.')}
21 ${_('Unable to Read Exception. It might be removed or non-existing.')}
13 % endif
22 % endif
14 </div>
23 </div>
15 </div>
24 </div>
16
25
17
26
18 % if c.traceback:
27 % if c.traceback:
19 <div class="panel panel-danger">
28 <div class="panel panel-danger">
20 <div class="panel-heading" id="advanced-delete">
29 <div class="panel-heading" id="advanced-delete">
21 <h3 class="panel-title">${_('Delete this Exception')}</h3>
30 <h3 class="panel-title">${_('Delete this Exception')}</h3>
22 </div>
31 </div>
23 <div class="panel-body">
32 <div class="panel-body">
24 ${h.secure_form(h.route_path('admin_settings_exception_tracker_delete', exception_id=c.exception_id), request=request)}
33 ${h.secure_form(h.route_path('admin_settings_exception_tracker_delete', exception_id=c.exception_id), request=request)}
25 <div style="margin: 0 0 20px 0" class="fake-space"></div>
34 <div style="margin: 0 0 20px 0" class="fake-space"></div>
26
35
27 <div class="field">
36 <div class="field">
28 <button class="btn btn-small btn-danger" type="submit"
37 <button class="btn btn-small btn-danger" type="submit"
29 onclick="return confirm('${_('Confirm to delete this exception')}');">
38 onclick="return confirm('${_('Confirm to delete this exception')}');">
30 <i class="icon-remove"></i>
39 <i class="icon-remove"></i>
31 ${_('Delete This Exception')}
40 ${_('Delete This Exception')}
32 </button>
41 </button>
33 </div>
42 </div>
34
43
35 ${h.end_form()}
44 ${h.end_form()}
36 </div>
45 </div>
37 </div>
46 </div>
38 % endif
47 % endif
General Comments 0
You need to be logged in to leave comments. Login now