##// END OF EJS Templates
exc_tracker: also check for smtp_server before doing heavy lifting of exception email sending
marcink -
r4289:6bfdee89 default
parent child Browse files
Show More
@@ -1,220 +1,222 b''
1 # -*- coding: utf-8 -*-
1 # -*- coding: utf-8 -*-
2
2
3 # Copyright (C) 2010-2019 RhodeCode GmbH
3 # Copyright (C) 2010-2019 RhodeCode GmbH
4 #
4 #
5 # This program is free software: you can redistribute it and/or modify
5 # This program is free software: you can redistribute it and/or modify
6 # it under the terms of the GNU Affero General Public License, version 3
6 # it under the terms of the GNU Affero General Public License, version 3
7 # (only), as published by the Free Software Foundation.
7 # (only), as published by the Free Software Foundation.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU Affero General Public License
14 # You should have received a copy of the GNU Affero General Public License
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
16 #
16 #
17 # This program is dual-licensed. If you wish to learn more about the
17 # This program is dual-licensed. If you wish to learn more about the
18 # RhodeCode Enterprise Edition, including its added features, Support services,
18 # RhodeCode Enterprise Edition, including its added features, Support services,
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
20
20
21 import os
21 import os
22 import time
22 import time
23 import datetime
23 import datetime
24 import msgpack
24 import msgpack
25 import logging
25 import logging
26 import traceback
26 import traceback
27 import tempfile
27 import tempfile
28 import glob
28 import glob
29
29
30 log = logging.getLogger(__name__)
30 log = logging.getLogger(__name__)
31
31
32 # NOTE: Any changes should be synced with exc_tracking at vcsserver.lib.exc_tracking
32 # NOTE: Any changes should be synced with exc_tracking at vcsserver.lib.exc_tracking
33 global_prefix = 'rhodecode'
33 global_prefix = 'rhodecode'
34 exc_store_dir_name = 'rc_exception_store_v1'
34 exc_store_dir_name = 'rc_exception_store_v1'
35
35
36
36
37 def exc_serialize(exc_id, tb, exc_type):
37 def exc_serialize(exc_id, tb, exc_type):
38
38
39 data = {
39 data = {
40 'version': 'v1',
40 'version': 'v1',
41 'exc_id': exc_id,
41 'exc_id': exc_id,
42 'exc_utc_date': datetime.datetime.utcnow().isoformat(),
42 'exc_utc_date': datetime.datetime.utcnow().isoformat(),
43 'exc_timestamp': repr(time.time()),
43 'exc_timestamp': repr(time.time()),
44 'exc_message': tb,
44 'exc_message': tb,
45 'exc_type': exc_type,
45 'exc_type': exc_type,
46 }
46 }
47 return msgpack.packb(data), data
47 return msgpack.packb(data), data
48
48
49
49
50 def exc_unserialize(tb):
50 def exc_unserialize(tb):
51 return msgpack.unpackb(tb)
51 return msgpack.unpackb(tb)
52
52
53 _exc_store = None
53 _exc_store = None
54
54
55
55
56 def get_exc_store():
56 def get_exc_store():
57 """
57 """
58 Get and create exception store if it's not existing
58 Get and create exception store if it's not existing
59 """
59 """
60 global _exc_store
60 global _exc_store
61 import rhodecode as app
61 import rhodecode as app
62
62
63 if _exc_store is not None:
63 if _exc_store is not None:
64 # quick global cache
64 # quick global cache
65 return _exc_store
65 return _exc_store
66
66
67 exc_store_dir = app.CONFIG.get('exception_tracker.store_path', '') or tempfile.gettempdir()
67 exc_store_dir = app.CONFIG.get('exception_tracker.store_path', '') or tempfile.gettempdir()
68 _exc_store_path = os.path.join(exc_store_dir, exc_store_dir_name)
68 _exc_store_path = os.path.join(exc_store_dir, exc_store_dir_name)
69
69
70 _exc_store_path = os.path.abspath(_exc_store_path)
70 _exc_store_path = os.path.abspath(_exc_store_path)
71 if not os.path.isdir(_exc_store_path):
71 if not os.path.isdir(_exc_store_path):
72 os.makedirs(_exc_store_path)
72 os.makedirs(_exc_store_path)
73 log.debug('Initializing exceptions store at %s', _exc_store_path)
73 log.debug('Initializing exceptions store at %s', _exc_store_path)
74 _exc_store = _exc_store_path
74 _exc_store = _exc_store_path
75
75
76 return _exc_store_path
76 return _exc_store_path
77
77
78
78
79 def _store_exception(exc_id, exc_type_name, exc_traceback, prefix, send_email=None):
79 def _store_exception(exc_id, exc_type_name, exc_traceback, prefix, send_email=None):
80 """
80 """
81 Low level function to store exception in the exception tracker
81 Low level function to store exception in the exception tracker
82 """
82 """
83 import rhodecode as app
83 import rhodecode as app
84
84
85 exc_store_path = get_exc_store()
85 exc_store_path = get_exc_store()
86 exc_data, org_data = exc_serialize(exc_id, exc_traceback, exc_type_name)
86 exc_data, org_data = exc_serialize(exc_id, exc_traceback, exc_type_name)
87 exc_pref_id = '{}_{}_{}'.format(exc_id, prefix, org_data['exc_timestamp'])
87 exc_pref_id = '{}_{}_{}'.format(exc_id, prefix, org_data['exc_timestamp'])
88 if not os.path.isdir(exc_store_path):
88 if not os.path.isdir(exc_store_path):
89 os.makedirs(exc_store_path)
89 os.makedirs(exc_store_path)
90 stored_exc_path = os.path.join(exc_store_path, exc_pref_id)
90 stored_exc_path = os.path.join(exc_store_path, exc_pref_id)
91 with open(stored_exc_path, 'wb') as f:
91 with open(stored_exc_path, 'wb') as f:
92 f.write(exc_data)
92 f.write(exc_data)
93 log.debug('Stored generated exception %s as: %s', exc_id, stored_exc_path)
93 log.debug('Stored generated exception %s as: %s', exc_id, stored_exc_path)
94
94
95 if send_email is None:
95 if send_email is None:
96 # NOTE(marcink): read app config unless we specify explicitly
96 # NOTE(marcink): read app config unless we specify explicitly
97 send_email = app.CONFIG.get('exception_tracker.send_email', False)
97 send_email = app.CONFIG.get('exception_tracker.send_email', False)
98 mail_server = app.CONFIG.get('smtp_server') or None
99 send_email = send_email and mail_server
98
100
99 if send_email:
101 if send_email:
100 try:
102 try:
101 send_exc_email(exc_id, exc_type_name)
103 send_exc_email(exc_id, exc_type_name)
102 except Exception:
104 except Exception:
103 log.exception('Failed to send exception email')
105 log.exception('Failed to send exception email')
104 pass
106 pass
105
107
106
108
107 def send_exc_email(exc_id, exc_type_name):
109 def send_exc_email(exc_id, exc_type_name):
108 import rhodecode as app
110 import rhodecode as app
109 from pyramid.threadlocal import get_current_request
111 from pyramid.threadlocal import get_current_request
110 from rhodecode.apps._base import TemplateArgs
112 from rhodecode.apps._base import TemplateArgs
111 from rhodecode.lib.utils2 import aslist
113 from rhodecode.lib.utils2 import aslist
112 from rhodecode.lib.celerylib import run_task, tasks
114 from rhodecode.lib.celerylib import run_task, tasks
113 from rhodecode.lib.base import attach_context_attributes
115 from rhodecode.lib.base import attach_context_attributes
114 from rhodecode.model.notification import EmailNotificationModel
116 from rhodecode.model.notification import EmailNotificationModel
115
117
116 request = get_current_request()
118 request = get_current_request()
117
119
118 recipients = aslist(app.CONFIG.get('exception_tracker.send_email_recipients', ''))
120 recipients = aslist(app.CONFIG.get('exception_tracker.send_email_recipients', ''))
119 log.debug('Sending Email exception to: `%s`', recipients or 'all super admins')
121 log.debug('Sending Email exception to: `%s`', recipients or 'all super admins')
120
122
121 # NOTE(marcink): needed for email template rendering
123 # NOTE(marcink): needed for email template rendering
122 user_id = None
124 user_id = None
123 if request:
125 if request:
124 user_id = request.user.user_id
126 user_id = request.user.user_id
125 attach_context_attributes(TemplateArgs(), request, user_id=user_id, is_api=True)
127 attach_context_attributes(TemplateArgs(), request, user_id=user_id, is_api=True)
126
128
127 email_kwargs = {
129 email_kwargs = {
128 'email_prefix': app.CONFIG.get('exception_tracker.email_prefix', '') or '[RHODECODE ERROR]',
130 'email_prefix': app.CONFIG.get('exception_tracker.email_prefix', '') or '[RHODECODE ERROR]',
129 'exc_url': request.route_url('admin_settings_exception_tracker_show', exception_id=exc_id),
131 'exc_url': request.route_url('admin_settings_exception_tracker_show', exception_id=exc_id),
130 'exc_id': exc_id,
132 'exc_id': exc_id,
131 'exc_type_name': exc_type_name,
133 'exc_type_name': exc_type_name,
132 'exc_traceback': read_exception(exc_id, prefix=None),
134 'exc_traceback': read_exception(exc_id, prefix=None),
133 }
135 }
134
136
135 (subject, headers, email_body,
137 (subject, headers, email_body,
136 email_body_plaintext) = EmailNotificationModel().render_email(
138 email_body_plaintext) = EmailNotificationModel().render_email(
137 EmailNotificationModel.TYPE_EMAIL_EXCEPTION, **email_kwargs)
139 EmailNotificationModel.TYPE_EMAIL_EXCEPTION, **email_kwargs)
138
140
139 run_task(tasks.send_email, recipients, subject,
141 run_task(tasks.send_email, recipients, subject,
140 email_body_plaintext, email_body)
142 email_body_plaintext, email_body)
141
143
142
144
143 def _prepare_exception(exc_info):
145 def _prepare_exception(exc_info):
144 exc_type, exc_value, exc_traceback = exc_info
146 exc_type, exc_value, exc_traceback = exc_info
145 exc_type_name = exc_type.__name__
147 exc_type_name = exc_type.__name__
146
148
147 tb = ''.join(traceback.format_exception(
149 tb = ''.join(traceback.format_exception(
148 exc_type, exc_value, exc_traceback, None))
150 exc_type, exc_value, exc_traceback, None))
149
151
150 return exc_type_name, tb
152 return exc_type_name, tb
151
153
152
154
153 def store_exception(exc_id, exc_info, prefix=global_prefix):
155 def store_exception(exc_id, exc_info, prefix=global_prefix):
154 """
156 """
155 Example usage::
157 Example usage::
156
158
157 exc_info = sys.exc_info()
159 exc_info = sys.exc_info()
158 store_exception(id(exc_info), exc_info)
160 store_exception(id(exc_info), exc_info)
159 """
161 """
160
162
161 try:
163 try:
162 exc_type_name, exc_traceback = _prepare_exception(exc_info)
164 exc_type_name, exc_traceback = _prepare_exception(exc_info)
163 _store_exception(exc_id=exc_id, exc_type_name=exc_type_name,
165 _store_exception(exc_id=exc_id, exc_type_name=exc_type_name,
164 exc_traceback=exc_traceback, prefix=prefix)
166 exc_traceback=exc_traceback, prefix=prefix)
165 return exc_id, exc_type_name
167 return exc_id, exc_type_name
166 except Exception:
168 except Exception:
167 log.exception('Failed to store exception `%s` information', exc_id)
169 log.exception('Failed to store exception `%s` information', exc_id)
168 # there's no way this can fail, it will crash server badly if it does.
170 # there's no way this can fail, it will crash server badly if it does.
169 pass
171 pass
170
172
171
173
172 def _find_exc_file(exc_id, prefix=global_prefix):
174 def _find_exc_file(exc_id, prefix=global_prefix):
173 exc_store_path = get_exc_store()
175 exc_store_path = get_exc_store()
174 if prefix:
176 if prefix:
175 exc_id = '{}_{}'.format(exc_id, prefix)
177 exc_id = '{}_{}'.format(exc_id, prefix)
176 else:
178 else:
177 # search without a prefix
179 # search without a prefix
178 exc_id = '{}'.format(exc_id)
180 exc_id = '{}'.format(exc_id)
179
181
180 found_exc_id = None
182 found_exc_id = None
181 matches = glob.glob(os.path.join(exc_store_path, exc_id) + '*')
183 matches = glob.glob(os.path.join(exc_store_path, exc_id) + '*')
182 if matches:
184 if matches:
183 found_exc_id = matches[0]
185 found_exc_id = matches[0]
184
186
185 return found_exc_id
187 return found_exc_id
186
188
187
189
188 def _read_exception(exc_id, prefix):
190 def _read_exception(exc_id, prefix):
189 exc_id_file_path = _find_exc_file(exc_id=exc_id, prefix=prefix)
191 exc_id_file_path = _find_exc_file(exc_id=exc_id, prefix=prefix)
190 if exc_id_file_path:
192 if exc_id_file_path:
191 with open(exc_id_file_path, 'rb') as f:
193 with open(exc_id_file_path, 'rb') as f:
192 return exc_unserialize(f.read())
194 return exc_unserialize(f.read())
193 else:
195 else:
194 log.debug('Exception File `%s` not found', exc_id_file_path)
196 log.debug('Exception File `%s` not found', exc_id_file_path)
195 return None
197 return None
196
198
197
199
198 def read_exception(exc_id, prefix=global_prefix):
200 def read_exception(exc_id, prefix=global_prefix):
199 try:
201 try:
200 return _read_exception(exc_id=exc_id, prefix=prefix)
202 return _read_exception(exc_id=exc_id, prefix=prefix)
201 except Exception:
203 except Exception:
202 log.exception('Failed to read exception `%s` information', exc_id)
204 log.exception('Failed to read exception `%s` information', exc_id)
203 # there's no way this can fail, it will crash server badly if it does.
205 # there's no way this can fail, it will crash server badly if it does.
204 return None
206 return None
205
207
206
208
207 def delete_exception(exc_id, prefix=global_prefix):
209 def delete_exception(exc_id, prefix=global_prefix):
208 try:
210 try:
209 exc_id_file_path = _find_exc_file(exc_id, prefix=prefix)
211 exc_id_file_path = _find_exc_file(exc_id, prefix=prefix)
210 if exc_id_file_path:
212 if exc_id_file_path:
211 os.remove(exc_id_file_path)
213 os.remove(exc_id_file_path)
212
214
213 except Exception:
215 except Exception:
214 log.exception('Failed to remove exception `%s` information', exc_id)
216 log.exception('Failed to remove exception `%s` information', exc_id)
215 # there's no way this can fail, it will crash server badly if it does.
217 # there's no way this can fail, it will crash server badly if it does.
216 pass
218 pass
217
219
218
220
219 def generate_id():
221 def generate_id():
220 return id(object())
222 return id(object())
General Comments 0
You need to be logged in to leave comments. Login now