##// END OF EJS Templates
exc-tracking: fixed send-email bug when called from a source not attached to user session.
marcink -
r4497:8b17d3ec stable
parent child Browse files
Show More
@@ -1,230 +1,230 b''
1 # -*- coding: utf-8 -*-
1 # -*- coding: utf-8 -*-
2
2
3 # Copyright (C) 2010-2020 RhodeCode GmbH
3 # Copyright (C) 2010-2020 RhodeCode GmbH
4 #
4 #
5 # This program is free software: you can redistribute it and/or modify
5 # This program is free software: you can redistribute it and/or modify
6 # it under the terms of the GNU Affero General Public License, version 3
6 # it under the terms of the GNU Affero General Public License, version 3
7 # (only), as published by the Free Software Foundation.
7 # (only), as published by the Free Software Foundation.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU Affero General Public License
14 # You should have received a copy of the GNU Affero General Public License
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
15 # along with this program. If not, see <http://www.gnu.org/licenses/>.
16 #
16 #
17 # This program is dual-licensed. If you wish to learn more about the
17 # This program is dual-licensed. If you wish to learn more about the
18 # RhodeCode Enterprise Edition, including its added features, Support services,
18 # RhodeCode Enterprise Edition, including its added features, Support services,
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
19 # and proprietary license terms, please see https://rhodecode.com/licenses/
20
20
21 import os
21 import os
22 import time
22 import time
23 import datetime
23 import datetime
24 import msgpack
24 import msgpack
25 import logging
25 import logging
26 import traceback
26 import traceback
27 import tempfile
27 import tempfile
28 import glob
28 import glob
29
29
30 log = logging.getLogger(__name__)
30 log = logging.getLogger(__name__)
31
31
32 # NOTE: Any changes should be synced with exc_tracking at vcsserver.lib.exc_tracking
32 # NOTE: Any changes should be synced with exc_tracking at vcsserver.lib.exc_tracking
33 global_prefix = 'rhodecode'
33 global_prefix = 'rhodecode'
34 exc_store_dir_name = 'rc_exception_store_v1'
34 exc_store_dir_name = 'rc_exception_store_v1'
35
35
36
36
37 def exc_serialize(exc_id, tb, exc_type, extra_data=None):
37 def exc_serialize(exc_id, tb, exc_type, extra_data=None):
38
38
39 data = {
39 data = {
40 'version': 'v1',
40 'version': 'v1',
41 'exc_id': exc_id,
41 'exc_id': exc_id,
42 'exc_utc_date': datetime.datetime.utcnow().isoformat(),
42 'exc_utc_date': datetime.datetime.utcnow().isoformat(),
43 'exc_timestamp': repr(time.time()),
43 'exc_timestamp': repr(time.time()),
44 'exc_message': tb,
44 'exc_message': tb,
45 'exc_type': exc_type,
45 'exc_type': exc_type,
46 }
46 }
47 if extra_data:
47 if extra_data:
48 data.update(extra_data)
48 data.update(extra_data)
49 return msgpack.packb(data), data
49 return msgpack.packb(data), data
50
50
51
51
52 def exc_unserialize(tb):
52 def exc_unserialize(tb):
53 return msgpack.unpackb(tb)
53 return msgpack.unpackb(tb)
54
54
55 _exc_store = None
55 _exc_store = None
56
56
57
57
58 def get_exc_store():
58 def get_exc_store():
59 """
59 """
60 Get and create exception store if it's not existing
60 Get and create exception store if it's not existing
61 """
61 """
62 global _exc_store
62 global _exc_store
63 import rhodecode as app
63 import rhodecode as app
64
64
65 if _exc_store is not None:
65 if _exc_store is not None:
66 # quick global cache
66 # quick global cache
67 return _exc_store
67 return _exc_store
68
68
69 exc_store_dir = app.CONFIG.get('exception_tracker.store_path', '') or tempfile.gettempdir()
69 exc_store_dir = app.CONFIG.get('exception_tracker.store_path', '') or tempfile.gettempdir()
70 _exc_store_path = os.path.join(exc_store_dir, exc_store_dir_name)
70 _exc_store_path = os.path.join(exc_store_dir, exc_store_dir_name)
71
71
72 _exc_store_path = os.path.abspath(_exc_store_path)
72 _exc_store_path = os.path.abspath(_exc_store_path)
73 if not os.path.isdir(_exc_store_path):
73 if not os.path.isdir(_exc_store_path):
74 os.makedirs(_exc_store_path)
74 os.makedirs(_exc_store_path)
75 log.debug('Initializing exceptions store at %s', _exc_store_path)
75 log.debug('Initializing exceptions store at %s', _exc_store_path)
76 _exc_store = _exc_store_path
76 _exc_store = _exc_store_path
77
77
78 return _exc_store_path
78 return _exc_store_path
79
79
80
80
81 def _store_exception(exc_id, exc_type_name, exc_traceback, prefix, send_email=None):
81 def _store_exception(exc_id, exc_type_name, exc_traceback, prefix, send_email=None):
82 """
82 """
83 Low level function to store exception in the exception tracker
83 Low level function to store exception in the exception tracker
84 """
84 """
85 from pyramid.threadlocal import get_current_request
85 from pyramid.threadlocal import get_current_request
86 import rhodecode as app
86 import rhodecode as app
87 request = get_current_request()
87 request = get_current_request()
88 extra_data = {}
88 extra_data = {}
89 # NOTE(marcink): store request information into exc_data
89 # NOTE(marcink): store request information into exc_data
90 if request:
90 if request:
91 extra_data['client_address'] = getattr(request, 'client_addr', '')
91 extra_data['client_address'] = getattr(request, 'client_addr', '')
92 extra_data['user_agent'] = getattr(request, 'user_agent', '')
92 extra_data['user_agent'] = getattr(request, 'user_agent', '')
93 extra_data['method'] = getattr(request, 'method', '')
93 extra_data['method'] = getattr(request, 'method', '')
94 extra_data['url'] = getattr(request, 'url', '')
94 extra_data['url'] = getattr(request, 'url', '')
95
95
96 exc_store_path = get_exc_store()
96 exc_store_path = get_exc_store()
97 exc_data, org_data = exc_serialize(exc_id, exc_traceback, exc_type_name, extra_data=extra_data)
97 exc_data, org_data = exc_serialize(exc_id, exc_traceback, exc_type_name, extra_data=extra_data)
98
98
99 exc_pref_id = '{}_{}_{}'.format(exc_id, prefix, org_data['exc_timestamp'])
99 exc_pref_id = '{}_{}_{}'.format(exc_id, prefix, org_data['exc_timestamp'])
100 if not os.path.isdir(exc_store_path):
100 if not os.path.isdir(exc_store_path):
101 os.makedirs(exc_store_path)
101 os.makedirs(exc_store_path)
102 stored_exc_path = os.path.join(exc_store_path, exc_pref_id)
102 stored_exc_path = os.path.join(exc_store_path, exc_pref_id)
103 with open(stored_exc_path, 'wb') as f:
103 with open(stored_exc_path, 'wb') as f:
104 f.write(exc_data)
104 f.write(exc_data)
105 log.debug('Stored generated exception %s as: %s', exc_id, stored_exc_path)
105 log.debug('Stored generated exception %s as: %s', exc_id, stored_exc_path)
106
106
107 if send_email is None:
107 if send_email is None:
108 # NOTE(marcink): read app config unless we specify explicitly
108 # NOTE(marcink): read app config unless we specify explicitly
109 send_email = app.CONFIG.get('exception_tracker.send_email', False)
109 send_email = app.CONFIG.get('exception_tracker.send_email', False)
110
110
111 mail_server = app.CONFIG.get('smtp_server') or None
111 mail_server = app.CONFIG.get('smtp_server') or None
112 send_email = send_email and mail_server
112 send_email = send_email and mail_server
113 if send_email:
113 if send_email:
114 try:
114 try:
115 send_exc_email(request, exc_id, exc_type_name)
115 send_exc_email(request, exc_id, exc_type_name)
116 except Exception:
116 except Exception:
117 log.exception('Failed to send exception email')
117 log.exception('Failed to send exception email')
118 pass
118 pass
119
119
120
120
121 def send_exc_email(request, exc_id, exc_type_name):
121 def send_exc_email(request, exc_id, exc_type_name):
122 import rhodecode as app
122 import rhodecode as app
123 from rhodecode.apps._base import TemplateArgs
123 from rhodecode.apps._base import TemplateArgs
124 from rhodecode.lib.utils2 import aslist
124 from rhodecode.lib.utils2 import aslist
125 from rhodecode.lib.celerylib import run_task, tasks
125 from rhodecode.lib.celerylib import run_task, tasks
126 from rhodecode.lib.base import attach_context_attributes
126 from rhodecode.lib.base import attach_context_attributes
127 from rhodecode.model.notification import EmailNotificationModel
127 from rhodecode.model.notification import EmailNotificationModel
128
128
129 recipients = aslist(app.CONFIG.get('exception_tracker.send_email_recipients', ''))
129 recipients = aslist(app.CONFIG.get('exception_tracker.send_email_recipients', ''))
130 log.debug('Sending Email exception to: `%s`', recipients or 'all super admins')
130 log.debug('Sending Email exception to: `%s`', recipients or 'all super admins')
131
131
132 # NOTE(marcink): needed for email template rendering
132 # NOTE(marcink): needed for email template rendering
133 user_id = None
133 user_id = None
134 if request:
134 if hasattr(request, 'user'):
135 user_id = request.user.user_id
135 user_id = request.user.user_id
136 attach_context_attributes(TemplateArgs(), request, user_id=user_id, is_api=True)
136 attach_context_attributes(TemplateArgs(), request, user_id=user_id, is_api=True)
137
137
138 email_kwargs = {
138 email_kwargs = {
139 'email_prefix': app.CONFIG.get('exception_tracker.email_prefix', '') or '[RHODECODE ERROR]',
139 'email_prefix': app.CONFIG.get('exception_tracker.email_prefix', '') or '[RHODECODE ERROR]',
140 'exc_url': request.route_url('admin_settings_exception_tracker_show', exception_id=exc_id),
140 'exc_url': request.route_url('admin_settings_exception_tracker_show', exception_id=exc_id),
141 'exc_id': exc_id,
141 'exc_id': exc_id,
142 'exc_type_name': exc_type_name,
142 'exc_type_name': exc_type_name,
143 'exc_traceback': read_exception(exc_id, prefix=None),
143 'exc_traceback': read_exception(exc_id, prefix=None),
144 }
144 }
145
145
146 (subject, email_body, email_body_plaintext) = EmailNotificationModel().render_email(
146 (subject, email_body, email_body_plaintext) = EmailNotificationModel().render_email(
147 EmailNotificationModel.TYPE_EMAIL_EXCEPTION, **email_kwargs)
147 EmailNotificationModel.TYPE_EMAIL_EXCEPTION, **email_kwargs)
148
148
149 run_task(tasks.send_email, recipients, subject,
149 run_task(tasks.send_email, recipients, subject,
150 email_body_plaintext, email_body)
150 email_body_plaintext, email_body)
151
151
152
152
153 def _prepare_exception(exc_info):
153 def _prepare_exception(exc_info):
154 exc_type, exc_value, exc_traceback = exc_info
154 exc_type, exc_value, exc_traceback = exc_info
155 exc_type_name = exc_type.__name__
155 exc_type_name = exc_type.__name__
156
156
157 tb = ''.join(traceback.format_exception(
157 tb = ''.join(traceback.format_exception(
158 exc_type, exc_value, exc_traceback, None))
158 exc_type, exc_value, exc_traceback, None))
159
159
160 return exc_type_name, tb
160 return exc_type_name, tb
161
161
162
162
163 def store_exception(exc_id, exc_info, prefix=global_prefix):
163 def store_exception(exc_id, exc_info, prefix=global_prefix):
164 """
164 """
165 Example usage::
165 Example usage::
166
166
167 exc_info = sys.exc_info()
167 exc_info = sys.exc_info()
168 store_exception(id(exc_info), exc_info)
168 store_exception(id(exc_info), exc_info)
169 """
169 """
170
170
171 try:
171 try:
172 exc_type_name, exc_traceback = _prepare_exception(exc_info)
172 exc_type_name, exc_traceback = _prepare_exception(exc_info)
173 _store_exception(exc_id=exc_id, exc_type_name=exc_type_name,
173 _store_exception(exc_id=exc_id, exc_type_name=exc_type_name,
174 exc_traceback=exc_traceback, prefix=prefix)
174 exc_traceback=exc_traceback, prefix=prefix)
175 return exc_id, exc_type_name
175 return exc_id, exc_type_name
176 except Exception:
176 except Exception:
177 log.exception('Failed to store exception `%s` information', exc_id)
177 log.exception('Failed to store exception `%s` information', exc_id)
178 # there's no way this can fail, it will crash server badly if it does.
178 # there's no way this can fail, it will crash server badly if it does.
179 pass
179 pass
180
180
181
181
182 def _find_exc_file(exc_id, prefix=global_prefix):
182 def _find_exc_file(exc_id, prefix=global_prefix):
183 exc_store_path = get_exc_store()
183 exc_store_path = get_exc_store()
184 if prefix:
184 if prefix:
185 exc_id = '{}_{}'.format(exc_id, prefix)
185 exc_id = '{}_{}'.format(exc_id, prefix)
186 else:
186 else:
187 # search without a prefix
187 # search without a prefix
188 exc_id = '{}'.format(exc_id)
188 exc_id = '{}'.format(exc_id)
189
189
190 found_exc_id = None
190 found_exc_id = None
191 matches = glob.glob(os.path.join(exc_store_path, exc_id) + '*')
191 matches = glob.glob(os.path.join(exc_store_path, exc_id) + '*')
192 if matches:
192 if matches:
193 found_exc_id = matches[0]
193 found_exc_id = matches[0]
194
194
195 return found_exc_id
195 return found_exc_id
196
196
197
197
198 def _read_exception(exc_id, prefix):
198 def _read_exception(exc_id, prefix):
199 exc_id_file_path = _find_exc_file(exc_id=exc_id, prefix=prefix)
199 exc_id_file_path = _find_exc_file(exc_id=exc_id, prefix=prefix)
200 if exc_id_file_path:
200 if exc_id_file_path:
201 with open(exc_id_file_path, 'rb') as f:
201 with open(exc_id_file_path, 'rb') as f:
202 return exc_unserialize(f.read())
202 return exc_unserialize(f.read())
203 else:
203 else:
204 log.debug('Exception File `%s` not found', exc_id_file_path)
204 log.debug('Exception File `%s` not found', exc_id_file_path)
205 return None
205 return None
206
206
207
207
208 def read_exception(exc_id, prefix=global_prefix):
208 def read_exception(exc_id, prefix=global_prefix):
209 try:
209 try:
210 return _read_exception(exc_id=exc_id, prefix=prefix)
210 return _read_exception(exc_id=exc_id, prefix=prefix)
211 except Exception:
211 except Exception:
212 log.exception('Failed to read exception `%s` information', exc_id)
212 log.exception('Failed to read exception `%s` information', exc_id)
213 # there's no way this can fail, it will crash server badly if it does.
213 # there's no way this can fail, it will crash server badly if it does.
214 return None
214 return None
215
215
216
216
217 def delete_exception(exc_id, prefix=global_prefix):
217 def delete_exception(exc_id, prefix=global_prefix):
218 try:
218 try:
219 exc_id_file_path = _find_exc_file(exc_id, prefix=prefix)
219 exc_id_file_path = _find_exc_file(exc_id, prefix=prefix)
220 if exc_id_file_path:
220 if exc_id_file_path:
221 os.remove(exc_id_file_path)
221 os.remove(exc_id_file_path)
222
222
223 except Exception:
223 except Exception:
224 log.exception('Failed to remove exception `%s` information', exc_id)
224 log.exception('Failed to remove exception `%s` information', exc_id)
225 # there's no way this can fail, it will crash server badly if it does.
225 # there's no way this can fail, it will crash server badly if it does.
226 pass
226 pass
227
227
228
228
229 def generate_id():
229 def generate_id():
230 return id(object())
230 return id(object())
General Comments 0
You need to be logged in to leave comments. Login now