##// END OF EJS Templates
exc-tracking: synced with ce
super-admin -
r1146:42e5e37a default
parent child Browse files
Show More
@@ -1,267 +1,272 b''
1 # RhodeCode VCSServer provides access to different vcs backends via network.
1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2023 RhodeCode GmbH
2 # Copyright (C) 2014-2023 RhodeCode GmbH
3 #
3 #
4 # This program is free software; you can redistribute it and/or modify
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 3 of the License, or
6 # the Free Software Foundation; either version 3 of the License, or
7 # (at your option) any later version.
7 # (at your option) any later version.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU General Public License
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software Foundation,
15 # along with this program; if not, write to the Free Software Foundation,
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
17
18
18 import io
19 import os
19 import os
20 import time
20 import time
21 import sys
21 import sys
22 import datetime
22 import datetime
23 import msgpack
23 import msgpack
24 import logging
24 import logging
25 import traceback
25 import traceback
26 import tempfile
26 import tempfile
27 import glob
27 import glob
28
28
29 log = logging.getLogger(__name__)
29 log = logging.getLogger(__name__)
30
30
31 # NOTE: Any changes should be synced with exc_tracking at rhodecode.lib.exc_tracking
31 # NOTE: Any changes should be synced with exc_tracking at rhodecode.lib.exc_tracking
32 global_prefix = 'vcsserver'
32 global_prefix = 'vcsserver'
33 exc_store_dir_name = 'rc_exception_store_v1'
33 exc_store_dir_name = 'rc_exception_store_v1'
34
34
35
35
36 def exc_serialize(exc_id, tb, exc_type, extra_data=None):
36 def exc_serialize(exc_id, tb, exc_type, extra_data=None):
37
38 data = {
37 data = {
39 'version': 'v1',
38 "version": "v1",
40 'exc_id': exc_id,
39 "exc_id": exc_id,
41 'exc_utc_date': datetime.datetime.utcnow().isoformat(),
40 "exc_utc_date": datetime.datetime.utcnow().isoformat(),
42 'exc_timestamp': repr(time.time()),
41 "exc_timestamp": repr(time.time()),
43 'exc_message': tb,
42 "exc_message": tb,
44 'exc_type': exc_type,
43 "exc_type": exc_type,
45 }
44 }
46 if extra_data:
45 if extra_data:
47 data.update(extra_data)
46 data.update(extra_data)
48 return msgpack.packb(data), data
47 return msgpack.packb(data), data
49
48
50
49
51 def exc_unserialize(tb):
50 def exc_unserialize(tb):
52 return msgpack.unpackb(tb)
51 return msgpack.unpackb(tb)
53
52
54
53
55 _exc_store = None
54 _exc_store = None
56
55
57
56
58 def get_exc_store():
57 def get_exc_store():
59 """
58 """
60 Get and create exception store if it's not existing
59 Get and create exception store if it's not existing
61 """
60 """
62 global _exc_store
61 global _exc_store
63
62
64 if _exc_store is not None:
63 if _exc_store is not None:
65 # quick global cache
64 # quick global cache
66 return _exc_store
65 return _exc_store
67
66
68 import vcsserver as app
67 import vcsserver as app
69
68
70 exc_store_dir = app.CONFIG.get('exception_tracker.store_path', '') or tempfile.gettempdir()
69 exc_store_dir = (
70 app.CONFIG.get("exception_tracker.store_path", "") or tempfile.gettempdir()
71 )
71 _exc_store_path = os.path.join(exc_store_dir, exc_store_dir_name)
72 _exc_store_path = os.path.join(exc_store_dir, exc_store_dir_name)
72
73
73 _exc_store_path = os.path.abspath(_exc_store_path)
74 _exc_store_path = os.path.abspath(_exc_store_path)
74 if not os.path.isdir(_exc_store_path):
75 if not os.path.isdir(_exc_store_path):
75 os.makedirs(_exc_store_path)
76 os.makedirs(_exc_store_path)
76 log.debug('Initializing exceptions store at %s', _exc_store_path)
77 log.debug("Initializing exceptions store at %s", _exc_store_path)
77 _exc_store = _exc_store_path
78 _exc_store = _exc_store_path
78
79
79 return _exc_store_path
80 return _exc_store_path
80
81
81
82
82 def get_detailed_tb(exc_info):
83 def get_detailed_tb(exc_info):
83 from io import StringIO
84
85 try:
84 try:
86 from pip._vendor.rich import traceback as rich_tb, scope as rich_scope, console as rich_console
85 from pip._vendor.rich import (
86 traceback as rich_tb,
87 scope as rich_scope,
88 console as rich_console,
89 )
87 except ImportError:
90 except ImportError:
88 try:
91 try:
89 from rich import traceback as rich_tb, scope as rich_scope, console as rich_console
92 from rich import (
93 traceback as rich_tb,
94 scope as rich_scope,
95 console as rich_console,
96 )
90 except ImportError:
97 except ImportError:
91 return None
98 return None
92
99
93 console = rich_console.Console(
100 console = rich_console.Console(width=160, file=io.StringIO())
94 width=160,
95 file=StringIO()
96 )
97
101
98 exc = rich_tb.Traceback.extract(*exc_info, show_locals=True)
102 exc = rich_tb.Traceback.extract(*exc_info, show_locals=True)
99
103
100 tb_rich = rich_tb.Traceback(
104 tb_rich = rich_tb.Traceback(
101 trace=exc,
105 trace=exc,
102 width=160,
106 width=160,
103 extra_lines=3,
107 extra_lines=3,
104 theme=None,
108 theme=None,
105 word_wrap=False,
109 word_wrap=False,
106 show_locals=False,
110 show_locals=False,
107 max_frames=100
111 max_frames=100,
108 )
112 )
109
113
110 formatted_locals = ""
111
112 # last_stack = exc.stacks[-1]
114 # last_stack = exc.stacks[-1]
113 # last_frame = last_stack.frames[-1]
115 # last_frame = last_stack.frames[-1]
114 # if last_frame and last_frame.locals:
116 # if last_frame and last_frame.locals:
115 # console.print(
117 # console.print(
116 # rich_scope.render_scope(
118 # rich_scope.render_scope(
117 # last_frame.locals,
119 # last_frame.locals,
118 # title=f'{last_frame.filename}:{last_frame.lineno}'))
120 # title=f'{last_frame.filename}:{last_frame.lineno}'))
119
121
120 console.print(tb_rich)
122 console.print(tb_rich)
121 formatted_locals = console.file.getvalue()
123 formatted_locals = console.file.getvalue()
122
124
123 return formatted_locals
125 return formatted_locals
124
126
125
127
126 def get_request_metadata(request=None) -> dict:
128 def get_request_metadata(request=None) -> dict:
127 request_metadata = {}
129 request_metadata = {}
128 if not request:
130 if not request:
129 from pyramid.threadlocal import get_current_request
131 from pyramid.threadlocal import get_current_request
132
130 request = get_current_request()
133 request = get_current_request()
131
134
132 # NOTE(marcink): store request information into exc_data
135 # NOTE(marcink): store request information into exc_data
133 if request:
136 if request:
134 request_metadata['client_address'] = getattr(request, 'client_addr', '')
137 request_metadata["client_address"] = getattr(request, "client_addr", "")
135 request_metadata['user_agent'] = getattr(request, 'user_agent', '')
138 request_metadata["user_agent"] = getattr(request, "user_agent", "")
136 request_metadata['method'] = getattr(request, 'method', '')
139 request_metadata["method"] = getattr(request, "method", "")
137 request_metadata['url'] = getattr(request, 'url', '')
140 request_metadata["url"] = getattr(request, "url", "")
138 return request_metadata
141 return request_metadata
139
142
140
143
141 def format_exc(exc_info):
144 def format_exc(exc_info):
142 exc_type, exc_value, exc_traceback = exc_info
145 exc_type, exc_value, exc_traceback = exc_info
143 tb = "++ TRACEBACK ++\n\n"
146 tb = "++ TRACEBACK ++\n\n"
144 tb += "".join(traceback.format_exception(exc_type, exc_value, exc_traceback, None))
147 tb += "".join(traceback.format_exception(exc_type, exc_value, exc_traceback, None))
145
148
146 detailed_tb = getattr(exc_value, "_org_exc_tb", None)
149 detailed_tb = getattr(exc_value, "_org_exc_tb", None)
147
150
148 if detailed_tb:
151 if detailed_tb:
149 remote_tb = detailed_tb
152 remote_tb = detailed_tb
150 if isinstance(detailed_tb, str):
153 if isinstance(detailed_tb, str):
151 remote_tb = [detailed_tb]
154 remote_tb = [detailed_tb]
152
155
153 tb += (
156 tb += (
154 "\n+++ BEG SOURCE EXCEPTION +++\n\n"
157 "\n+++ BEG SOURCE EXCEPTION +++\n\n"
155 "{}\n"
158 "{}\n"
156 "+++ END SOURCE EXCEPTION +++\n"
159 "+++ END SOURCE EXCEPTION +++\n"
157 "".format("\n".join(remote_tb))
160 "".format("\n".join(remote_tb))
158 )
161 )
159
162
160 # Avoid that remote_tb also appears in the frame
163 # Avoid that remote_tb also appears in the frame
161 del remote_tb
164 del remote_tb
162
165
163 locals_tb = get_detailed_tb(exc_info)
166 locals_tb = get_detailed_tb(exc_info)
164 if locals_tb:
167 if locals_tb:
165 tb += f"\n+++ DETAILS +++\n\n{locals_tb}\n" ""
168 tb += f"\n+++ DETAILS +++\n\n{locals_tb}\n" ""
166 return tb
169 return tb
167
170
168
171
169 def _store_exception(exc_id, exc_info, prefix, request_path=''):
172 def _store_exception(exc_id, exc_info, prefix, request_path=''):
170 """
173 """
171 Low level function to store exception in the exception tracker
174 Low level function to store exception in the exception tracker
172 """
175 """
173
176
174 extra_data = {}
177 extra_data = {}
175 extra_data.update(get_request_metadata())
178 extra_data.update(get_request_metadata())
176
179
177 exc_type, exc_value, exc_traceback = exc_info
180 exc_type, exc_value, exc_traceback = exc_info
178 tb = format_exc(exc_info)
181 tb = format_exc(exc_info)
179
182
180 exc_type_name = exc_type.__name__
183 exc_type_name = exc_type.__name__
181 exc_data, org_data = exc_serialize(exc_id, tb, exc_type_name, extra_data=extra_data)
184 exc_data, org_data = exc_serialize(exc_id, tb, exc_type_name, extra_data=extra_data)
182
185
183 exc_pref_id = '{}_{}_{}'.format(exc_id, prefix, org_data['exc_timestamp'])
186 exc_pref_id = f"{exc_id}_{prefix}_{org_data['exc_timestamp']}"
184 exc_store_path = get_exc_store()
187 exc_store_path = get_exc_store()
185 if not os.path.isdir(exc_store_path):
188 if not os.path.isdir(exc_store_path):
186 os.makedirs(exc_store_path)
189 os.makedirs(exc_store_path)
187 stored_exc_path = os.path.join(exc_store_path, exc_pref_id)
190 stored_exc_path = os.path.join(exc_store_path, exc_pref_id)
188 with open(stored_exc_path, 'wb') as f:
191 with open(stored_exc_path, "wb") as f:
189 f.write(exc_data)
192 f.write(exc_data)
190 log.debug('Stored generated exception %s as: %s', exc_id, stored_exc_path)
193 log.debug("Stored generated exception %s as: %s", exc_id, stored_exc_path)
191
194
192 log.error(
195 if request_path:
193 'error occurred handling this request.\n'
196 log.error(
194 'Path: `%s`, %s',
197 'error occurred handling this request.\n'
195 request_path, tb)
198 'Path: `%s`, %s',
199 request_path, tb)
196
200
197
201
198 def store_exception(exc_id, exc_info, prefix=global_prefix, request_path=''):
202 def store_exception(exc_id, exc_info, prefix=global_prefix, request_path=''):
199 """
203 """
200 Example usage::
204 Example usage::
201
205
202 exc_info = sys.exc_info()
206 exc_info = sys.exc_info()
203 store_exception(id(exc_info), exc_info)
207 store_exception(id(exc_info), exc_info)
204 """
208 """
205
209
206 try:
210 try:
207 exc_type = exc_info[0]
211 exc_type = exc_info[0]
208 exc_type_name = exc_type.__name__
212 exc_type_name = exc_type.__name__
209
213
210 _store_exception(exc_id=exc_id, exc_info=exc_info, prefix=prefix,
214 _store_exception(
211 request_path=request_path)
215 exc_id=exc_id, exc_info=exc_info, prefix=prefix, request_path=request_path,
216 )
212 return exc_id, exc_type_name
217 return exc_id, exc_type_name
213 except Exception:
218 except Exception:
214 log.exception('Failed to store exception `%s` information', exc_id)
219 log.exception("Failed to store exception `%s` information", exc_id)
215 # there's no way this can fail, it will crash server badly if it does.
220 # there's no way this can fail, it will crash server badly if it does.
216 pass
221 pass
217
222
218
223
219 def _find_exc_file(exc_id, prefix=global_prefix):
224 def _find_exc_file(exc_id, prefix=global_prefix):
220 exc_store_path = get_exc_store()
225 exc_store_path = get_exc_store()
221 if prefix:
226 if prefix:
222 exc_id = f'{exc_id}_{prefix}'
227 exc_id = f"{exc_id}_{prefix}"
223 else:
228 else:
224 # search without a prefix
229 # search without a prefix
225 exc_id = f'{exc_id}'
230 exc_id = f"{exc_id}"
226
231
227 found_exc_id = None
232 found_exc_id = None
228 matches = glob.glob(os.path.join(exc_store_path, exc_id) + '*')
233 matches = glob.glob(os.path.join(exc_store_path, exc_id) + "*")
229 if matches:
234 if matches:
230 found_exc_id = matches[0]
235 found_exc_id = matches[0]
231
236
232 return found_exc_id
237 return found_exc_id
233
238
234
239
235 def _read_exception(exc_id, prefix):
240 def _read_exception(exc_id, prefix):
236 exc_id_file_path = _find_exc_file(exc_id=exc_id, prefix=prefix)
241 exc_id_file_path = _find_exc_file(exc_id=exc_id, prefix=prefix)
237 if exc_id_file_path:
242 if exc_id_file_path:
238 with open(exc_id_file_path, 'rb') as f:
243 with open(exc_id_file_path, "rb") as f:
239 return exc_unserialize(f.read())
244 return exc_unserialize(f.read())
240 else:
245 else:
241 log.debug('Exception File `%s` not found', exc_id_file_path)
246 log.debug("Exception File `%s` not found", exc_id_file_path)
242 return None
247 return None
243
248
244
249
245 def read_exception(exc_id, prefix=global_prefix):
250 def read_exception(exc_id, prefix=global_prefix):
246 try:
251 try:
247 return _read_exception(exc_id=exc_id, prefix=prefix)
252 return _read_exception(exc_id=exc_id, prefix=prefix)
248 except Exception:
253 except Exception:
249 log.exception('Failed to read exception `%s` information', exc_id)
254 log.exception("Failed to read exception `%s` information", exc_id)
250 # there's no way this can fail, it will crash server badly if it does.
255 # there's no way this can fail, it will crash server badly if it does.
251 return None
256 return None
252
257
253
258
254 def delete_exception(exc_id, prefix=global_prefix):
259 def delete_exception(exc_id, prefix=global_prefix):
255 try:
260 try:
256 exc_id_file_path = _find_exc_file(exc_id, prefix=prefix)
261 exc_id_file_path = _find_exc_file(exc_id, prefix=prefix)
257 if exc_id_file_path:
262 if exc_id_file_path:
258 os.remove(exc_id_file_path)
263 os.remove(exc_id_file_path)
259
264
260 except Exception:
265 except Exception:
261 log.exception('Failed to remove exception `%s` information', exc_id)
266 log.exception("Failed to remove exception `%s` information", exc_id)
262 # there's no way this can fail, it will crash server badly if it does.
267 # there's no way this can fail, it will crash server badly if it does.
263 pass
268 pass
264
269
265
270
266 def generate_id():
271 def generate_id():
267 return id(object())
272 return id(object())
General Comments 0
You need to be logged in to leave comments. Login now