##// END OF EJS Templates
exc-tracking: synced with CE code
super-admin -
r1166:020860b6 default
parent child Browse files
Show More
@@ -1,272 +1,273 b''
1 # RhodeCode VCSServer provides access to different vcs backends via network.
1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2023 RhodeCode GmbH
2 # Copyright (C) 2014-2023 RhodeCode GmbH
3 #
3 #
4 # This program is free software; you can redistribute it and/or modify
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 3 of the License, or
6 # the Free Software Foundation; either version 3 of the License, or
7 # (at your option) any later version.
7 # (at your option) any later version.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU General Public License
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software Foundation,
15 # along with this program; if not, write to the Free Software Foundation,
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
17
18 import io
18 import io
19 import os
19 import os
20 import time
20 import time
21 import sys
21 import sys
22 import datetime
22 import datetime
23 import msgpack
23 import msgpack
24 import logging
24 import logging
25 import traceback
25 import traceback
26 import tempfile
26 import tempfile
27 import glob
27 import glob
28
28
29 log = logging.getLogger(__name__)
29 log = logging.getLogger(__name__)
30
30
31 # NOTE: Any changes should be synced with exc_tracking at rhodecode.lib.exc_tracking
31 # NOTE: Any changes should be synced with exc_tracking at rhodecode.lib.exc_tracking
32 global_prefix = 'vcsserver'
32 global_prefix = 'vcsserver'
33 exc_store_dir_name = 'rc_exception_store_v1'
33 exc_store_dir_name = 'rc_exception_store_v1'
34
34
35
35
36 def exc_serialize(exc_id, tb, exc_type, extra_data=None):
36 def exc_serialize(exc_id, tb, exc_type, extra_data=None):
37 data = {
37 data = {
38 "version": "v1",
38 "version": "v1",
39 "exc_id": exc_id,
39 "exc_id": exc_id,
40 "exc_utc_date": datetime.datetime.utcnow().isoformat(),
40 "exc_utc_date": datetime.datetime.utcnow().isoformat(),
41 "exc_timestamp": repr(time.time()),
41 "exc_timestamp": repr(time.time()),
42 "exc_message": tb,
42 "exc_message": tb,
43 "exc_type": exc_type,
43 "exc_type": exc_type,
44 }
44 }
45 if extra_data:
45 if extra_data:
46 data.update(extra_data)
46 data.update(extra_data)
47 return msgpack.packb(data), data
47 return msgpack.packb(data), data
48
48
49
49
50 def exc_unserialize(tb):
50 def exc_unserialize(tb):
51 return msgpack.unpackb(tb)
51 return msgpack.unpackb(tb)
52
52
53
53
54 _exc_store = None
54 _exc_store = None
55
55
56
56
57 def get_exc_store():
57 def get_exc_store():
58 """
58 """
59 Get and create exception store if it's not existing
59 Get and create exception store if it's not existing
60 """
60 """
61 global _exc_store
61 global _exc_store
62
62
63 if _exc_store is not None:
63 if _exc_store is not None:
64 # quick global cache
64 # quick global cache
65 return _exc_store
65 return _exc_store
66
66
67 import vcsserver as app
67 import vcsserver as app
68
68
69 exc_store_dir = (
69 exc_store_dir = (
70 app.CONFIG.get("exception_tracker.store_path", "") or tempfile.gettempdir()
70 app.CONFIG.get("exception_tracker.store_path", "") or tempfile.gettempdir()
71 )
71 )
72 _exc_store_path = os.path.join(exc_store_dir, exc_store_dir_name)
72 _exc_store_path = os.path.join(exc_store_dir, exc_store_dir_name)
73
73
74 _exc_store_path = os.path.abspath(_exc_store_path)
74 _exc_store_path = os.path.abspath(_exc_store_path)
75 if not os.path.isdir(_exc_store_path):
75 if not os.path.isdir(_exc_store_path):
76 os.makedirs(_exc_store_path)
76 os.makedirs(_exc_store_path)
77 log.debug("Initializing exceptions store at %s", _exc_store_path)
77 log.debug("Initializing exceptions store at %s", _exc_store_path)
78 _exc_store = _exc_store_path
78 _exc_store = _exc_store_path
79
79
80 return _exc_store_path
80 return _exc_store_path
81
81
82
82
83 def get_detailed_tb(exc_info):
83 def get_detailed_tb(exc_info):
84 try:
84 try:
85 from pip._vendor.rich import (
85 from pip._vendor.rich import (
86 traceback as rich_tb,
86 traceback as rich_tb,
87 scope as rich_scope,
87 scope as rich_scope,
88 console as rich_console,
88 console as rich_console,
89 )
89 )
90 except ImportError:
90 except ImportError:
91 try:
91 try:
92 from rich import (
92 from rich import (
93 traceback as rich_tb,
93 traceback as rich_tb,
94 scope as rich_scope,
94 scope as rich_scope,
95 console as rich_console,
95 console as rich_console,
96 )
96 )
97 except ImportError:
97 except ImportError:
98 return None
98 return None
99
99
100 console = rich_console.Console(width=160, file=io.StringIO())
100 console = rich_console.Console(width=160, file=io.StringIO())
101
101
102 exc = rich_tb.Traceback.extract(*exc_info, show_locals=True)
102 exc = rich_tb.Traceback.extract(*exc_info, show_locals=True)
103
103
104 tb_rich = rich_tb.Traceback(
104 tb_rich = rich_tb.Traceback(
105 trace=exc,
105 trace=exc,
106 width=160,
106 width=160,
107 extra_lines=3,
107 extra_lines=3,
108 theme=None,
108 theme=None,
109 word_wrap=False,
109 word_wrap=False,
110 show_locals=False,
110 show_locals=False,
111 max_frames=100,
111 max_frames=100,
112 )
112 )
113
113
114 # last_stack = exc.stacks[-1]
114 # last_stack = exc.stacks[-1]
115 # last_frame = last_stack.frames[-1]
115 # last_frame = last_stack.frames[-1]
116 # if last_frame and last_frame.locals:
116 # if last_frame and last_frame.locals:
117 # console.print(
117 # console.print(
118 # rich_scope.render_scope(
118 # rich_scope.render_scope(
119 # last_frame.locals,
119 # last_frame.locals,
120 # title=f'{last_frame.filename}:{last_frame.lineno}'))
120 # title=f'{last_frame.filename}:{last_frame.lineno}'))
121
121
122 console.print(tb_rich)
122 console.print(tb_rich)
123 formatted_locals = console.file.getvalue()
123 formatted_locals = console.file.getvalue()
124
124
125 return formatted_locals
125 return formatted_locals
126
126
127
127
128 def get_request_metadata(request=None) -> dict:
128 def get_request_metadata(request=None) -> dict:
129 request_metadata = {}
129 request_metadata = {}
130 if not request:
130 if not request:
131 from pyramid.threadlocal import get_current_request
131 from pyramid.threadlocal import get_current_request
132
132
133 request = get_current_request()
133 request = get_current_request()
134
134
135 # NOTE(marcink): store request information into exc_data
135 # NOTE(marcink): store request information into exc_data
136 if request:
136 if request:
137 request_metadata["client_address"] = getattr(request, "client_addr", "")
137 request_metadata["client_address"] = getattr(request, "client_addr", "")
138 request_metadata["user_agent"] = getattr(request, "user_agent", "")
138 request_metadata["user_agent"] = getattr(request, "user_agent", "")
139 request_metadata["method"] = getattr(request, "method", "")
139 request_metadata["method"] = getattr(request, "method", "")
140 request_metadata["url"] = getattr(request, "url", "")
140 request_metadata["url"] = getattr(request, "url", "")
141 return request_metadata
141 return request_metadata
142
142
143
143
144 def format_exc(exc_info):
144 def format_exc(exc_info, use_detailed_tb=True):
145 exc_type, exc_value, exc_traceback = exc_info
145 exc_type, exc_value, exc_traceback = exc_info
146 tb = "++ TRACEBACK ++\n\n"
146 tb = "++ TRACEBACK ++\n\n"
147 tb += "".join(traceback.format_exception(exc_type, exc_value, exc_traceback, None))
147 tb += "".join(traceback.format_exception(exc_type, exc_value, exc_traceback, None))
148
148
149 detailed_tb = getattr(exc_value, "_org_exc_tb", None)
149 detailed_tb = getattr(exc_value, "_org_exc_tb", None)
150
150
151 if detailed_tb:
151 if detailed_tb:
152 remote_tb = detailed_tb
152 remote_tb = detailed_tb
153 if isinstance(detailed_tb, str):
153 if isinstance(detailed_tb, str):
154 remote_tb = [detailed_tb]
154 remote_tb = [detailed_tb]
155
155
156 tb += (
156 tb += (
157 "\n+++ BEG SOURCE EXCEPTION +++\n\n"
157 "\n+++ BEG SOURCE EXCEPTION +++\n\n"
158 "{}\n"
158 "{}\n"
159 "+++ END SOURCE EXCEPTION +++\n"
159 "+++ END SOURCE EXCEPTION +++\n"
160 "".format("\n".join(remote_tb))
160 "".format("\n".join(remote_tb))
161 )
161 )
162
162
163 # Avoid that remote_tb also appears in the frame
163 # Avoid that remote_tb also appears in the frame
164 del remote_tb
164 del remote_tb
165
165
166 locals_tb = get_detailed_tb(exc_info)
166 if use_detailed_tb:
167 if locals_tb:
167 locals_tb = get_detailed_tb(exc_info)
168 tb += f"\n+++ DETAILS +++\n\n{locals_tb}\n" ""
168 if locals_tb:
169 tb += f"\n+++ DETAILS +++\n\n{locals_tb}\n" ""
169 return tb
170 return tb
170
171
171
172
172 def _store_exception(exc_id, exc_info, prefix, request_path=''):
173 def _store_exception(exc_id, exc_info, prefix, request_path=''):
173 """
174 """
174 Low level function to store exception in the exception tracker
175 Low level function to store exception in the exception tracker
175 """
176 """
176
177
177 extra_data = {}
178 extra_data = {}
178 extra_data.update(get_request_metadata())
179 extra_data.update(get_request_metadata())
179
180
180 exc_type, exc_value, exc_traceback = exc_info
181 exc_type, exc_value, exc_traceback = exc_info
181 tb = format_exc(exc_info)
182 tb = format_exc(exc_info)
182
183
183 exc_type_name = exc_type.__name__
184 exc_type_name = exc_type.__name__
184 exc_data, org_data = exc_serialize(exc_id, tb, exc_type_name, extra_data=extra_data)
185 exc_data, org_data = exc_serialize(exc_id, tb, exc_type_name, extra_data=extra_data)
185
186
186 exc_pref_id = f"{exc_id}_{prefix}_{org_data['exc_timestamp']}"
187 exc_pref_id = f"{exc_id}_{prefix}_{org_data['exc_timestamp']}"
187 exc_store_path = get_exc_store()
188 exc_store_path = get_exc_store()
188 if not os.path.isdir(exc_store_path):
189 if not os.path.isdir(exc_store_path):
189 os.makedirs(exc_store_path)
190 os.makedirs(exc_store_path)
190 stored_exc_path = os.path.join(exc_store_path, exc_pref_id)
191 stored_exc_path = os.path.join(exc_store_path, exc_pref_id)
191 with open(stored_exc_path, "wb") as f:
192 with open(stored_exc_path, "wb") as f:
192 f.write(exc_data)
193 f.write(exc_data)
193 log.debug("Stored generated exception %s as: %s", exc_id, stored_exc_path)
194 log.debug("Stored generated exception %s as: %s", exc_id, stored_exc_path)
194
195
195 if request_path:
196 if request_path:
196 log.error(
197 log.error(
197 'error occurred handling this request.\n'
198 'error occurred handling this request.\n'
198 'Path: `%s`, %s',
199 'Path: `%s`, %s',
199 request_path, tb)
200 request_path, tb)
200
201
201
202
202 def store_exception(exc_id, exc_info, prefix=global_prefix, request_path=''):
203 def store_exception(exc_id, exc_info, prefix=global_prefix, request_path=''):
203 """
204 """
204 Example usage::
205 Example usage::
205
206
206 exc_info = sys.exc_info()
207 exc_info = sys.exc_info()
207 store_exception(id(exc_info), exc_info)
208 store_exception(id(exc_info), exc_info)
208 """
209 """
209
210
210 try:
211 try:
211 exc_type = exc_info[0]
212 exc_type = exc_info[0]
212 exc_type_name = exc_type.__name__
213 exc_type_name = exc_type.__name__
213
214
214 _store_exception(
215 _store_exception(
215 exc_id=exc_id, exc_info=exc_info, prefix=prefix, request_path=request_path,
216 exc_id=exc_id, exc_info=exc_info, prefix=prefix, request_path=request_path,
216 )
217 )
217 return exc_id, exc_type_name
218 return exc_id, exc_type_name
218 except Exception:
219 except Exception:
219 log.exception("Failed to store exception `%s` information", exc_id)
220 log.exception("Failed to store exception `%s` information", exc_id)
220 # there's no way this can fail, it will crash server badly if it does.
221 # there's no way this can fail, it will crash server badly if it does.
221 pass
222 pass
222
223
223
224
224 def _find_exc_file(exc_id, prefix=global_prefix):
225 def _find_exc_file(exc_id, prefix=global_prefix):
225 exc_store_path = get_exc_store()
226 exc_store_path = get_exc_store()
226 if prefix:
227 if prefix:
227 exc_id = f"{exc_id}_{prefix}"
228 exc_id = f"{exc_id}_{prefix}"
228 else:
229 else:
229 # search without a prefix
230 # search without a prefix
230 exc_id = f"{exc_id}"
231 exc_id = f"{exc_id}"
231
232
232 found_exc_id = None
233 found_exc_id = None
233 matches = glob.glob(os.path.join(exc_store_path, exc_id) + "*")
234 matches = glob.glob(os.path.join(exc_store_path, exc_id) + "*")
234 if matches:
235 if matches:
235 found_exc_id = matches[0]
236 found_exc_id = matches[0]
236
237
237 return found_exc_id
238 return found_exc_id
238
239
239
240
240 def _read_exception(exc_id, prefix):
241 def _read_exception(exc_id, prefix):
241 exc_id_file_path = _find_exc_file(exc_id=exc_id, prefix=prefix)
242 exc_id_file_path = _find_exc_file(exc_id=exc_id, prefix=prefix)
242 if exc_id_file_path:
243 if exc_id_file_path:
243 with open(exc_id_file_path, "rb") as f:
244 with open(exc_id_file_path, "rb") as f:
244 return exc_unserialize(f.read())
245 return exc_unserialize(f.read())
245 else:
246 else:
246 log.debug("Exception File `%s` not found", exc_id_file_path)
247 log.debug("Exception File `%s` not found", exc_id_file_path)
247 return None
248 return None
248
249
249
250
250 def read_exception(exc_id, prefix=global_prefix):
251 def read_exception(exc_id, prefix=global_prefix):
251 try:
252 try:
252 return _read_exception(exc_id=exc_id, prefix=prefix)
253 return _read_exception(exc_id=exc_id, prefix=prefix)
253 except Exception:
254 except Exception:
254 log.exception("Failed to read exception `%s` information", exc_id)
255 log.exception("Failed to read exception `%s` information", exc_id)
255 # there's no way this can fail, it will crash server badly if it does.
256 # there's no way this can fail, it will crash server badly if it does.
256 return None
257 return None
257
258
258
259
259 def delete_exception(exc_id, prefix=global_prefix):
260 def delete_exception(exc_id, prefix=global_prefix):
260 try:
261 try:
261 exc_id_file_path = _find_exc_file(exc_id, prefix=prefix)
262 exc_id_file_path = _find_exc_file(exc_id, prefix=prefix)
262 if exc_id_file_path:
263 if exc_id_file_path:
263 os.remove(exc_id_file_path)
264 os.remove(exc_id_file_path)
264
265
265 except Exception:
266 except Exception:
266 log.exception("Failed to remove exception `%s` information", exc_id)
267 log.exception("Failed to remove exception `%s` information", exc_id)
267 # there's no way this can fail, it will crash server badly if it does.
268 # there's no way this can fail, it will crash server badly if it does.
268 pass
269 pass
269
270
270
271
271 def generate_id():
272 def generate_id():
272 return id(object())
273 return id(object())
General Comments 0
You need to be logged in to leave comments. Login now