##// END OF EJS Templates
vcsserver: log exceptions into the logs
milka -
r939:b166ecdf stable
parent child Browse files
Show More
@@ -1,704 +1,705 b''
1 # RhodeCode VCSServer provides access to different vcs backends via network.
1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 # Copyright (C) 2014-2020 RhodeCode GmbH
2 # Copyright (C) 2014-2020 RhodeCode GmbH
3 #
3 #
4 # This program is free software; you can redistribute it and/or modify
4 # This program is free software; you can redistribute it and/or modify
5 # it under the terms of the GNU General Public License as published by
5 # it under the terms of the GNU General Public License as published by
6 # the Free Software Foundation; either version 3 of the License, or
6 # the Free Software Foundation; either version 3 of the License, or
7 # (at your option) any later version.
7 # (at your option) any later version.
8 #
8 #
9 # This program is distributed in the hope that it will be useful,
9 # This program is distributed in the hope that it will be useful,
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 # GNU General Public License for more details.
12 # GNU General Public License for more details.
13 #
13 #
14 # You should have received a copy of the GNU General Public License
14 # You should have received a copy of the GNU General Public License
15 # along with this program; if not, write to the Free Software Foundation,
15 # along with this program; if not, write to the Free Software Foundation,
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17
17
18 import os
18 import os
19 import sys
19 import sys
20 import base64
20 import base64
21 import locale
21 import locale
22 import logging
22 import logging
23 import uuid
23 import uuid
24 import wsgiref.util
24 import wsgiref.util
25 import traceback
25 import traceback
26 import tempfile
26 import tempfile
27 import psutil
27 import psutil
28 from itertools import chain
28 from itertools import chain
29 from cStringIO import StringIO
29 from cStringIO import StringIO
30
30
31 import simplejson as json
31 import simplejson as json
32 import msgpack
32 import msgpack
33 from pyramid.config import Configurator
33 from pyramid.config import Configurator
34 from pyramid.settings import asbool, aslist
34 from pyramid.settings import asbool, aslist
35 from pyramid.wsgi import wsgiapp
35 from pyramid.wsgi import wsgiapp
36 from pyramid.compat import configparser
36 from pyramid.compat import configparser
37 from pyramid.response import Response
37 from pyramid.response import Response
38
38
39 from vcsserver.utils import safe_int
39 from vcsserver.utils import safe_int
40
40
41 log = logging.getLogger(__name__)
41 log = logging.getLogger(__name__)
42
42
43 # due to Mercurial/glibc2.27 problems we need to detect if locale settings are
43 # due to Mercurial/glibc2.27 problems we need to detect if locale settings are
44 # causing problems and "fix" it in case they do and fallback to LC_ALL = C
44 # causing problems and "fix" it in case they do and fallback to LC_ALL = C
45
45
46 try:
46 try:
47 locale.setlocale(locale.LC_ALL, '')
47 locale.setlocale(locale.LC_ALL, '')
48 except locale.Error as e:
48 except locale.Error as e:
49 log.error(
49 log.error(
50 'LOCALE ERROR: failed to set LC_ALL, fallback to LC_ALL=C, org error: %s', e)
50 'LOCALE ERROR: failed to set LC_ALL, fallback to LC_ALL=C, org error: %s', e)
51 os.environ['LC_ALL'] = 'C'
51 os.environ['LC_ALL'] = 'C'
52
52
53 import vcsserver
53 import vcsserver
54 from vcsserver import remote_wsgi, scm_app, settings, hgpatches
54 from vcsserver import remote_wsgi, scm_app, settings, hgpatches
55 from vcsserver.git_lfs.app import GIT_LFS_CONTENT_TYPE, GIT_LFS_PROTO_PAT
55 from vcsserver.git_lfs.app import GIT_LFS_CONTENT_TYPE, GIT_LFS_PROTO_PAT
56 from vcsserver.echo_stub import remote_wsgi as remote_wsgi_stub
56 from vcsserver.echo_stub import remote_wsgi as remote_wsgi_stub
57 from vcsserver.echo_stub.echo_app import EchoApp
57 from vcsserver.echo_stub.echo_app import EchoApp
58 from vcsserver.exceptions import HTTPRepoLocked, HTTPRepoBranchProtected
58 from vcsserver.exceptions import HTTPRepoLocked, HTTPRepoBranchProtected
59 from vcsserver.lib.exc_tracking import store_exception
59 from vcsserver.lib.exc_tracking import store_exception
60 from vcsserver.server import VcsServer
60 from vcsserver.server import VcsServer
61
61
62 try:
62 try:
63 from vcsserver.git import GitFactory, GitRemote
63 from vcsserver.git import GitFactory, GitRemote
64 except ImportError:
64 except ImportError:
65 GitFactory = None
65 GitFactory = None
66 GitRemote = None
66 GitRemote = None
67
67
68 try:
68 try:
69 from vcsserver.hg import MercurialFactory, HgRemote
69 from vcsserver.hg import MercurialFactory, HgRemote
70 except ImportError:
70 except ImportError:
71 MercurialFactory = None
71 MercurialFactory = None
72 HgRemote = None
72 HgRemote = None
73
73
74 try:
74 try:
75 from vcsserver.svn import SubversionFactory, SvnRemote
75 from vcsserver.svn import SubversionFactory, SvnRemote
76 except ImportError:
76 except ImportError:
77 SubversionFactory = None
77 SubversionFactory = None
78 SvnRemote = None
78 SvnRemote = None
79
79
80
80
81 def _is_request_chunked(environ):
81 def _is_request_chunked(environ):
82 stream = environ.get('HTTP_TRANSFER_ENCODING', '') == 'chunked'
82 stream = environ.get('HTTP_TRANSFER_ENCODING', '') == 'chunked'
83 return stream
83 return stream
84
84
85
85
86 def _int_setting(settings, name, default):
86 def _int_setting(settings, name, default):
87 settings[name] = int(settings.get(name, default))
87 settings[name] = int(settings.get(name, default))
88 return settings[name]
88 return settings[name]
89
89
90
90
91 def _bool_setting(settings, name, default):
91 def _bool_setting(settings, name, default):
92 input_val = settings.get(name, default)
92 input_val = settings.get(name, default)
93 if isinstance(input_val, unicode):
93 if isinstance(input_val, unicode):
94 input_val = input_val.encode('utf8')
94 input_val = input_val.encode('utf8')
95 settings[name] = asbool(input_val)
95 settings[name] = asbool(input_val)
96 return settings[name]
96 return settings[name]
97
97
98
98
99 def _list_setting(settings, name, default):
99 def _list_setting(settings, name, default):
100 raw_value = settings.get(name, default)
100 raw_value = settings.get(name, default)
101
101
102 # Otherwise we assume it uses pyramids space/newline separation.
102 # Otherwise we assume it uses pyramids space/newline separation.
103 settings[name] = aslist(raw_value)
103 settings[name] = aslist(raw_value)
104 return settings[name]
104 return settings[name]
105
105
106
106
107 def _string_setting(settings, name, default, lower=True, default_when_empty=False):
107 def _string_setting(settings, name, default, lower=True, default_when_empty=False):
108 value = settings.get(name, default)
108 value = settings.get(name, default)
109
109
110 if default_when_empty and not value:
110 if default_when_empty and not value:
111 # use default value when value is empty
111 # use default value when value is empty
112 value = default
112 value = default
113
113
114 if lower:
114 if lower:
115 value = value.lower()
115 value = value.lower()
116 settings[name] = value
116 settings[name] = value
117 return settings[name]
117 return settings[name]
118
118
119
119
120 def log_max_fd():
120 def log_max_fd():
121 try:
121 try:
122 maxfd = psutil.Process().rlimit(psutil.RLIMIT_NOFILE)[1]
122 maxfd = psutil.Process().rlimit(psutil.RLIMIT_NOFILE)[1]
123 log.info('Max file descriptors value: %s', maxfd)
123 log.info('Max file descriptors value: %s', maxfd)
124 except Exception:
124 except Exception:
125 pass
125 pass
126
126
127
127
128 class VCS(object):
128 class VCS(object):
129 def __init__(self, locale_conf=None, cache_config=None):
129 def __init__(self, locale_conf=None, cache_config=None):
130 self.locale = locale_conf
130 self.locale = locale_conf
131 self.cache_config = cache_config
131 self.cache_config = cache_config
132 self._configure_locale()
132 self._configure_locale()
133
133
134 log_max_fd()
134 log_max_fd()
135
135
136 if GitFactory and GitRemote:
136 if GitFactory and GitRemote:
137 git_factory = GitFactory()
137 git_factory = GitFactory()
138 self._git_remote = GitRemote(git_factory)
138 self._git_remote = GitRemote(git_factory)
139 else:
139 else:
140 log.info("Git client import failed")
140 log.info("Git client import failed")
141
141
142 if MercurialFactory and HgRemote:
142 if MercurialFactory and HgRemote:
143 hg_factory = MercurialFactory()
143 hg_factory = MercurialFactory()
144 self._hg_remote = HgRemote(hg_factory)
144 self._hg_remote = HgRemote(hg_factory)
145 else:
145 else:
146 log.info("Mercurial client import failed")
146 log.info("Mercurial client import failed")
147
147
148 if SubversionFactory and SvnRemote:
148 if SubversionFactory and SvnRemote:
149 svn_factory = SubversionFactory()
149 svn_factory = SubversionFactory()
150
150
151 # hg factory is used for svn url validation
151 # hg factory is used for svn url validation
152 hg_factory = MercurialFactory()
152 hg_factory = MercurialFactory()
153 self._svn_remote = SvnRemote(svn_factory, hg_factory=hg_factory)
153 self._svn_remote = SvnRemote(svn_factory, hg_factory=hg_factory)
154 else:
154 else:
155 log.info("Subversion client import failed")
155 log.info("Subversion client import failed")
156
156
157 self._vcsserver = VcsServer()
157 self._vcsserver = VcsServer()
158
158
159 def _configure_locale(self):
159 def _configure_locale(self):
160 if self.locale:
160 if self.locale:
161 log.info('Settings locale: `LC_ALL` to %s', self.locale)
161 log.info('Settings locale: `LC_ALL` to %s', self.locale)
162 else:
162 else:
163 log.info(
163 log.info(
164 'Configuring locale subsystem based on environment variables')
164 'Configuring locale subsystem based on environment variables')
165 try:
165 try:
166 # If self.locale is the empty string, then the locale
166 # If self.locale is the empty string, then the locale
167 # module will use the environment variables. See the
167 # module will use the environment variables. See the
168 # documentation of the package `locale`.
168 # documentation of the package `locale`.
169 locale.setlocale(locale.LC_ALL, self.locale)
169 locale.setlocale(locale.LC_ALL, self.locale)
170
170
171 language_code, encoding = locale.getlocale()
171 language_code, encoding = locale.getlocale()
172 log.info(
172 log.info(
173 'Locale set to language code "%s" with encoding "%s".',
173 'Locale set to language code "%s" with encoding "%s".',
174 language_code, encoding)
174 language_code, encoding)
175 except locale.Error:
175 except locale.Error:
176 log.exception(
176 log.exception(
177 'Cannot set locale, not configuring the locale system')
177 'Cannot set locale, not configuring the locale system')
178
178
179
179
180 class WsgiProxy(object):
180 class WsgiProxy(object):
181 def __init__(self, wsgi):
181 def __init__(self, wsgi):
182 self.wsgi = wsgi
182 self.wsgi = wsgi
183
183
184 def __call__(self, environ, start_response):
184 def __call__(self, environ, start_response):
185 input_data = environ['wsgi.input'].read()
185 input_data = environ['wsgi.input'].read()
186 input_data = msgpack.unpackb(input_data)
186 input_data = msgpack.unpackb(input_data)
187
187
188 error = None
188 error = None
189 try:
189 try:
190 data, status, headers = self.wsgi.handle(
190 data, status, headers = self.wsgi.handle(
191 input_data['environment'], input_data['input_data'],
191 input_data['environment'], input_data['input_data'],
192 *input_data['args'], **input_data['kwargs'])
192 *input_data['args'], **input_data['kwargs'])
193 except Exception as e:
193 except Exception as e:
194 data, status, headers = [], None, None
194 data, status, headers = [], None, None
195 error = {
195 error = {
196 'message': str(e),
196 'message': str(e),
197 '_vcs_kind': getattr(e, '_vcs_kind', None)
197 '_vcs_kind': getattr(e, '_vcs_kind', None)
198 }
198 }
199
199
200 start_response(200, {})
200 start_response(200, {})
201 return self._iterator(error, status, headers, data)
201 return self._iterator(error, status, headers, data)
202
202
203 def _iterator(self, error, status, headers, data):
203 def _iterator(self, error, status, headers, data):
204 initial_data = [
204 initial_data = [
205 error,
205 error,
206 status,
206 status,
207 headers,
207 headers,
208 ]
208 ]
209
209
210 for d in chain(initial_data, data):
210 for d in chain(initial_data, data):
211 yield msgpack.packb(d)
211 yield msgpack.packb(d)
212
212
213
213
214 def not_found(request):
214 def not_found(request):
215 return {'status': '404 NOT FOUND'}
215 return {'status': '404 NOT FOUND'}
216
216
217
217
218 class VCSViewPredicate(object):
218 class VCSViewPredicate(object):
219 def __init__(self, val, config):
219 def __init__(self, val, config):
220 self.remotes = val
220 self.remotes = val
221
221
222 def text(self):
222 def text(self):
223 return 'vcs view method = %s' % (self.remotes.keys(),)
223 return 'vcs view method = %s' % (self.remotes.keys(),)
224
224
225 phash = text
225 phash = text
226
226
227 def __call__(self, context, request):
227 def __call__(self, context, request):
228 """
228 """
229 View predicate that returns true if given backend is supported by
229 View predicate that returns true if given backend is supported by
230 defined remotes.
230 defined remotes.
231 """
231 """
232 backend = request.matchdict.get('backend')
232 backend = request.matchdict.get('backend')
233 return backend in self.remotes
233 return backend in self.remotes
234
234
235
235
236 class HTTPApplication(object):
236 class HTTPApplication(object):
237 ALLOWED_EXCEPTIONS = ('KeyError', 'URLError')
237 ALLOWED_EXCEPTIONS = ('KeyError', 'URLError')
238
238
239 remote_wsgi = remote_wsgi
239 remote_wsgi = remote_wsgi
240 _use_echo_app = False
240 _use_echo_app = False
241
241
242 def __init__(self, settings=None, global_config=None):
242 def __init__(self, settings=None, global_config=None):
243 self._sanitize_settings_and_apply_defaults(settings)
243 self._sanitize_settings_and_apply_defaults(settings)
244
244
245 self.config = Configurator(settings=settings)
245 self.config = Configurator(settings=settings)
246 self.global_config = global_config
246 self.global_config = global_config
247 self.config.include('vcsserver.lib.rc_cache')
247 self.config.include('vcsserver.lib.rc_cache')
248
248
249 settings_locale = settings.get('locale', '') or 'en_US.UTF-8'
249 settings_locale = settings.get('locale', '') or 'en_US.UTF-8'
250 vcs = VCS(locale_conf=settings_locale, cache_config=settings)
250 vcs = VCS(locale_conf=settings_locale, cache_config=settings)
251 self._remotes = {
251 self._remotes = {
252 'hg': vcs._hg_remote,
252 'hg': vcs._hg_remote,
253 'git': vcs._git_remote,
253 'git': vcs._git_remote,
254 'svn': vcs._svn_remote,
254 'svn': vcs._svn_remote,
255 'server': vcs._vcsserver,
255 'server': vcs._vcsserver,
256 }
256 }
257 if settings.get('dev.use_echo_app', 'false').lower() == 'true':
257 if settings.get('dev.use_echo_app', 'false').lower() == 'true':
258 self._use_echo_app = True
258 self._use_echo_app = True
259 log.warning("Using EchoApp for VCS operations.")
259 log.warning("Using EchoApp for VCS operations.")
260 self.remote_wsgi = remote_wsgi_stub
260 self.remote_wsgi = remote_wsgi_stub
261
261
262 self._configure_settings(global_config, settings)
262 self._configure_settings(global_config, settings)
263
263
264 self._configure()
264 self._configure()
265
265
266 def _configure_settings(self, global_config, app_settings):
266 def _configure_settings(self, global_config, app_settings):
267 """
267 """
268 Configure the settings module.
268 Configure the settings module.
269 """
269 """
270 settings_merged = global_config.copy()
270 settings_merged = global_config.copy()
271 settings_merged.update(app_settings)
271 settings_merged.update(app_settings)
272
272
273 git_path = app_settings.get('git_path', None)
273 git_path = app_settings.get('git_path', None)
274 if git_path:
274 if git_path:
275 settings.GIT_EXECUTABLE = git_path
275 settings.GIT_EXECUTABLE = git_path
276 binary_dir = app_settings.get('core.binary_dir', None)
276 binary_dir = app_settings.get('core.binary_dir', None)
277 if binary_dir:
277 if binary_dir:
278 settings.BINARY_DIR = binary_dir
278 settings.BINARY_DIR = binary_dir
279
279
280 # Store the settings to make them available to other modules.
280 # Store the settings to make them available to other modules.
281 vcsserver.PYRAMID_SETTINGS = settings_merged
281 vcsserver.PYRAMID_SETTINGS = settings_merged
282 vcsserver.CONFIG = settings_merged
282 vcsserver.CONFIG = settings_merged
283
283
284 def _sanitize_settings_and_apply_defaults(self, settings):
284 def _sanitize_settings_and_apply_defaults(self, settings):
285 temp_store = tempfile.gettempdir()
285 temp_store = tempfile.gettempdir()
286 default_cache_dir = os.path.join(temp_store, 'rc_cache')
286 default_cache_dir = os.path.join(temp_store, 'rc_cache')
287
287
288 # save default, cache dir, and use it for all backends later.
288 # save default, cache dir, and use it for all backends later.
289 default_cache_dir = _string_setting(
289 default_cache_dir = _string_setting(
290 settings,
290 settings,
291 'cache_dir',
291 'cache_dir',
292 default_cache_dir, lower=False, default_when_empty=True)
292 default_cache_dir, lower=False, default_when_empty=True)
293
293
294 # ensure we have our dir created
294 # ensure we have our dir created
295 if not os.path.isdir(default_cache_dir):
295 if not os.path.isdir(default_cache_dir):
296 os.makedirs(default_cache_dir, mode=0o755)
296 os.makedirs(default_cache_dir, mode=0o755)
297
297
298 # exception store cache
298 # exception store cache
299 _string_setting(
299 _string_setting(
300 settings,
300 settings,
301 'exception_tracker.store_path',
301 'exception_tracker.store_path',
302 temp_store, lower=False, default_when_empty=True)
302 temp_store, lower=False, default_when_empty=True)
303
303
304 # repo_object cache
304 # repo_object cache
305 _string_setting(
305 _string_setting(
306 settings,
306 settings,
307 'rc_cache.repo_object.backend',
307 'rc_cache.repo_object.backend',
308 'dogpile.cache.rc.file_namespace', lower=False)
308 'dogpile.cache.rc.file_namespace', lower=False)
309 _int_setting(
309 _int_setting(
310 settings,
310 settings,
311 'rc_cache.repo_object.expiration_time',
311 'rc_cache.repo_object.expiration_time',
312 30 * 24 * 60 * 60)
312 30 * 24 * 60 * 60)
313 _string_setting(
313 _string_setting(
314 settings,
314 settings,
315 'rc_cache.repo_object.arguments.filename',
315 'rc_cache.repo_object.arguments.filename',
316 os.path.join(default_cache_dir, 'vcsserver_cache_1'), lower=False)
316 os.path.join(default_cache_dir, 'vcsserver_cache_1'), lower=False)
317
317
318 def _configure(self):
318 def _configure(self):
319 self.config.add_renderer(name='msgpack', factory=self._msgpack_renderer_factory)
319 self.config.add_renderer(name='msgpack', factory=self._msgpack_renderer_factory)
320
320
321 self.config.add_route('service', '/_service')
321 self.config.add_route('service', '/_service')
322 self.config.add_route('status', '/status')
322 self.config.add_route('status', '/status')
323 self.config.add_route('hg_proxy', '/proxy/hg')
323 self.config.add_route('hg_proxy', '/proxy/hg')
324 self.config.add_route('git_proxy', '/proxy/git')
324 self.config.add_route('git_proxy', '/proxy/git')
325
325
326 # rpc methods
326 # rpc methods
327 self.config.add_route('vcs', '/{backend}')
327 self.config.add_route('vcs', '/{backend}')
328
328
329 # streaming rpc remote methods
329 # streaming rpc remote methods
330 self.config.add_route('vcs_stream', '/{backend}/stream')
330 self.config.add_route('vcs_stream', '/{backend}/stream')
331
331
332 # vcs operations clone/push as streaming
332 # vcs operations clone/push as streaming
333 self.config.add_route('stream_git', '/stream/git/*repo_name')
333 self.config.add_route('stream_git', '/stream/git/*repo_name')
334 self.config.add_route('stream_hg', '/stream/hg/*repo_name')
334 self.config.add_route('stream_hg', '/stream/hg/*repo_name')
335
335
336 self.config.add_view(self.status_view, route_name='status', renderer='json')
336 self.config.add_view(self.status_view, route_name='status', renderer='json')
337 self.config.add_view(self.service_view, route_name='service', renderer='msgpack')
337 self.config.add_view(self.service_view, route_name='service', renderer='msgpack')
338
338
339 self.config.add_view(self.hg_proxy(), route_name='hg_proxy')
339 self.config.add_view(self.hg_proxy(), route_name='hg_proxy')
340 self.config.add_view(self.git_proxy(), route_name='git_proxy')
340 self.config.add_view(self.git_proxy(), route_name='git_proxy')
341 self.config.add_view(self.vcs_view, route_name='vcs', renderer='msgpack',
341 self.config.add_view(self.vcs_view, route_name='vcs', renderer='msgpack',
342 vcs_view=self._remotes)
342 vcs_view=self._remotes)
343 self.config.add_view(self.vcs_stream_view, route_name='vcs_stream',
343 self.config.add_view(self.vcs_stream_view, route_name='vcs_stream',
344 vcs_view=self._remotes)
344 vcs_view=self._remotes)
345
345
346 self.config.add_view(self.hg_stream(), route_name='stream_hg')
346 self.config.add_view(self.hg_stream(), route_name='stream_hg')
347 self.config.add_view(self.git_stream(), route_name='stream_git')
347 self.config.add_view(self.git_stream(), route_name='stream_git')
348
348
349 self.config.add_view_predicate('vcs_view', VCSViewPredicate)
349 self.config.add_view_predicate('vcs_view', VCSViewPredicate)
350
350
351 self.config.add_notfound_view(not_found, renderer='json')
351 self.config.add_notfound_view(not_found, renderer='json')
352
352
353 self.config.add_view(self.handle_vcs_exception, context=Exception)
353 self.config.add_view(self.handle_vcs_exception, context=Exception)
354
354
355 self.config.add_tween(
355 self.config.add_tween(
356 'vcsserver.tweens.request_wrapper.RequestWrapperTween',
356 'vcsserver.tweens.request_wrapper.RequestWrapperTween',
357 )
357 )
358 self.config.add_request_method(
358 self.config.add_request_method(
359 'vcsserver.lib.request_counter.get_request_counter',
359 'vcsserver.lib.request_counter.get_request_counter',
360 'request_count')
360 'request_count')
361
361
362 self.config.add_request_method(
362 self.config.add_request_method(
363 'vcsserver.lib._vendor.statsd.get_statsd_client',
363 'vcsserver.lib._vendor.statsd.get_statsd_client',
364 'statsd', reify=True)
364 'statsd', reify=True)
365
365
366 def wsgi_app(self):
366 def wsgi_app(self):
367 return self.config.make_wsgi_app()
367 return self.config.make_wsgi_app()
368
368
369 def _vcs_view_params(self, request):
369 def _vcs_view_params(self, request):
370 remote = self._remotes[request.matchdict['backend']]
370 remote = self._remotes[request.matchdict['backend']]
371 payload = msgpack.unpackb(request.body, use_list=True)
371 payload = msgpack.unpackb(request.body, use_list=True)
372 method = payload.get('method')
372 method = payload.get('method')
373 params = payload['params']
373 params = payload['params']
374 wire = params.get('wire')
374 wire = params.get('wire')
375 args = params.get('args')
375 args = params.get('args')
376 kwargs = params.get('kwargs')
376 kwargs = params.get('kwargs')
377 context_uid = None
377 context_uid = None
378
378
379 if wire:
379 if wire:
380 try:
380 try:
381 wire['context'] = context_uid = uuid.UUID(wire['context'])
381 wire['context'] = context_uid = uuid.UUID(wire['context'])
382 except KeyError:
382 except KeyError:
383 pass
383 pass
384 args.insert(0, wire)
384 args.insert(0, wire)
385 repo_state_uid = wire.get('repo_state_uid') if wire else None
385 repo_state_uid = wire.get('repo_state_uid') if wire else None
386
386
387 # NOTE(marcink): trading complexity for slight performance
387 # NOTE(marcink): trading complexity for slight performance
388 if log.isEnabledFor(logging.DEBUG):
388 if log.isEnabledFor(logging.DEBUG):
389 no_args_methods = [
389 no_args_methods = [
390
390
391 ]
391 ]
392 if method in no_args_methods:
392 if method in no_args_methods:
393 call_args = ''
393 call_args = ''
394 else:
394 else:
395 call_args = args[1:]
395 call_args = args[1:]
396
396
397 log.debug('method requested:%s with args:%s kwargs:%s context_uid: %s, repo_state_uid:%s',
397 log.debug('method requested:%s with args:%s kwargs:%s context_uid: %s, repo_state_uid:%s',
398 method, call_args, kwargs, context_uid, repo_state_uid)
398 method, call_args, kwargs, context_uid, repo_state_uid)
399
399
400 return payload, remote, method, args, kwargs
400 return payload, remote, method, args, kwargs
401
401
402 def vcs_view(self, request):
402 def vcs_view(self, request):
403
403
404 payload, remote, method, args, kwargs = self._vcs_view_params(request)
404 payload, remote, method, args, kwargs = self._vcs_view_params(request)
405 payload_id = payload.get('id')
405 payload_id = payload.get('id')
406
406
407 try:
407 try:
408 resp = getattr(remote, method)(*args, **kwargs)
408 resp = getattr(remote, method)(*args, **kwargs)
409 except Exception as e:
409 except Exception as e:
410 exc_info = list(sys.exc_info())
410 exc_info = list(sys.exc_info())
411 exc_type, exc_value, exc_traceback = exc_info
411 exc_type, exc_value, exc_traceback = exc_info
412
412
413 org_exc = getattr(e, '_org_exc', None)
413 org_exc = getattr(e, '_org_exc', None)
414 org_exc_name = None
414 org_exc_name = None
415 org_exc_tb = ''
415 org_exc_tb = ''
416 if org_exc:
416 if org_exc:
417 org_exc_name = org_exc.__class__.__name__
417 org_exc_name = org_exc.__class__.__name__
418 org_exc_tb = getattr(e, '_org_exc_tb', '')
418 org_exc_tb = getattr(e, '_org_exc_tb', '')
419 # replace our "faked" exception with our org
419 # replace our "faked" exception with our org
420 exc_info[0] = org_exc.__class__
420 exc_info[0] = org_exc.__class__
421 exc_info[1] = org_exc
421 exc_info[1] = org_exc
422
422
423 should_store_exc = True
423 should_store_exc = True
424 if org_exc:
424 if org_exc:
425 def get_exc_fqn(_exc_obj):
425 def get_exc_fqn(_exc_obj):
426 module_name = getattr(org_exc.__class__, '__module__', 'UNKNOWN')
426 module_name = getattr(org_exc.__class__, '__module__', 'UNKNOWN')
427 return module_name + '.' + org_exc_name
427 return module_name + '.' + org_exc_name
428
428
429 exc_fqn = get_exc_fqn(org_exc)
429 exc_fqn = get_exc_fqn(org_exc)
430
430
431 if exc_fqn in ['mercurial.error.RepoLookupError',
431 if exc_fqn in ['mercurial.error.RepoLookupError',
432 'vcsserver.exceptions.RefNotFoundException']:
432 'vcsserver.exceptions.RefNotFoundException']:
433 should_store_exc = False
433 should_store_exc = False
434
434
435 if should_store_exc:
435 if should_store_exc:
436 store_exception(id(exc_info), exc_info)
436 store_exception(id(exc_info), exc_info, request_path=request.path)
437
437
438 tb_info = ''.join(
438 tb_info = ''.join(
439 traceback.format_exception(exc_type, exc_value, exc_traceback))
439 traceback.format_exception(exc_type, exc_value, exc_traceback))
440
440
441 type_ = e.__class__.__name__
441 type_ = e.__class__.__name__
442 if type_ not in self.ALLOWED_EXCEPTIONS:
442 if type_ not in self.ALLOWED_EXCEPTIONS:
443 type_ = None
443 type_ = None
444
444
445 resp = {
445 resp = {
446 'id': payload_id,
446 'id': payload_id,
447 'error': {
447 'error': {
448 'message': e.message,
448 'message': e.message,
449 'traceback': tb_info,
449 'traceback': tb_info,
450 'org_exc': org_exc_name,
450 'org_exc': org_exc_name,
451 'org_exc_tb': org_exc_tb,
451 'org_exc_tb': org_exc_tb,
452 'type': type_
452 'type': type_
453 }
453 }
454 }
454 }
455
455 try:
456 try:
456 resp['error']['_vcs_kind'] = getattr(e, '_vcs_kind', None)
457 resp['error']['_vcs_kind'] = getattr(e, '_vcs_kind', None)
457 except AttributeError:
458 except AttributeError:
458 pass
459 pass
459 else:
460 else:
460 resp = {
461 resp = {
461 'id': payload_id,
462 'id': payload_id,
462 'result': resp
463 'result': resp
463 }
464 }
464
465
465 return resp
466 return resp
466
467
467 def vcs_stream_view(self, request):
468 def vcs_stream_view(self, request):
468 payload, remote, method, args, kwargs = self._vcs_view_params(request)
469 payload, remote, method, args, kwargs = self._vcs_view_params(request)
469 # this method has a stream: marker we remove it here
470 # this method has a stream: marker we remove it here
470 method = method.split('stream:')[-1]
471 method = method.split('stream:')[-1]
471 chunk_size = safe_int(payload.get('chunk_size')) or 4096
472 chunk_size = safe_int(payload.get('chunk_size')) or 4096
472
473
473 try:
474 try:
474 resp = getattr(remote, method)(*args, **kwargs)
475 resp = getattr(remote, method)(*args, **kwargs)
475 except Exception as e:
476 except Exception as e:
476 raise
477 raise
477
478
478 def get_chunked_data(method_resp):
479 def get_chunked_data(method_resp):
479 stream = StringIO(method_resp)
480 stream = StringIO(method_resp)
480 while 1:
481 while 1:
481 chunk = stream.read(chunk_size)
482 chunk = stream.read(chunk_size)
482 if not chunk:
483 if not chunk:
483 break
484 break
484 yield chunk
485 yield chunk
485
486
486 response = Response(app_iter=get_chunked_data(resp))
487 response = Response(app_iter=get_chunked_data(resp))
487 response.content_type = 'application/octet-stream'
488 response.content_type = 'application/octet-stream'
488
489
489 return response
490 return response
490
491
491 def status_view(self, request):
492 def status_view(self, request):
492 import vcsserver
493 import vcsserver
493 return {'status': 'OK', 'vcsserver_version': vcsserver.__version__,
494 return {'status': 'OK', 'vcsserver_version': vcsserver.__version__,
494 'pid': os.getpid()}
495 'pid': os.getpid()}
495
496
496 def service_view(self, request):
497 def service_view(self, request):
497 import vcsserver
498 import vcsserver
498
499
499 payload = msgpack.unpackb(request.body, use_list=True)
500 payload = msgpack.unpackb(request.body, use_list=True)
500 server_config, app_config = {}, {}
501 server_config, app_config = {}, {}
501
502
502 try:
503 try:
503 path = self.global_config['__file__']
504 path = self.global_config['__file__']
504 config = configparser.RawConfigParser()
505 config = configparser.RawConfigParser()
505
506
506 config.read(path)
507 config.read(path)
507
508
508 if config.has_section('server:main'):
509 if config.has_section('server:main'):
509 server_config = dict(config.items('server:main'))
510 server_config = dict(config.items('server:main'))
510 if config.has_section('app:main'):
511 if config.has_section('app:main'):
511 app_config = dict(config.items('app:main'))
512 app_config = dict(config.items('app:main'))
512
513
513 except Exception:
514 except Exception:
514 log.exception('Failed to read .ini file for display')
515 log.exception('Failed to read .ini file for display')
515
516
516 environ = os.environ.items()
517 environ = os.environ.items()
517
518
518 resp = {
519 resp = {
519 'id': payload.get('id'),
520 'id': payload.get('id'),
520 'result': dict(
521 'result': dict(
521 version=vcsserver.__version__,
522 version=vcsserver.__version__,
522 config=server_config,
523 config=server_config,
523 app_config=app_config,
524 app_config=app_config,
524 environ=environ,
525 environ=environ,
525 payload=payload,
526 payload=payload,
526 )
527 )
527 }
528 }
528 return resp
529 return resp
529
530
530 def _msgpack_renderer_factory(self, info):
531 def _msgpack_renderer_factory(self, info):
531 def _render(value, system):
532 def _render(value, system):
532 request = system.get('request')
533 request = system.get('request')
533 if request is not None:
534 if request is not None:
534 response = request.response
535 response = request.response
535 ct = response.content_type
536 ct = response.content_type
536 if ct == response.default_content_type:
537 if ct == response.default_content_type:
537 response.content_type = 'application/x-msgpack'
538 response.content_type = 'application/x-msgpack'
538 return msgpack.packb(value)
539 return msgpack.packb(value)
539 return _render
540 return _render
540
541
541 def set_env_from_config(self, environ, config):
542 def set_env_from_config(self, environ, config):
542 dict_conf = {}
543 dict_conf = {}
543 try:
544 try:
544 for elem in config:
545 for elem in config:
545 if elem[0] == 'rhodecode':
546 if elem[0] == 'rhodecode':
546 dict_conf = json.loads(elem[2])
547 dict_conf = json.loads(elem[2])
547 break
548 break
548 except Exception:
549 except Exception:
549 log.exception('Failed to fetch SCM CONFIG')
550 log.exception('Failed to fetch SCM CONFIG')
550 return
551 return
551
552
552 username = dict_conf.get('username')
553 username = dict_conf.get('username')
553 if username:
554 if username:
554 environ['REMOTE_USER'] = username
555 environ['REMOTE_USER'] = username
555 # mercurial specific, some extension api rely on this
556 # mercurial specific, some extension api rely on this
556 environ['HGUSER'] = username
557 environ['HGUSER'] = username
557
558
558 ip = dict_conf.get('ip')
559 ip = dict_conf.get('ip')
559 if ip:
560 if ip:
560 environ['REMOTE_HOST'] = ip
561 environ['REMOTE_HOST'] = ip
561
562
562 if _is_request_chunked(environ):
563 if _is_request_chunked(environ):
563 # set the compatibility flag for webob
564 # set the compatibility flag for webob
564 environ['wsgi.input_terminated'] = True
565 environ['wsgi.input_terminated'] = True
565
566
566 def hg_proxy(self):
567 def hg_proxy(self):
567 @wsgiapp
568 @wsgiapp
568 def _hg_proxy(environ, start_response):
569 def _hg_proxy(environ, start_response):
569 app = WsgiProxy(self.remote_wsgi.HgRemoteWsgi())
570 app = WsgiProxy(self.remote_wsgi.HgRemoteWsgi())
570 return app(environ, start_response)
571 return app(environ, start_response)
571 return _hg_proxy
572 return _hg_proxy
572
573
573 def git_proxy(self):
574 def git_proxy(self):
574 @wsgiapp
575 @wsgiapp
575 def _git_proxy(environ, start_response):
576 def _git_proxy(environ, start_response):
576 app = WsgiProxy(self.remote_wsgi.GitRemoteWsgi())
577 app = WsgiProxy(self.remote_wsgi.GitRemoteWsgi())
577 return app(environ, start_response)
578 return app(environ, start_response)
578 return _git_proxy
579 return _git_proxy
579
580
580 def hg_stream(self):
581 def hg_stream(self):
581 if self._use_echo_app:
582 if self._use_echo_app:
582 @wsgiapp
583 @wsgiapp
583 def _hg_stream(environ, start_response):
584 def _hg_stream(environ, start_response):
584 app = EchoApp('fake_path', 'fake_name', None)
585 app = EchoApp('fake_path', 'fake_name', None)
585 return app(environ, start_response)
586 return app(environ, start_response)
586 return _hg_stream
587 return _hg_stream
587 else:
588 else:
588 @wsgiapp
589 @wsgiapp
589 def _hg_stream(environ, start_response):
590 def _hg_stream(environ, start_response):
590 log.debug('http-app: handling hg stream')
591 log.debug('http-app: handling hg stream')
591 repo_path = environ['HTTP_X_RC_REPO_PATH']
592 repo_path = environ['HTTP_X_RC_REPO_PATH']
592 repo_name = environ['HTTP_X_RC_REPO_NAME']
593 repo_name = environ['HTTP_X_RC_REPO_NAME']
593 packed_config = base64.b64decode(
594 packed_config = base64.b64decode(
594 environ['HTTP_X_RC_REPO_CONFIG'])
595 environ['HTTP_X_RC_REPO_CONFIG'])
595 config = msgpack.unpackb(packed_config)
596 config = msgpack.unpackb(packed_config)
596 app = scm_app.create_hg_wsgi_app(
597 app = scm_app.create_hg_wsgi_app(
597 repo_path, repo_name, config)
598 repo_path, repo_name, config)
598
599
599 # Consistent path information for hgweb
600 # Consistent path information for hgweb
600 environ['PATH_INFO'] = environ['HTTP_X_RC_PATH_INFO']
601 environ['PATH_INFO'] = environ['HTTP_X_RC_PATH_INFO']
601 environ['REPO_NAME'] = repo_name
602 environ['REPO_NAME'] = repo_name
602 self.set_env_from_config(environ, config)
603 self.set_env_from_config(environ, config)
603
604
604 log.debug('http-app: starting app handler '
605 log.debug('http-app: starting app handler '
605 'with %s and process request', app)
606 'with %s and process request', app)
606 return app(environ, ResponseFilter(start_response))
607 return app(environ, ResponseFilter(start_response))
607 return _hg_stream
608 return _hg_stream
608
609
609 def git_stream(self):
610 def git_stream(self):
610 if self._use_echo_app:
611 if self._use_echo_app:
611 @wsgiapp
612 @wsgiapp
612 def _git_stream(environ, start_response):
613 def _git_stream(environ, start_response):
613 app = EchoApp('fake_path', 'fake_name', None)
614 app = EchoApp('fake_path', 'fake_name', None)
614 return app(environ, start_response)
615 return app(environ, start_response)
615 return _git_stream
616 return _git_stream
616 else:
617 else:
617 @wsgiapp
618 @wsgiapp
618 def _git_stream(environ, start_response):
619 def _git_stream(environ, start_response):
619 log.debug('http-app: handling git stream')
620 log.debug('http-app: handling git stream')
620 repo_path = environ['HTTP_X_RC_REPO_PATH']
621 repo_path = environ['HTTP_X_RC_REPO_PATH']
621 repo_name = environ['HTTP_X_RC_REPO_NAME']
622 repo_name = environ['HTTP_X_RC_REPO_NAME']
622 packed_config = base64.b64decode(
623 packed_config = base64.b64decode(
623 environ['HTTP_X_RC_REPO_CONFIG'])
624 environ['HTTP_X_RC_REPO_CONFIG'])
624 config = msgpack.unpackb(packed_config)
625 config = msgpack.unpackb(packed_config)
625
626
626 environ['PATH_INFO'] = environ['HTTP_X_RC_PATH_INFO']
627 environ['PATH_INFO'] = environ['HTTP_X_RC_PATH_INFO']
627 self.set_env_from_config(environ, config)
628 self.set_env_from_config(environ, config)
628
629
629 content_type = environ.get('CONTENT_TYPE', '')
630 content_type = environ.get('CONTENT_TYPE', '')
630
631
631 path = environ['PATH_INFO']
632 path = environ['PATH_INFO']
632 is_lfs_request = GIT_LFS_CONTENT_TYPE in content_type
633 is_lfs_request = GIT_LFS_CONTENT_TYPE in content_type
633 log.debug(
634 log.debug(
634 'LFS: Detecting if request `%s` is LFS server path based '
635 'LFS: Detecting if request `%s` is LFS server path based '
635 'on content type:`%s`, is_lfs:%s',
636 'on content type:`%s`, is_lfs:%s',
636 path, content_type, is_lfs_request)
637 path, content_type, is_lfs_request)
637
638
638 if not is_lfs_request:
639 if not is_lfs_request:
639 # fallback detection by path
640 # fallback detection by path
640 if GIT_LFS_PROTO_PAT.match(path):
641 if GIT_LFS_PROTO_PAT.match(path):
641 is_lfs_request = True
642 is_lfs_request = True
642 log.debug(
643 log.debug(
643 'LFS: fallback detection by path of: `%s`, is_lfs:%s',
644 'LFS: fallback detection by path of: `%s`, is_lfs:%s',
644 path, is_lfs_request)
645 path, is_lfs_request)
645
646
646 if is_lfs_request:
647 if is_lfs_request:
647 app = scm_app.create_git_lfs_wsgi_app(
648 app = scm_app.create_git_lfs_wsgi_app(
648 repo_path, repo_name, config)
649 repo_path, repo_name, config)
649 else:
650 else:
650 app = scm_app.create_git_wsgi_app(
651 app = scm_app.create_git_wsgi_app(
651 repo_path, repo_name, config)
652 repo_path, repo_name, config)
652
653
653 log.debug('http-app: starting app handler '
654 log.debug('http-app: starting app handler '
654 'with %s and process request', app)
655 'with %s and process request', app)
655
656
656 return app(environ, start_response)
657 return app(environ, start_response)
657
658
658 return _git_stream
659 return _git_stream
659
660
660 def handle_vcs_exception(self, exception, request):
661 def handle_vcs_exception(self, exception, request):
661 _vcs_kind = getattr(exception, '_vcs_kind', '')
662 _vcs_kind = getattr(exception, '_vcs_kind', '')
662 if _vcs_kind == 'repo_locked':
663 if _vcs_kind == 'repo_locked':
663 # Get custom repo-locked status code if present.
664 # Get custom repo-locked status code if present.
664 status_code = request.headers.get('X-RC-Locked-Status-Code')
665 status_code = request.headers.get('X-RC-Locked-Status-Code')
665 return HTTPRepoLocked(
666 return HTTPRepoLocked(
666 title=exception.message, status_code=status_code)
667 title=exception.message, status_code=status_code)
667
668
668 elif _vcs_kind == 'repo_branch_protected':
669 elif _vcs_kind == 'repo_branch_protected':
669 # Get custom repo-branch-protected status code if present.
670 # Get custom repo-branch-protected status code if present.
670 return HTTPRepoBranchProtected(title=exception.message)
671 return HTTPRepoBranchProtected(title=exception.message)
671
672
672 exc_info = request.exc_info
673 exc_info = request.exc_info
673 store_exception(id(exc_info), exc_info)
674 store_exception(id(exc_info), exc_info)
674
675
675 traceback_info = 'unavailable'
676 traceback_info = 'unavailable'
676 if request.exc_info:
677 if request.exc_info:
677 exc_type, exc_value, exc_tb = request.exc_info
678 exc_type, exc_value, exc_tb = request.exc_info
678 traceback_info = ''.join(traceback.format_exception(exc_type, exc_value, exc_tb))
679 traceback_info = ''.join(traceback.format_exception(exc_type, exc_value, exc_tb))
679
680
680 log.error(
681 log.error(
681 'error occurred handling this request for path: %s, \n tb: %s',
682 'error occurred handling this request for path: %s, \n tb: %s',
682 request.path, traceback_info)
683 request.path, traceback_info)
683 raise exception
684 raise exception
684
685
685
686
686 class ResponseFilter(object):
687 class ResponseFilter(object):
687
688
688 def __init__(self, start_response):
689 def __init__(self, start_response):
689 self._start_response = start_response
690 self._start_response = start_response
690
691
691 def __call__(self, status, response_headers, exc_info=None):
692 def __call__(self, status, response_headers, exc_info=None):
692 headers = tuple(
693 headers = tuple(
693 (h, v) for h, v in response_headers
694 (h, v) for h, v in response_headers
694 if not wsgiref.util.is_hop_by_hop(h))
695 if not wsgiref.util.is_hop_by_hop(h))
695 return self._start_response(status, headers, exc_info)
696 return self._start_response(status, headers, exc_info)
696
697
697
698
698 def main(global_config, **settings):
699 def main(global_config, **settings):
699 if MercurialFactory:
700 if MercurialFactory:
700 hgpatches.patch_largefiles_capabilities()
701 hgpatches.patch_largefiles_capabilities()
701 hgpatches.patch_subrepo_type_mapping()
702 hgpatches.patch_subrepo_type_mapping()
702
703
703 app = HTTPApplication(settings=settings, global_config=global_config)
704 app = HTTPApplication(settings=settings, global_config=global_config)
704 return app.wsgi_app()
705 return app.wsgi_app()
@@ -1,169 +1,175 b''
1 # -*- coding: utf-8 -*-
1 # -*- coding: utf-8 -*-
2
2
3 # RhodeCode VCSServer provides access to different vcs backends via network.
3 # RhodeCode VCSServer provides access to different vcs backends via network.
4 # Copyright (C) 2014-2020 RhodeCode GmbH
4 # Copyright (C) 2014-2020 RhodeCode GmbH
5 #
5 #
6 # This program is free software; you can redistribute it and/or modify
6 # This program is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 3 of the License, or
8 # the Free Software Foundation; either version 3 of the License, or
9 # (at your option) any later version.
9 # (at your option) any later version.
10 #
10 #
11 # This program is distributed in the hope that it will be useful,
11 # This program is distributed in the hope that it will be useful,
12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 # GNU General Public License for more details.
14 # GNU General Public License for more details.
15 #
15 #
16 # You should have received a copy of the GNU General Public License
16 # You should have received a copy of the GNU General Public License
17 # along with this program; if not, write to the Free Software Foundation,
17 # along with this program; if not, write to the Free Software Foundation,
18 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
18 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19
19
20
20
21 import os
21 import os
22 import time
22 import time
23 import datetime
23 import datetime
24 import msgpack
24 import msgpack
25 import logging
25 import logging
26 import traceback
26 import traceback
27 import tempfile
27 import tempfile
28
28
29 from pyramid import compat
29 from pyramid import compat
30
30
31 log = logging.getLogger(__name__)
31 log = logging.getLogger(__name__)
32
32
33 # NOTE: Any changes should be synced with exc_tracking at rhodecode.lib.exc_tracking
33 # NOTE: Any changes should be synced with exc_tracking at rhodecode.lib.exc_tracking
34 global_prefix = 'vcsserver'
34 global_prefix = 'vcsserver'
35 exc_store_dir_name = 'rc_exception_store_v1'
35 exc_store_dir_name = 'rc_exception_store_v1'
36
36
37
37
38 def exc_serialize(exc_id, tb, exc_type):
38 def exc_serialize(exc_id, tb, exc_type):
39
39
40 data = {
40 data = {
41 'version': 'v1',
41 'version': 'v1',
42 'exc_id': exc_id,
42 'exc_id': exc_id,
43 'exc_utc_date': datetime.datetime.utcnow().isoformat(),
43 'exc_utc_date': datetime.datetime.utcnow().isoformat(),
44 'exc_timestamp': repr(time.time()),
44 'exc_timestamp': repr(time.time()),
45 'exc_message': tb,
45 'exc_message': tb,
46 'exc_type': exc_type,
46 'exc_type': exc_type,
47 }
47 }
48 return msgpack.packb(data), data
48 return msgpack.packb(data), data
49
49
50
50
51 def exc_unserialize(tb):
51 def exc_unserialize(tb):
52 return msgpack.unpackb(tb)
52 return msgpack.unpackb(tb)
53
53
54
54
55 def get_exc_store():
55 def get_exc_store():
56 """
56 """
57 Get and create exception store if it's not existing
57 Get and create exception store if it's not existing
58 """
58 """
59 import vcsserver as app
59 import vcsserver as app
60
60
61 exc_store_dir = app.CONFIG.get('exception_tracker.store_path', '') or tempfile.gettempdir()
61 exc_store_dir = app.CONFIG.get('exception_tracker.store_path', '') or tempfile.gettempdir()
62 _exc_store_path = os.path.join(exc_store_dir, exc_store_dir_name)
62 _exc_store_path = os.path.join(exc_store_dir, exc_store_dir_name)
63
63
64 _exc_store_path = os.path.abspath(_exc_store_path)
64 _exc_store_path = os.path.abspath(_exc_store_path)
65 if not os.path.isdir(_exc_store_path):
65 if not os.path.isdir(_exc_store_path):
66 os.makedirs(_exc_store_path)
66 os.makedirs(_exc_store_path)
67 log.debug('Initializing exceptions store at %s', _exc_store_path)
67 log.debug('Initializing exceptions store at %s', _exc_store_path)
68 return _exc_store_path
68 return _exc_store_path
69
69
70
70
71 def _store_exception(exc_id, exc_info, prefix):
71 def _store_exception(exc_id, exc_info, prefix, request_path=''):
72 exc_type, exc_value, exc_traceback = exc_info
72 exc_type, exc_value, exc_traceback = exc_info
73
73
74 tb = ''.join(traceback.format_exception(
74 tb = ''.join(traceback.format_exception(
75 exc_type, exc_value, exc_traceback, None))
75 exc_type, exc_value, exc_traceback, None))
76
76
77 detailed_tb = getattr(exc_value, '_org_exc_tb', None)
77 detailed_tb = getattr(exc_value, '_org_exc_tb', None)
78
78
79 if detailed_tb:
79 if detailed_tb:
80 if isinstance(detailed_tb, compat.string_types):
80 if isinstance(detailed_tb, compat.string_types):
81 remote_tb = [detailed_tb]
81 remote_tb = [detailed_tb]
82
82
83 tb += (
83 tb += (
84 '\n+++ BEG SOURCE EXCEPTION +++\n\n'
84 '\n+++ BEG SOURCE EXCEPTION +++\n\n'
85 '{}\n'
85 '{}\n'
86 '+++ END SOURCE EXCEPTION +++\n'
86 '+++ END SOURCE EXCEPTION +++\n'
87 ''.format('\n'.join(remote_tb))
87 ''.format('\n'.join(remote_tb))
88 )
88 )
89
89
90 # Avoid that remote_tb also appears in the frame
90 # Avoid that remote_tb also appears in the frame
91 del remote_tb
91 del remote_tb
92
92
93 exc_type_name = exc_type.__name__
93 exc_type_name = exc_type.__name__
94 exc_store_path = get_exc_store()
94 exc_store_path = get_exc_store()
95 exc_data, org_data = exc_serialize(exc_id, tb, exc_type_name)
95 exc_data, org_data = exc_serialize(exc_id, tb, exc_type_name)
96 exc_pref_id = '{}_{}_{}'.format(exc_id, prefix, org_data['exc_timestamp'])
96 exc_pref_id = '{}_{}_{}'.format(exc_id, prefix, org_data['exc_timestamp'])
97 if not os.path.isdir(exc_store_path):
97 if not os.path.isdir(exc_store_path):
98 os.makedirs(exc_store_path)
98 os.makedirs(exc_store_path)
99 stored_exc_path = os.path.join(exc_store_path, exc_pref_id)
99 stored_exc_path = os.path.join(exc_store_path, exc_pref_id)
100 with open(stored_exc_path, 'wb') as f:
100 with open(stored_exc_path, 'wb') as f:
101 f.write(exc_data)
101 f.write(exc_data)
102 log.debug('Stored generated exception %s as: %s', exc_id, stored_exc_path)
102 log.debug('Stored generated exception %s as: %s', exc_id, stored_exc_path)
103
103
104 log.error(
105 'error occurred handling this request.\n'
106 'Path: `%s`, tb: %s',
107 request_path, tb)
104
108
105 def store_exception(exc_id, exc_info, prefix=global_prefix):
109
110 def store_exception(exc_id, exc_info, prefix=global_prefix, request_path=''):
106 """
111 """
107 Example usage::
112 Example usage::
108
113
109 exc_info = sys.exc_info()
114 exc_info = sys.exc_info()
110 store_exception(id(exc_info), exc_info)
115 store_exception(id(exc_info), exc_info)
111 """
116 """
112
117
113 try:
118 try:
114 _store_exception(exc_id=exc_id, exc_info=exc_info, prefix=prefix)
119 _store_exception(exc_id=exc_id, exc_info=exc_info, prefix=prefix,
120 request_path=request_path)
115 except Exception:
121 except Exception:
116 log.exception('Failed to store exception `%s` information', exc_id)
122 log.exception('Failed to store exception `%s` information', exc_id)
117 # there's no way this can fail, it will crash server badly if it does.
123 # there's no way this can fail, it will crash server badly if it does.
118 pass
124 pass
119
125
120
126
121 def _find_exc_file(exc_id, prefix=global_prefix):
127 def _find_exc_file(exc_id, prefix=global_prefix):
122 exc_store_path = get_exc_store()
128 exc_store_path = get_exc_store()
123 if prefix:
129 if prefix:
124 exc_id = '{}_{}'.format(exc_id, prefix)
130 exc_id = '{}_{}'.format(exc_id, prefix)
125 else:
131 else:
126 # search without a prefix
132 # search without a prefix
127 exc_id = '{}'.format(exc_id)
133 exc_id = '{}'.format(exc_id)
128
134
129 # we need to search the store for such start pattern as above
135 # we need to search the store for such start pattern as above
130 for fname in os.listdir(exc_store_path):
136 for fname in os.listdir(exc_store_path):
131 if fname.startswith(exc_id):
137 if fname.startswith(exc_id):
132 exc_id = os.path.join(exc_store_path, fname)
138 exc_id = os.path.join(exc_store_path, fname)
133 break
139 break
134 continue
140 continue
135 else:
141 else:
136 exc_id = None
142 exc_id = None
137
143
138 return exc_id
144 return exc_id
139
145
140
146
141 def _read_exception(exc_id, prefix):
147 def _read_exception(exc_id, prefix):
142 exc_id_file_path = _find_exc_file(exc_id=exc_id, prefix=prefix)
148 exc_id_file_path = _find_exc_file(exc_id=exc_id, prefix=prefix)
143 if exc_id_file_path:
149 if exc_id_file_path:
144 with open(exc_id_file_path, 'rb') as f:
150 with open(exc_id_file_path, 'rb') as f:
145 return exc_unserialize(f.read())
151 return exc_unserialize(f.read())
146 else:
152 else:
147 log.debug('Exception File `%s` not found', exc_id_file_path)
153 log.debug('Exception File `%s` not found', exc_id_file_path)
148 return None
154 return None
149
155
150
156
151 def read_exception(exc_id, prefix=global_prefix):
157 def read_exception(exc_id, prefix=global_prefix):
152 try:
158 try:
153 return _read_exception(exc_id=exc_id, prefix=prefix)
159 return _read_exception(exc_id=exc_id, prefix=prefix)
154 except Exception:
160 except Exception:
155 log.exception('Failed to read exception `%s` information', exc_id)
161 log.exception('Failed to read exception `%s` information', exc_id)
156 # there's no way this can fail, it will crash server badly if it does.
162 # there's no way this can fail, it will crash server badly if it does.
157 return None
163 return None
158
164
159
165
160 def delete_exception(exc_id, prefix=global_prefix):
166 def delete_exception(exc_id, prefix=global_prefix):
161 try:
167 try:
162 exc_id_file_path = _find_exc_file(exc_id, prefix=prefix)
168 exc_id_file_path = _find_exc_file(exc_id, prefix=prefix)
163 if exc_id_file_path:
169 if exc_id_file_path:
164 os.remove(exc_id_file_path)
170 os.remove(exc_id_file_path)
165
171
166 except Exception:
172 except Exception:
167 log.exception('Failed to remove exception `%s` information', exc_id)
173 log.exception('Failed to remove exception `%s` information', exc_id)
168 # there's no way this can fail, it will crash server badly if it does.
174 # there's no way this can fail, it will crash server badly if it does.
169 pass
175 pass
General Comments 0
You need to be logged in to leave comments. Login now