##// END OF EJS Templates
exception_store: rename .ini option for future
marcink -
r540:e2101026 default
parent child Browse files
Show More
@@ -1,87 +1,87 b''
1 1 ################################################################################
2 2 # RhodeCode VCSServer with HTTP Backend - configuration #
3 3 # #
4 4 ################################################################################
5 5
6 6
7 7 [server:main]
8 8 ## COMMON ##
9 9 host = 0.0.0.0
10 10 port = 9900
11 11
12 12 use = egg:waitress#main
13 13
14 14
15 15 [app:main]
16 16 use = egg:rhodecode-vcsserver
17 17
18 18 pyramid.default_locale_name = en
19 19 pyramid.includes =
20 20
21 21 ## default locale used by VCS systems
22 22 locale = en_US.UTF-8
23 23
24 24
25 25 ## path to binaries for vcsserver, it should be set by the installer
26 26 ## at installation time, e.g /home/user/vcsserver-1/profile/bin
27 27 core.binary_dir = ""
28 28
29 29 ## custom exception store path, defaults to TMPDIR
30 exception_store_path =
30 exception_tracker.store_path =
31 31
32 32 ## Default cache dir for caches. Putting this into a ramdisk
33 33 ## can boost performance, eg. /tmpfs/data_ramdisk, however this directory might require
34 34 ## large ammount of space
35 35 cache_dir = %(here)s/rcdev/data
36 36
37 37 ## cache region for storing repo_objects cache
38 38 rc_cache.repo_object.backend = dogpile.cache.rc.memory_lru
39 39 ## cache auto-expires after N seconds
40 40 rc_cache.repo_object.expiration_time = 300
41 41 ## max size of LRU, old values will be discarded if the size of cache reaches max_size
42 42 rc_cache.repo_object.max_size = 100
43 43
44 44
45 45 ################################
46 46 ### LOGGING CONFIGURATION ####
47 47 ################################
48 48 [loggers]
49 49 keys = root, vcsserver
50 50
51 51 [handlers]
52 52 keys = console
53 53
54 54 [formatters]
55 55 keys = generic
56 56
57 57 #############
58 58 ## LOGGERS ##
59 59 #############
60 60 [logger_root]
61 61 level = NOTSET
62 62 handlers = console
63 63
64 64 [logger_vcsserver]
65 65 level = DEBUG
66 66 handlers =
67 67 qualname = vcsserver
68 68 propagate = 1
69 69
70 70
71 71 ##############
72 72 ## HANDLERS ##
73 73 ##############
74 74
75 75 [handler_console]
76 76 class = StreamHandler
77 77 args = (sys.stderr,)
78 78 level = DEBUG
79 79 formatter = generic
80 80
81 81 ################
82 82 ## FORMATTERS ##
83 83 ################
84 84
85 85 [formatter_generic]
86 86 format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s
87 87 datefmt = %Y-%m-%d %H:%M:%S
@@ -1,108 +1,108 b''
1 1 ################################################################################
2 2 # RhodeCode VCSServer with HTTP Backend - configuration #
3 3 # #
4 4 ################################################################################
5 5
6 6
7 7 [server:main]
8 8 ## COMMON ##
9 9 host = 127.0.0.1
10 10 port = 9900
11 11
12 12
13 13 ##########################
14 14 ## GUNICORN WSGI SERVER ##
15 15 ##########################
16 16 ## run with gunicorn --log-config vcsserver.ini --paste vcsserver.ini
17 17 use = egg:gunicorn#main
18 18 ## Sets the number of process workers. Recommended
19 19 ## value is (2 * NUMBER_OF_CPUS + 1), eg 2CPU = 5 workers
20 20 workers = 2
21 21 ## process name
22 22 proc_name = rhodecode_vcsserver
23 23 ## type of worker class, currently `sync` is the only option allowed.
24 24 worker_class = sync
25 25 ## The maximum number of simultaneous clients. Valid only for Gevent
26 26 #worker_connections = 10
27 27 ## max number of requests that worker will handle before being gracefully
28 28 ## restarted, could prevent memory leaks
29 29 max_requests = 1000
30 30 max_requests_jitter = 30
31 31 ## amount of time a worker can spend with handling a request before it
32 32 ## gets killed and restarted. Set to 6hrs
33 33 timeout = 21600
34 34
35 35
36 36 [app:main]
37 37 use = egg:rhodecode-vcsserver
38 38
39 39 pyramid.default_locale_name = en
40 40 pyramid.includes =
41 41
42 42 ## default locale used by VCS systems
43 43 locale = en_US.UTF-8
44 44
45 45
46 46 ## path to binaries for vcsserver, it should be set by the installer
47 47 ## at installation time, e.g /home/user/vcsserver-1/profile/bin
48 48 core.binary_dir = ""
49 49
50 50 ## custom exception store path, defaults to TMPDIR
51 exception_store_path =
51 exception_tracker.store_path =
52 52
53 53 ## Default cache dir for caches. Putting this into a ramdisk
54 54 ## can boost performance, eg. /tmpfs/data_ramdisk, however this directory might require
55 55 ## large ammount of space
56 56 cache_dir = %(here)s/rcdev/data
57 57
58 58 ## cache region for storing repo_objects cache
59 59 rc_cache.repo_object.backend = dogpile.cache.rc.memory_lru
60 60 ## cache auto-expires after N seconds
61 61 rc_cache.repo_object.expiration_time = 300
62 62 ## max size of LRU, old values will be discarded if the size of cache reaches max_size
63 63 rc_cache.repo_object.max_size = 100
64 64
65 65
66 66 ################################
67 67 ### LOGGING CONFIGURATION ####
68 68 ################################
69 69 [loggers]
70 70 keys = root, vcsserver
71 71
72 72 [handlers]
73 73 keys = console
74 74
75 75 [formatters]
76 76 keys = generic
77 77
78 78 #############
79 79 ## LOGGERS ##
80 80 #############
81 81 [logger_root]
82 82 level = NOTSET
83 83 handlers = console
84 84
85 85 [logger_vcsserver]
86 86 level = DEBUG
87 87 handlers =
88 88 qualname = vcsserver
89 89 propagate = 1
90 90
91 91
92 92 ##############
93 93 ## HANDLERS ##
94 94 ##############
95 95
96 96 [handler_console]
97 97 class = StreamHandler
98 98 args = (sys.stderr,)
99 99 level = DEBUG
100 100 formatter = generic
101 101
102 102 ################
103 103 ## FORMATTERS ##
104 104 ################
105 105
106 106 [formatter_generic]
107 107 format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s
108 108 datefmt = %Y-%m-%d %H:%M:%S
@@ -1,598 +1,598 b''
1 1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 2 # Copyright (C) 2014-2018 RhodeCode GmbH
3 3 #
4 4 # This program is free software; you can redistribute it and/or modify
5 5 # it under the terms of the GNU General Public License as published by
6 6 # the Free Software Foundation; either version 3 of the License, or
7 7 # (at your option) any later version.
8 8 #
9 9 # This program is distributed in the hope that it will be useful,
10 10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 12 # GNU General Public License for more details.
13 13 #
14 14 # You should have received a copy of the GNU General Public License
15 15 # along with this program; if not, write to the Free Software Foundation,
16 16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17 17
18 18 import os
19 19 import sys
20 20 import base64
21 21 import locale
22 22 import logging
23 23 import uuid
24 24 import wsgiref.util
25 25 import traceback
26 26 import tempfile
27 27 from itertools import chain
28 28
29 29 import simplejson as json
30 30 import msgpack
31 31 from pyramid.config import Configurator
32 32 from pyramid.settings import asbool, aslist
33 33 from pyramid.wsgi import wsgiapp
34 34 from pyramid.compat import configparser
35 35
36 36
37 37 log = logging.getLogger(__name__)
38 38
39 39 # due to Mercurial/glibc2.27 problems we need to detect if locale settings are
40 40 # causing problems and "fix" it in case they do and fallback to LC_ALL = C
41 41
42 42 try:
43 43 locale.setlocale(locale.LC_ALL, '')
44 44 except locale.Error as e:
45 45 log.error(
46 46 'LOCALE ERROR: failed to set LC_ALL, fallback to LC_ALL=C, org error: %s', e)
47 47 os.environ['LC_ALL'] = 'C'
48 48
49 49 import vcsserver
50 50 from vcsserver import remote_wsgi, scm_app, settings, hgpatches
51 51 from vcsserver.git_lfs.app import GIT_LFS_CONTENT_TYPE, GIT_LFS_PROTO_PAT
52 52 from vcsserver.echo_stub import remote_wsgi as remote_wsgi_stub
53 53 from vcsserver.echo_stub.echo_app import EchoApp
54 54 from vcsserver.exceptions import HTTPRepoLocked, HTTPRepoBranchProtected
55 55 from vcsserver.lib.exc_tracking import store_exception
56 56 from vcsserver.server import VcsServer
57 57
58 58 try:
59 59 from vcsserver.git import GitFactory, GitRemote
60 60 except ImportError:
61 61 GitFactory = None
62 62 GitRemote = None
63 63
64 64 try:
65 65 from vcsserver.hg import MercurialFactory, HgRemote
66 66 except ImportError:
67 67 MercurialFactory = None
68 68 HgRemote = None
69 69
70 70 try:
71 71 from vcsserver.svn import SubversionFactory, SvnRemote
72 72 except ImportError:
73 73 SubversionFactory = None
74 74 SvnRemote = None
75 75
76 76
77 77 def _is_request_chunked(environ):
78 78 stream = environ.get('HTTP_TRANSFER_ENCODING', '') == 'chunked'
79 79 return stream
80 80
81 81
82 82 def _int_setting(settings, name, default):
83 83 settings[name] = int(settings.get(name, default))
84 84 return settings[name]
85 85
86 86
87 87 def _bool_setting(settings, name, default):
88 88 input_val = settings.get(name, default)
89 89 if isinstance(input_val, unicode):
90 90 input_val = input_val.encode('utf8')
91 91 settings[name] = asbool(input_val)
92 92 return settings[name]
93 93
94 94
95 95 def _list_setting(settings, name, default):
96 96 raw_value = settings.get(name, default)
97 97
98 98 # Otherwise we assume it uses pyramids space/newline separation.
99 99 settings[name] = aslist(raw_value)
100 100 return settings[name]
101 101
102 102
103 103 def _string_setting(settings, name, default, lower=True, default_when_empty=False):
104 104 value = settings.get(name, default)
105 105
106 106 if default_when_empty and not value:
107 107 # use default value when value is empty
108 108 value = default
109 109
110 110 if lower:
111 111 value = value.lower()
112 112 settings[name] = value
113 113 return settings[name]
114 114
115 115
116 116 class VCS(object):
117 117 def __init__(self, locale=None, cache_config=None):
118 118 self.locale = locale
119 119 self.cache_config = cache_config
120 120 self._configure_locale()
121 121
122 122 if GitFactory and GitRemote:
123 123 git_factory = GitFactory()
124 124 self._git_remote = GitRemote(git_factory)
125 125 else:
126 126 log.info("Git client import failed")
127 127
128 128 if MercurialFactory and HgRemote:
129 129 hg_factory = MercurialFactory()
130 130 self._hg_remote = HgRemote(hg_factory)
131 131 else:
132 132 log.info("Mercurial client import failed")
133 133
134 134 if SubversionFactory and SvnRemote:
135 135 svn_factory = SubversionFactory()
136 136
137 137 # hg factory is used for svn url validation
138 138 hg_factory = MercurialFactory()
139 139 self._svn_remote = SvnRemote(svn_factory, hg_factory=hg_factory)
140 140 else:
141 141 log.info("Subversion client import failed")
142 142
143 143 self._vcsserver = VcsServer()
144 144
145 145 def _configure_locale(self):
146 146 if self.locale:
147 147 log.info('Settings locale: `LC_ALL` to %s' % self.locale)
148 148 else:
149 149 log.info(
150 150 'Configuring locale subsystem based on environment variables')
151 151 try:
152 152 # If self.locale is the empty string, then the locale
153 153 # module will use the environment variables. See the
154 154 # documentation of the package `locale`.
155 155 locale.setlocale(locale.LC_ALL, self.locale)
156 156
157 157 language_code, encoding = locale.getlocale()
158 158 log.info(
159 159 'Locale set to language code "%s" with encoding "%s".',
160 160 language_code, encoding)
161 161 except locale.Error:
162 162 log.exception(
163 163 'Cannot set locale, not configuring the locale system')
164 164
165 165
166 166 class WsgiProxy(object):
167 167 def __init__(self, wsgi):
168 168 self.wsgi = wsgi
169 169
170 170 def __call__(self, environ, start_response):
171 171 input_data = environ['wsgi.input'].read()
172 172 input_data = msgpack.unpackb(input_data)
173 173
174 174 error = None
175 175 try:
176 176 data, status, headers = self.wsgi.handle(
177 177 input_data['environment'], input_data['input_data'],
178 178 *input_data['args'], **input_data['kwargs'])
179 179 except Exception as e:
180 180 data, status, headers = [], None, None
181 181 error = {
182 182 'message': str(e),
183 183 '_vcs_kind': getattr(e, '_vcs_kind', None)
184 184 }
185 185
186 186 start_response(200, {})
187 187 return self._iterator(error, status, headers, data)
188 188
189 189 def _iterator(self, error, status, headers, data):
190 190 initial_data = [
191 191 error,
192 192 status,
193 193 headers,
194 194 ]
195 195
196 196 for d in chain(initial_data, data):
197 197 yield msgpack.packb(d)
198 198
199 199
200 200 class HTTPApplication(object):
201 201 ALLOWED_EXCEPTIONS = ('KeyError', 'URLError')
202 202
203 203 remote_wsgi = remote_wsgi
204 204 _use_echo_app = False
205 205
206 206 def __init__(self, settings=None, global_config=None):
207 207 self._sanitize_settings_and_apply_defaults(settings)
208 208
209 209 self.config = Configurator(settings=settings)
210 210 self.global_config = global_config
211 211 self.config.include('vcsserver.lib.rc_cache')
212 212
213 213 locale = settings.get('locale', '') or 'en_US.UTF-8'
214 214 vcs = VCS(locale=locale, cache_config=settings)
215 215 self._remotes = {
216 216 'hg': vcs._hg_remote,
217 217 'git': vcs._git_remote,
218 218 'svn': vcs._svn_remote,
219 219 'server': vcs._vcsserver,
220 220 }
221 221 if settings.get('dev.use_echo_app', 'false').lower() == 'true':
222 222 self._use_echo_app = True
223 223 log.warning("Using EchoApp for VCS operations.")
224 224 self.remote_wsgi = remote_wsgi_stub
225 225
226 226 self._configure_settings(global_config, settings)
227 227 self._configure()
228 228
229 229 def _configure_settings(self, global_config, app_settings):
230 230 """
231 231 Configure the settings module.
232 232 """
233 233 settings_merged = global_config.copy()
234 234 settings_merged.update(app_settings)
235 235
236 236 git_path = app_settings.get('git_path', None)
237 237 if git_path:
238 238 settings.GIT_EXECUTABLE = git_path
239 239 binary_dir = app_settings.get('core.binary_dir', None)
240 240 if binary_dir:
241 241 settings.BINARY_DIR = binary_dir
242 242
243 243 # Store the settings to make them available to other modules.
244 244 vcsserver.PYRAMID_SETTINGS = settings_merged
245 245 vcsserver.CONFIG = settings_merged
246 246
247 247 def _sanitize_settings_and_apply_defaults(self, settings):
248 248
249 249 default_cache_dir = os.path.join(tempfile.gettempdir(), 'rc_cache')
250 250
251 251 # save default, cache dir, and use it for all backends later.
252 252 default_cache_dir = _string_setting(
253 253 settings,
254 254 'cache_dir',
255 255 default_cache_dir, lower=False, default_when_empty=True)
256 256
257 257 # ensure we have our dir created
258 258 if not os.path.isdir(default_cache_dir):
259 259 os.makedirs(default_cache_dir, mode=0755)
260 260
261 261 # exception store cache
262 262 _string_setting(
263 263 settings,
264 'exception_store_path',
264 'exception_tracker.store_path',
265 265 default_cache_dir, lower=False)
266 266
267 267 # repo_object cache
268 268 _string_setting(
269 269 settings,
270 270 'rc_cache.repo_object.backend',
271 271 'dogpile.cache.rc.memory_lru')
272 272 _int_setting(
273 273 settings,
274 274 'rc_cache.repo_object.expiration_time',
275 275 300)
276 276 _int_setting(
277 277 settings,
278 278 'rc_cache.repo_object.max_size',
279 279 1024)
280 280
281 281 def _configure(self):
282 282 self.config.add_renderer(
283 283 name='msgpack',
284 284 factory=self._msgpack_renderer_factory)
285 285
286 286 self.config.add_route('service', '/_service')
287 287 self.config.add_route('status', '/status')
288 288 self.config.add_route('hg_proxy', '/proxy/hg')
289 289 self.config.add_route('git_proxy', '/proxy/git')
290 290 self.config.add_route('vcs', '/{backend}')
291 291 self.config.add_route('stream_git', '/stream/git/*repo_name')
292 292 self.config.add_route('stream_hg', '/stream/hg/*repo_name')
293 293
294 294 self.config.add_view(
295 295 self.status_view, route_name='status', renderer='json')
296 296 self.config.add_view(
297 297 self.service_view, route_name='service', renderer='msgpack')
298 298
299 299 self.config.add_view(self.hg_proxy(), route_name='hg_proxy')
300 300 self.config.add_view(self.git_proxy(), route_name='git_proxy')
301 301 self.config.add_view(
302 302 self.vcs_view, route_name='vcs', renderer='msgpack',
303 303 custom_predicates=[self.is_vcs_view])
304 304
305 305 self.config.add_view(self.hg_stream(), route_name='stream_hg')
306 306 self.config.add_view(self.git_stream(), route_name='stream_git')
307 307
308 308 def notfound(request):
309 309 return {'status': '404 NOT FOUND'}
310 310 self.config.add_notfound_view(notfound, renderer='json')
311 311
312 312 self.config.add_view(self.handle_vcs_exception, context=Exception)
313 313
314 314 self.config.add_tween(
315 315 'vcsserver.tweens.RequestWrapperTween',
316 316 )
317 317
318 318 def wsgi_app(self):
319 319 return self.config.make_wsgi_app()
320 320
321 321 def vcs_view(self, request):
322 322 remote = self._remotes[request.matchdict['backend']]
323 323 payload = msgpack.unpackb(request.body, use_list=True)
324 324 method = payload.get('method')
325 325 params = payload.get('params')
326 326 wire = params.get('wire')
327 327 args = params.get('args')
328 328 kwargs = params.get('kwargs')
329 329 context_uid = None
330 330
331 331 if wire:
332 332 try:
333 333 wire['context'] = context_uid = uuid.UUID(wire['context'])
334 334 except KeyError:
335 335 pass
336 336 args.insert(0, wire)
337 337
338 338 log.debug('method called:%s with kwargs:%s context_uid: %s',
339 339 method, kwargs, context_uid)
340 340 try:
341 341 resp = getattr(remote, method)(*args, **kwargs)
342 342 except Exception as e:
343 343 exc_info = list(sys.exc_info())
344 344 exc_type, exc_value, exc_traceback = exc_info
345 345
346 346 org_exc = getattr(e, '_org_exc', None)
347 347 org_exc_name = None
348 348 if org_exc:
349 349 org_exc_name = org_exc.__class__.__name__
350 350 # replace our "faked" exception with our org
351 351 exc_info[0] = org_exc.__class__
352 352 exc_info[1] = org_exc
353 353
354 354 store_exception(id(exc_info), exc_info)
355 355
356 356 tb_info = ''.join(
357 357 traceback.format_exception(exc_type, exc_value, exc_traceback))
358 358
359 359 type_ = e.__class__.__name__
360 360 if type_ not in self.ALLOWED_EXCEPTIONS:
361 361 type_ = None
362 362
363 363 resp = {
364 364 'id': payload.get('id'),
365 365 'error': {
366 366 'message': e.message,
367 367 'traceback': tb_info,
368 368 'org_exc': org_exc_name,
369 369 'type': type_
370 370 }
371 371 }
372 372 try:
373 373 resp['error']['_vcs_kind'] = getattr(e, '_vcs_kind', None)
374 374 except AttributeError:
375 375 pass
376 376 else:
377 377 resp = {
378 378 'id': payload.get('id'),
379 379 'result': resp
380 380 }
381 381
382 382 return resp
383 383
384 384 def status_view(self, request):
385 385 import vcsserver
386 386 return {'status': 'OK', 'vcsserver_version': vcsserver.__version__,
387 387 'pid': os.getpid()}
388 388
389 389 def service_view(self, request):
390 390 import vcsserver
391 391
392 392 payload = msgpack.unpackb(request.body, use_list=True)
393 393
394 394 try:
395 395 path = self.global_config['__file__']
396 396 config = configparser.ConfigParser()
397 397 config.read(path)
398 398 parsed_ini = config
399 399 if parsed_ini.has_section('server:main'):
400 400 parsed_ini = dict(parsed_ini.items('server:main'))
401 401 except Exception:
402 402 log.exception('Failed to read .ini file for display')
403 403 parsed_ini = {}
404 404
405 405 resp = {
406 406 'id': payload.get('id'),
407 407 'result': dict(
408 408 version=vcsserver.__version__,
409 409 config=parsed_ini,
410 410 payload=payload,
411 411 )
412 412 }
413 413 return resp
414 414
415 415 def _msgpack_renderer_factory(self, info):
416 416 def _render(value, system):
417 417 value = msgpack.packb(value)
418 418 request = system.get('request')
419 419 if request is not None:
420 420 response = request.response
421 421 ct = response.content_type
422 422 if ct == response.default_content_type:
423 423 response.content_type = 'application/x-msgpack'
424 424 return value
425 425 return _render
426 426
427 427 def set_env_from_config(self, environ, config):
428 428 dict_conf = {}
429 429 try:
430 430 for elem in config:
431 431 if elem[0] == 'rhodecode':
432 432 dict_conf = json.loads(elem[2])
433 433 break
434 434 except Exception:
435 435 log.exception('Failed to fetch SCM CONFIG')
436 436 return
437 437
438 438 username = dict_conf.get('username')
439 439 if username:
440 440 environ['REMOTE_USER'] = username
441 441 # mercurial specific, some extension api rely on this
442 442 environ['HGUSER'] = username
443 443
444 444 ip = dict_conf.get('ip')
445 445 if ip:
446 446 environ['REMOTE_HOST'] = ip
447 447
448 448 if _is_request_chunked(environ):
449 449 # set the compatibility flag for webob
450 450 environ['wsgi.input_terminated'] = True
451 451
452 452 def hg_proxy(self):
453 453 @wsgiapp
454 454 def _hg_proxy(environ, start_response):
455 455 app = WsgiProxy(self.remote_wsgi.HgRemoteWsgi())
456 456 return app(environ, start_response)
457 457 return _hg_proxy
458 458
459 459 def git_proxy(self):
460 460 @wsgiapp
461 461 def _git_proxy(environ, start_response):
462 462 app = WsgiProxy(self.remote_wsgi.GitRemoteWsgi())
463 463 return app(environ, start_response)
464 464 return _git_proxy
465 465
466 466 def hg_stream(self):
467 467 if self._use_echo_app:
468 468 @wsgiapp
469 469 def _hg_stream(environ, start_response):
470 470 app = EchoApp('fake_path', 'fake_name', None)
471 471 return app(environ, start_response)
472 472 return _hg_stream
473 473 else:
474 474 @wsgiapp
475 475 def _hg_stream(environ, start_response):
476 476 log.debug('http-app: handling hg stream')
477 477 repo_path = environ['HTTP_X_RC_REPO_PATH']
478 478 repo_name = environ['HTTP_X_RC_REPO_NAME']
479 479 packed_config = base64.b64decode(
480 480 environ['HTTP_X_RC_REPO_CONFIG'])
481 481 config = msgpack.unpackb(packed_config)
482 482 app = scm_app.create_hg_wsgi_app(
483 483 repo_path, repo_name, config)
484 484
485 485 # Consistent path information for hgweb
486 486 environ['PATH_INFO'] = environ['HTTP_X_RC_PATH_INFO']
487 487 environ['REPO_NAME'] = repo_name
488 488 self.set_env_from_config(environ, config)
489 489
490 490 log.debug('http-app: starting app handler '
491 491 'with %s and process request', app)
492 492 return app(environ, ResponseFilter(start_response))
493 493 return _hg_stream
494 494
495 495 def git_stream(self):
496 496 if self._use_echo_app:
497 497 @wsgiapp
498 498 def _git_stream(environ, start_response):
499 499 app = EchoApp('fake_path', 'fake_name', None)
500 500 return app(environ, start_response)
501 501 return _git_stream
502 502 else:
503 503 @wsgiapp
504 504 def _git_stream(environ, start_response):
505 505 log.debug('http-app: handling git stream')
506 506 repo_path = environ['HTTP_X_RC_REPO_PATH']
507 507 repo_name = environ['HTTP_X_RC_REPO_NAME']
508 508 packed_config = base64.b64decode(
509 509 environ['HTTP_X_RC_REPO_CONFIG'])
510 510 config = msgpack.unpackb(packed_config)
511 511
512 512 environ['PATH_INFO'] = environ['HTTP_X_RC_PATH_INFO']
513 513 self.set_env_from_config(environ, config)
514 514
515 515 content_type = environ.get('CONTENT_TYPE', '')
516 516
517 517 path = environ['PATH_INFO']
518 518 is_lfs_request = GIT_LFS_CONTENT_TYPE in content_type
519 519 log.debug(
520 520 'LFS: Detecting if request `%s` is LFS server path based '
521 521 'on content type:`%s`, is_lfs:%s',
522 522 path, content_type, is_lfs_request)
523 523
524 524 if not is_lfs_request:
525 525 # fallback detection by path
526 526 if GIT_LFS_PROTO_PAT.match(path):
527 527 is_lfs_request = True
528 528 log.debug(
529 529 'LFS: fallback detection by path of: `%s`, is_lfs:%s',
530 530 path, is_lfs_request)
531 531
532 532 if is_lfs_request:
533 533 app = scm_app.create_git_lfs_wsgi_app(
534 534 repo_path, repo_name, config)
535 535 else:
536 536 app = scm_app.create_git_wsgi_app(
537 537 repo_path, repo_name, config)
538 538
539 539 log.debug('http-app: starting app handler '
540 540 'with %s and process request', app)
541 541
542 542 return app(environ, start_response)
543 543
544 544 return _git_stream
545 545
546 546 def is_vcs_view(self, context, request):
547 547 """
548 548 View predicate that returns true if given backend is supported by
549 549 defined remotes.
550 550 """
551 551 backend = request.matchdict.get('backend')
552 552 return backend in self._remotes
553 553
554 554 def handle_vcs_exception(self, exception, request):
555 555 _vcs_kind = getattr(exception, '_vcs_kind', '')
556 556 if _vcs_kind == 'repo_locked':
557 557 # Get custom repo-locked status code if present.
558 558 status_code = request.headers.get('X-RC-Locked-Status-Code')
559 559 return HTTPRepoLocked(
560 560 title=exception.message, status_code=status_code)
561 561
562 562 elif _vcs_kind == 'repo_branch_protected':
563 563 # Get custom repo-branch-protected status code if present.
564 564 return HTTPRepoBranchProtected(title=exception.message)
565 565
566 566 exc_info = request.exc_info
567 567 store_exception(id(exc_info), exc_info)
568 568
569 569 traceback_info = 'unavailable'
570 570 if request.exc_info:
571 571 exc_type, exc_value, exc_tb = request.exc_info
572 572 traceback_info = ''.join(traceback.format_exception(exc_type, exc_value, exc_tb))
573 573
574 574 log.error(
575 575 'error occurred handling this request for path: %s, \n tb: %s',
576 576 request.path, traceback_info)
577 577 raise exception
578 578
579 579
580 580 class ResponseFilter(object):
581 581
582 582 def __init__(self, start_response):
583 583 self._start_response = start_response
584 584
585 585 def __call__(self, status, response_headers, exc_info=None):
586 586 headers = tuple(
587 587 (h, v) for h, v in response_headers
588 588 if not wsgiref.util.is_hop_by_hop(h))
589 589 return self._start_response(status, headers, exc_info)
590 590
591 591
592 592 def main(global_config, **settings):
593 593 if MercurialFactory:
594 594 hgpatches.patch_largefiles_capabilities()
595 595 hgpatches.patch_subrepo_type_mapping()
596 596
597 597 app = HTTPApplication(settings=settings, global_config=global_config)
598 598 return app.wsgi_app()
@@ -1,151 +1,151 b''
1 1 # -*- coding: utf-8 -*-
2 2
3 3 # RhodeCode VCSServer provides access to different vcs backends via network.
4 4 # Copyright (C) 2014-2018 RhodeCode GmbH
5 5 #
6 6 # This program is free software; you can redistribute it and/or modify
7 7 # it under the terms of the GNU General Public License as published by
8 8 # the Free Software Foundation; either version 3 of the License, or
9 9 # (at your option) any later version.
10 10 #
11 11 # This program is distributed in the hope that it will be useful,
12 12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
13 13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 14 # GNU General Public License for more details.
15 15 #
16 16 # You should have received a copy of the GNU General Public License
17 17 # along with this program; if not, write to the Free Software Foundation,
18 18 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19 19
20 20
21 21 import os
22 22 import time
23 23 import datetime
24 24 import msgpack
25 25 import logging
26 26 import traceback
27 27 import tempfile
28 28
29 29
30 30 log = logging.getLogger(__name__)
31 31
32 32 # NOTE: Any changes should be synced with exc_tracking at rhodecode.lib.exc_tracking
33 33 global_prefix = 'vcsserver'
34 34 exc_store_dir_name = 'rc_exception_store_v1'
35 35
36 36
37 37 def exc_serialize(exc_id, tb, exc_type):
38 38
39 39 data = {
40 40 'version': 'v1',
41 41 'exc_id': exc_id,
42 42 'exc_utc_date': datetime.datetime.utcnow().isoformat(),
43 43 'exc_timestamp': repr(time.time()),
44 44 'exc_message': tb,
45 45 'exc_type': exc_type,
46 46 }
47 47 return msgpack.packb(data), data
48 48
49 49
50 50 def exc_unserialize(tb):
51 51 return msgpack.unpackb(tb)
52 52
53 53
54 54 def get_exc_store():
55 55 """
56 56 Get and create exception store if it's not existing
57 57 """
58 58 import vcsserver as app
59 59
60 exc_store_dir = app.CONFIG.get('exception_store_path', '') or tempfile.gettempdir()
60 exc_store_dir = app.CONFIG.get('exception_tracker.store_path', '') or tempfile.gettempdir()
61 61 _exc_store_path = os.path.join(exc_store_dir, exc_store_dir_name)
62 62
63 63 _exc_store_path = os.path.abspath(_exc_store_path)
64 64 if not os.path.isdir(_exc_store_path):
65 65 os.makedirs(_exc_store_path)
66 66 log.debug('Initializing exceptions store at %s', _exc_store_path)
67 67 return _exc_store_path
68 68
69 69
70 70 def _store_exception(exc_id, exc_info, prefix):
71 71 exc_type, exc_value, exc_traceback = exc_info
72 72 tb = ''.join(traceback.format_exception(
73 73 exc_type, exc_value, exc_traceback, None))
74 74
75 75 exc_type_name = exc_type.__name__
76 76 exc_store_path = get_exc_store()
77 77 exc_data, org_data = exc_serialize(exc_id, tb, exc_type_name)
78 78 exc_pref_id = '{}_{}_{}'.format(exc_id, prefix, org_data['exc_timestamp'])
79 79 if not os.path.isdir(exc_store_path):
80 80 os.makedirs(exc_store_path)
81 81 stored_exc_path = os.path.join(exc_store_path, exc_pref_id)
82 82 with open(stored_exc_path, 'wb') as f:
83 83 f.write(exc_data)
84 84 log.debug('Stored generated exception %s as: %s', exc_id, stored_exc_path)
85 85
86 86
87 87 def store_exception(exc_id, exc_info, prefix=global_prefix):
88 88 """
89 89 Example usage::
90 90
91 91 exc_info = sys.exc_info()
92 92 store_exception(id(exc_info), exc_info)
93 93 """
94 94
95 95 try:
96 96 _store_exception(exc_id=exc_id, exc_info=exc_info, prefix=prefix)
97 97 except Exception:
98 98 log.exception('Failed to store exception `%s` information', exc_id)
99 99 # there's no way this can fail, it will crash server badly if it does.
100 100 pass
101 101
102 102
103 103 def _find_exc_file(exc_id, prefix=global_prefix):
104 104 exc_store_path = get_exc_store()
105 105 if prefix:
106 106 exc_id = '{}_{}'.format(exc_id, prefix)
107 107 else:
108 108 # search without a prefix
109 109 exc_id = '{}'.format(exc_id)
110 110
111 111 # we need to search the store for such start pattern as above
112 112 for fname in os.listdir(exc_store_path):
113 113 if fname.startswith(exc_id):
114 114 exc_id = os.path.join(exc_store_path, fname)
115 115 break
116 116 continue
117 117 else:
118 118 exc_id = None
119 119
120 120 return exc_id
121 121
122 122
123 123 def _read_exception(exc_id, prefix):
124 124 exc_id_file_path = _find_exc_file(exc_id=exc_id, prefix=prefix)
125 125 if exc_id_file_path:
126 126 with open(exc_id_file_path, 'rb') as f:
127 127 return exc_unserialize(f.read())
128 128 else:
129 129 log.debug('Exception File `%s` not found', exc_id_file_path)
130 130 return None
131 131
132 132
133 133 def read_exception(exc_id, prefix=global_prefix):
134 134 try:
135 135 return _read_exception(exc_id=exc_id, prefix=prefix)
136 136 except Exception:
137 137 log.exception('Failed to read exception `%s` information', exc_id)
138 138 # there's no way this can fail, it will crash server badly if it does.
139 139 return None
140 140
141 141
142 142 def delete_exception(exc_id, prefix=global_prefix):
143 143 try:
144 144 exc_id_file_path = _find_exc_file(exc_id, prefix=prefix)
145 145 if exc_id_file_path:
146 146 os.remove(exc_id_file_path)
147 147
148 148 except Exception:
149 149 log.exception('Failed to remove exception `%s` information', exc_id)
150 150 # there's no way this can fail, it will crash server badly if it does.
151 151 pass
General Comments 0
You need to be logged in to leave comments. Login now