##// END OF EJS Templates
exc_store: allow to specify a custom path for exception store.
marcink -
r539:de749a38 default
parent child Browse files
Show More
@@ -1,79 +1,87 b''
1 1 ################################################################################
2 2 # RhodeCode VCSServer with HTTP Backend - configuration #
3 3 # #
4 4 ################################################################################
5 5
6 6
7 7 [server:main]
8 8 ## COMMON ##
9 9 host = 0.0.0.0
10 10 port = 9900
11 11
12 12 use = egg:waitress#main
13 13
14 14
15 15 [app:main]
16 16 use = egg:rhodecode-vcsserver
17 17
18 18 pyramid.default_locale_name = en
19 19 pyramid.includes =
20 20
21 21 ## default locale used by VCS systems
22 22 locale = en_US.UTF-8
23 23
24 24
25 25 ## path to binaries for vcsserver, it should be set by the installer
26 26 ## at installation time, e.g /home/user/vcsserver-1/profile/bin
27 27 core.binary_dir = ""
28 28
29 ## custom exception store path, defaults to TMPDIR
30 exception_store_path =
31
32 ## Default cache dir for caches. Putting this into a ramdisk
33 ## can boost performance, eg. /tmpfs/data_ramdisk, however this directory might require
34 ## large ammount of space
35 cache_dir = %(here)s/rcdev/data
36
29 37 ## cache region for storing repo_objects cache
30 38 rc_cache.repo_object.backend = dogpile.cache.rc.memory_lru
31 39 ## cache auto-expires after N seconds
32 40 rc_cache.repo_object.expiration_time = 300
33 41 ## max size of LRU, old values will be discarded if the size of cache reaches max_size
34 42 rc_cache.repo_object.max_size = 100
35 43
36 44
37 45 ################################
38 46 ### LOGGING CONFIGURATION ####
39 47 ################################
40 48 [loggers]
41 49 keys = root, vcsserver
42 50
43 51 [handlers]
44 52 keys = console
45 53
46 54 [formatters]
47 55 keys = generic
48 56
49 57 #############
50 58 ## LOGGERS ##
51 59 #############
52 60 [logger_root]
53 61 level = NOTSET
54 62 handlers = console
55 63
56 64 [logger_vcsserver]
57 65 level = DEBUG
58 66 handlers =
59 67 qualname = vcsserver
60 68 propagate = 1
61 69
62 70
63 71 ##############
64 72 ## HANDLERS ##
65 73 ##############
66 74
67 75 [handler_console]
68 76 class = StreamHandler
69 77 args = (sys.stderr,)
70 78 level = DEBUG
71 79 formatter = generic
72 80
73 81 ################
74 82 ## FORMATTERS ##
75 83 ################
76 84
77 85 [formatter_generic]
78 86 format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s
79 87 datefmt = %Y-%m-%d %H:%M:%S
@@ -1,100 +1,108 b''
1 1 ################################################################################
2 2 # RhodeCode VCSServer with HTTP Backend - configuration #
3 3 # #
4 4 ################################################################################
5 5
6 6
7 7 [server:main]
8 8 ## COMMON ##
9 9 host = 127.0.0.1
10 10 port = 9900
11 11
12 12
13 13 ##########################
14 14 ## GUNICORN WSGI SERVER ##
15 15 ##########################
16 16 ## run with gunicorn --log-config vcsserver.ini --paste vcsserver.ini
17 17 use = egg:gunicorn#main
18 18 ## Sets the number of process workers. Recommended
19 19 ## value is (2 * NUMBER_OF_CPUS + 1), eg 2CPU = 5 workers
20 20 workers = 2
21 21 ## process name
22 22 proc_name = rhodecode_vcsserver
23 23 ## type of worker class, currently `sync` is the only option allowed.
24 24 worker_class = sync
25 25 ## The maximum number of simultaneous clients. Valid only for Gevent
26 26 #worker_connections = 10
27 27 ## max number of requests that worker will handle before being gracefully
28 28 ## restarted, could prevent memory leaks
29 29 max_requests = 1000
30 30 max_requests_jitter = 30
31 31 ## amount of time a worker can spend with handling a request before it
32 32 ## gets killed and restarted. Set to 6hrs
33 33 timeout = 21600
34 34
35 35
36 36 [app:main]
37 37 use = egg:rhodecode-vcsserver
38 38
39 39 pyramid.default_locale_name = en
40 40 pyramid.includes =
41 41
42 42 ## default locale used by VCS systems
43 43 locale = en_US.UTF-8
44 44
45 45
46 46 ## path to binaries for vcsserver, it should be set by the installer
47 47 ## at installation time, e.g /home/user/vcsserver-1/profile/bin
48 48 core.binary_dir = ""
49 49
50 ## custom exception store path, defaults to TMPDIR
51 exception_store_path =
52
53 ## Default cache dir for caches. Putting this into a ramdisk
54 ## can boost performance, eg. /tmpfs/data_ramdisk, however this directory might require
55 ## large ammount of space
56 cache_dir = %(here)s/rcdev/data
57
50 58 ## cache region for storing repo_objects cache
51 59 rc_cache.repo_object.backend = dogpile.cache.rc.memory_lru
52 60 ## cache auto-expires after N seconds
53 61 rc_cache.repo_object.expiration_time = 300
54 62 ## max size of LRU, old values will be discarded if the size of cache reaches max_size
55 63 rc_cache.repo_object.max_size = 100
56 64
57 65
58 66 ################################
59 67 ### LOGGING CONFIGURATION ####
60 68 ################################
61 69 [loggers]
62 70 keys = root, vcsserver
63 71
64 72 [handlers]
65 73 keys = console
66 74
67 75 [formatters]
68 76 keys = generic
69 77
70 78 #############
71 79 ## LOGGERS ##
72 80 #############
73 81 [logger_root]
74 82 level = NOTSET
75 83 handlers = console
76 84
77 85 [logger_vcsserver]
78 86 level = DEBUG
79 87 handlers =
80 88 qualname = vcsserver
81 89 propagate = 1
82 90
83 91
84 92 ##############
85 93 ## HANDLERS ##
86 94 ##############
87 95
88 96 [handler_console]
89 97 class = StreamHandler
90 98 args = (sys.stderr,)
91 99 level = DEBUG
92 100 formatter = generic
93 101
94 102 ################
95 103 ## FORMATTERS ##
96 104 ################
97 105
98 106 [formatter_generic]
99 107 format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s
100 108 datefmt = %Y-%m-%d %H:%M:%S
@@ -1,21 +1,28 b''
1 1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 2 # Copyright (C) 2014-2018 RhodeCode GmbH
3 3 #
4 4 # This program is free software; you can redistribute it and/or modify
5 5 # it under the terms of the GNU General Public License as published by
6 6 # the Free Software Foundation; either version 3 of the License, or
7 7 # (at your option) any later version.
8 8 #
9 9 # This program is distributed in the hope that it will be useful,
10 10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 12 # GNU General Public License for more details.
13 13 #
14 14 # You should have received a copy of the GNU General Public License
15 15 # along with this program; if not, write to the Free Software Foundation,
16 16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17 17
18 18 import pkgutil
19 19
20 20
21 21 __version__ = pkgutil.get_data('vcsserver', 'VERSION').strip()
22
23 # link to config for pyramid
24 CONFIG = {}
25
26 # Populated with the settings dictionary from application init in
27 #
28 PYRAMID_SETTINGS = {}
@@ -1,563 +1,598 b''
1 1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 2 # Copyright (C) 2014-2018 RhodeCode GmbH
3 3 #
4 4 # This program is free software; you can redistribute it and/or modify
5 5 # it under the terms of the GNU General Public License as published by
6 6 # the Free Software Foundation; either version 3 of the License, or
7 7 # (at your option) any later version.
8 8 #
9 9 # This program is distributed in the hope that it will be useful,
10 10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 12 # GNU General Public License for more details.
13 13 #
14 14 # You should have received a copy of the GNU General Public License
15 15 # along with this program; if not, write to the Free Software Foundation,
16 16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17 17
18 18 import os
19 19 import sys
20 20 import base64
21 21 import locale
22 22 import logging
23 23 import uuid
24 24 import wsgiref.util
25 25 import traceback
26 import tempfile
26 27 from itertools import chain
27 28
28 29 import simplejson as json
29 30 import msgpack
30 31 from pyramid.config import Configurator
31 32 from pyramid.settings import asbool, aslist
32 33 from pyramid.wsgi import wsgiapp
33 34 from pyramid.compat import configparser
34 35
35 36
36 37 log = logging.getLogger(__name__)
37 38
38 39 # due to Mercurial/glibc2.27 problems we need to detect if locale settings are
39 40 # causing problems and "fix" it in case they do and fallback to LC_ALL = C
40 41
41 42 try:
42 43 locale.setlocale(locale.LC_ALL, '')
43 44 except locale.Error as e:
44 45 log.error(
45 46 'LOCALE ERROR: failed to set LC_ALL, fallback to LC_ALL=C, org error: %s', e)
46 47 os.environ['LC_ALL'] = 'C'
47 48
48
49 import vcsserver
49 50 from vcsserver import remote_wsgi, scm_app, settings, hgpatches
50 51 from vcsserver.git_lfs.app import GIT_LFS_CONTENT_TYPE, GIT_LFS_PROTO_PAT
51 52 from vcsserver.echo_stub import remote_wsgi as remote_wsgi_stub
52 53 from vcsserver.echo_stub.echo_app import EchoApp
53 54 from vcsserver.exceptions import HTTPRepoLocked, HTTPRepoBranchProtected
54 55 from vcsserver.lib.exc_tracking import store_exception
55 56 from vcsserver.server import VcsServer
56 57
57 58 try:
58 59 from vcsserver.git import GitFactory, GitRemote
59 60 except ImportError:
60 61 GitFactory = None
61 62 GitRemote = None
62 63
63 64 try:
64 65 from vcsserver.hg import MercurialFactory, HgRemote
65 66 except ImportError:
66 67 MercurialFactory = None
67 68 HgRemote = None
68 69
69 70 try:
70 71 from vcsserver.svn import SubversionFactory, SvnRemote
71 72 except ImportError:
72 73 SubversionFactory = None
73 74 SvnRemote = None
74 75
75 76
76
77
78 77 def _is_request_chunked(environ):
79 78 stream = environ.get('HTTP_TRANSFER_ENCODING', '') == 'chunked'
80 79 return stream
81 80
82 81
83 82 def _int_setting(settings, name, default):
84 83 settings[name] = int(settings.get(name, default))
84 return settings[name]
85 85
86 86
87 87 def _bool_setting(settings, name, default):
88 88 input_val = settings.get(name, default)
89 89 if isinstance(input_val, unicode):
90 90 input_val = input_val.encode('utf8')
91 91 settings[name] = asbool(input_val)
92 return settings[name]
92 93
93 94
94 95 def _list_setting(settings, name, default):
95 96 raw_value = settings.get(name, default)
96 97
97 98 # Otherwise we assume it uses pyramids space/newline separation.
98 99 settings[name] = aslist(raw_value)
100 return settings[name]
99 101
100 102
101 def _string_setting(settings, name, default, lower=True):
103 def _string_setting(settings, name, default, lower=True, default_when_empty=False):
102 104 value = settings.get(name, default)
105
106 if default_when_empty and not value:
107 # use default value when value is empty
108 value = default
109
103 110 if lower:
104 111 value = value.lower()
105 112 settings[name] = value
113 return settings[name]
106 114
107 115
108 116 class VCS(object):
109 117 def __init__(self, locale=None, cache_config=None):
110 118 self.locale = locale
111 119 self.cache_config = cache_config
112 120 self._configure_locale()
113 121
114 122 if GitFactory and GitRemote:
115 123 git_factory = GitFactory()
116 124 self._git_remote = GitRemote(git_factory)
117 125 else:
118 126 log.info("Git client import failed")
119 127
120 128 if MercurialFactory and HgRemote:
121 129 hg_factory = MercurialFactory()
122 130 self._hg_remote = HgRemote(hg_factory)
123 131 else:
124 132 log.info("Mercurial client import failed")
125 133
126 134 if SubversionFactory and SvnRemote:
127 135 svn_factory = SubversionFactory()
128 136
129 137 # hg factory is used for svn url validation
130 138 hg_factory = MercurialFactory()
131 139 self._svn_remote = SvnRemote(svn_factory, hg_factory=hg_factory)
132 140 else:
133 141 log.info("Subversion client import failed")
134 142
135 143 self._vcsserver = VcsServer()
136 144
137 145 def _configure_locale(self):
138 146 if self.locale:
139 147 log.info('Settings locale: `LC_ALL` to %s' % self.locale)
140 148 else:
141 149 log.info(
142 150 'Configuring locale subsystem based on environment variables')
143 151 try:
144 152 # If self.locale is the empty string, then the locale
145 153 # module will use the environment variables. See the
146 154 # documentation of the package `locale`.
147 155 locale.setlocale(locale.LC_ALL, self.locale)
148 156
149 157 language_code, encoding = locale.getlocale()
150 158 log.info(
151 159 'Locale set to language code "%s" with encoding "%s".',
152 160 language_code, encoding)
153 161 except locale.Error:
154 162 log.exception(
155 163 'Cannot set locale, not configuring the locale system')
156 164
157 165
158 166 class WsgiProxy(object):
159 167 def __init__(self, wsgi):
160 168 self.wsgi = wsgi
161 169
162 170 def __call__(self, environ, start_response):
163 171 input_data = environ['wsgi.input'].read()
164 172 input_data = msgpack.unpackb(input_data)
165 173
166 174 error = None
167 175 try:
168 176 data, status, headers = self.wsgi.handle(
169 177 input_data['environment'], input_data['input_data'],
170 178 *input_data['args'], **input_data['kwargs'])
171 179 except Exception as e:
172 180 data, status, headers = [], None, None
173 181 error = {
174 182 'message': str(e),
175 183 '_vcs_kind': getattr(e, '_vcs_kind', None)
176 184 }
177 185
178 186 start_response(200, {})
179 187 return self._iterator(error, status, headers, data)
180 188
181 189 def _iterator(self, error, status, headers, data):
182 190 initial_data = [
183 191 error,
184 192 status,
185 193 headers,
186 194 ]
187 195
188 196 for d in chain(initial_data, data):
189 197 yield msgpack.packb(d)
190 198
191 199
192 200 class HTTPApplication(object):
193 201 ALLOWED_EXCEPTIONS = ('KeyError', 'URLError')
194 202
195 203 remote_wsgi = remote_wsgi
196 204 _use_echo_app = False
197 205
198 206 def __init__(self, settings=None, global_config=None):
199 207 self._sanitize_settings_and_apply_defaults(settings)
200 208
201 209 self.config = Configurator(settings=settings)
202 210 self.global_config = global_config
203 211 self.config.include('vcsserver.lib.rc_cache')
204 212
205 213 locale = settings.get('locale', '') or 'en_US.UTF-8'
206 214 vcs = VCS(locale=locale, cache_config=settings)
207 215 self._remotes = {
208 216 'hg': vcs._hg_remote,
209 217 'git': vcs._git_remote,
210 218 'svn': vcs._svn_remote,
211 219 'server': vcs._vcsserver,
212 220 }
213 221 if settings.get('dev.use_echo_app', 'false').lower() == 'true':
214 222 self._use_echo_app = True
215 223 log.warning("Using EchoApp for VCS operations.")
216 224 self.remote_wsgi = remote_wsgi_stub
217 self._configure_settings(settings)
225
226 self._configure_settings(global_config, settings)
218 227 self._configure()
219 228
220 def _configure_settings(self, app_settings):
229 def _configure_settings(self, global_config, app_settings):
221 230 """
222 231 Configure the settings module.
223 232 """
233 settings_merged = global_config.copy()
234 settings_merged.update(app_settings)
235
224 236 git_path = app_settings.get('git_path', None)
225 237 if git_path:
226 238 settings.GIT_EXECUTABLE = git_path
227 239 binary_dir = app_settings.get('core.binary_dir', None)
228 240 if binary_dir:
229 241 settings.BINARY_DIR = binary_dir
230 242
243 # Store the settings to make them available to other modules.
244 vcsserver.PYRAMID_SETTINGS = settings_merged
245 vcsserver.CONFIG = settings_merged
246
231 247 def _sanitize_settings_and_apply_defaults(self, settings):
248
249 default_cache_dir = os.path.join(tempfile.gettempdir(), 'rc_cache')
250
251 # save default, cache dir, and use it for all backends later.
252 default_cache_dir = _string_setting(
253 settings,
254 'cache_dir',
255 default_cache_dir, lower=False, default_when_empty=True)
256
257 # ensure we have our dir created
258 if not os.path.isdir(default_cache_dir):
259 os.makedirs(default_cache_dir, mode=0755)
260
261 # exception store cache
262 _string_setting(
263 settings,
264 'exception_store_path',
265 default_cache_dir, lower=False)
266
232 267 # repo_object cache
233 268 _string_setting(
234 269 settings,
235 270 'rc_cache.repo_object.backend',
236 271 'dogpile.cache.rc.memory_lru')
237 272 _int_setting(
238 273 settings,
239 274 'rc_cache.repo_object.expiration_time',
240 275 300)
241 276 _int_setting(
242 277 settings,
243 278 'rc_cache.repo_object.max_size',
244 279 1024)
245 280
246 281 def _configure(self):
247 282 self.config.add_renderer(
248 283 name='msgpack',
249 284 factory=self._msgpack_renderer_factory)
250 285
251 286 self.config.add_route('service', '/_service')
252 287 self.config.add_route('status', '/status')
253 288 self.config.add_route('hg_proxy', '/proxy/hg')
254 289 self.config.add_route('git_proxy', '/proxy/git')
255 290 self.config.add_route('vcs', '/{backend}')
256 291 self.config.add_route('stream_git', '/stream/git/*repo_name')
257 292 self.config.add_route('stream_hg', '/stream/hg/*repo_name')
258 293
259 294 self.config.add_view(
260 295 self.status_view, route_name='status', renderer='json')
261 296 self.config.add_view(
262 297 self.service_view, route_name='service', renderer='msgpack')
263 298
264 299 self.config.add_view(self.hg_proxy(), route_name='hg_proxy')
265 300 self.config.add_view(self.git_proxy(), route_name='git_proxy')
266 301 self.config.add_view(
267 302 self.vcs_view, route_name='vcs', renderer='msgpack',
268 303 custom_predicates=[self.is_vcs_view])
269 304
270 305 self.config.add_view(self.hg_stream(), route_name='stream_hg')
271 306 self.config.add_view(self.git_stream(), route_name='stream_git')
272 307
273 308 def notfound(request):
274 309 return {'status': '404 NOT FOUND'}
275 310 self.config.add_notfound_view(notfound, renderer='json')
276 311
277 312 self.config.add_view(self.handle_vcs_exception, context=Exception)
278 313
279 314 self.config.add_tween(
280 315 'vcsserver.tweens.RequestWrapperTween',
281 316 )
282 317
283 318 def wsgi_app(self):
284 319 return self.config.make_wsgi_app()
285 320
286 321 def vcs_view(self, request):
287 322 remote = self._remotes[request.matchdict['backend']]
288 323 payload = msgpack.unpackb(request.body, use_list=True)
289 324 method = payload.get('method')
290 325 params = payload.get('params')
291 326 wire = params.get('wire')
292 327 args = params.get('args')
293 328 kwargs = params.get('kwargs')
294 329 context_uid = None
295 330
296 331 if wire:
297 332 try:
298 333 wire['context'] = context_uid = uuid.UUID(wire['context'])
299 334 except KeyError:
300 335 pass
301 336 args.insert(0, wire)
302 337
303 338 log.debug('method called:%s with kwargs:%s context_uid: %s',
304 339 method, kwargs, context_uid)
305 340 try:
306 341 resp = getattr(remote, method)(*args, **kwargs)
307 342 except Exception as e:
308 343 exc_info = list(sys.exc_info())
309 344 exc_type, exc_value, exc_traceback = exc_info
310 345
311 346 org_exc = getattr(e, '_org_exc', None)
312 347 org_exc_name = None
313 348 if org_exc:
314 349 org_exc_name = org_exc.__class__.__name__
315 350 # replace our "faked" exception with our org
316 351 exc_info[0] = org_exc.__class__
317 352 exc_info[1] = org_exc
318 353
319 354 store_exception(id(exc_info), exc_info)
320 355
321 356 tb_info = ''.join(
322 357 traceback.format_exception(exc_type, exc_value, exc_traceback))
323 358
324 359 type_ = e.__class__.__name__
325 360 if type_ not in self.ALLOWED_EXCEPTIONS:
326 361 type_ = None
327 362
328 363 resp = {
329 364 'id': payload.get('id'),
330 365 'error': {
331 366 'message': e.message,
332 367 'traceback': tb_info,
333 368 'org_exc': org_exc_name,
334 369 'type': type_
335 370 }
336 371 }
337 372 try:
338 373 resp['error']['_vcs_kind'] = getattr(e, '_vcs_kind', None)
339 374 except AttributeError:
340 375 pass
341 376 else:
342 377 resp = {
343 378 'id': payload.get('id'),
344 379 'result': resp
345 380 }
346 381
347 382 return resp
348 383
349 384 def status_view(self, request):
350 385 import vcsserver
351 386 return {'status': 'OK', 'vcsserver_version': vcsserver.__version__,
352 387 'pid': os.getpid()}
353 388
354 389 def service_view(self, request):
355 390 import vcsserver
356 391
357 392 payload = msgpack.unpackb(request.body, use_list=True)
358 393
359 394 try:
360 395 path = self.global_config['__file__']
361 396 config = configparser.ConfigParser()
362 397 config.read(path)
363 398 parsed_ini = config
364 399 if parsed_ini.has_section('server:main'):
365 400 parsed_ini = dict(parsed_ini.items('server:main'))
366 401 except Exception:
367 402 log.exception('Failed to read .ini file for display')
368 403 parsed_ini = {}
369 404
370 405 resp = {
371 406 'id': payload.get('id'),
372 407 'result': dict(
373 408 version=vcsserver.__version__,
374 409 config=parsed_ini,
375 410 payload=payload,
376 411 )
377 412 }
378 413 return resp
379 414
380 415 def _msgpack_renderer_factory(self, info):
381 416 def _render(value, system):
382 417 value = msgpack.packb(value)
383 418 request = system.get('request')
384 419 if request is not None:
385 420 response = request.response
386 421 ct = response.content_type
387 422 if ct == response.default_content_type:
388 423 response.content_type = 'application/x-msgpack'
389 424 return value
390 425 return _render
391 426
392 427 def set_env_from_config(self, environ, config):
393 428 dict_conf = {}
394 429 try:
395 430 for elem in config:
396 431 if elem[0] == 'rhodecode':
397 432 dict_conf = json.loads(elem[2])
398 433 break
399 434 except Exception:
400 435 log.exception('Failed to fetch SCM CONFIG')
401 436 return
402 437
403 438 username = dict_conf.get('username')
404 439 if username:
405 440 environ['REMOTE_USER'] = username
406 441 # mercurial specific, some extension api rely on this
407 442 environ['HGUSER'] = username
408 443
409 444 ip = dict_conf.get('ip')
410 445 if ip:
411 446 environ['REMOTE_HOST'] = ip
412 447
413 448 if _is_request_chunked(environ):
414 449 # set the compatibility flag for webob
415 450 environ['wsgi.input_terminated'] = True
416 451
417 452 def hg_proxy(self):
418 453 @wsgiapp
419 454 def _hg_proxy(environ, start_response):
420 455 app = WsgiProxy(self.remote_wsgi.HgRemoteWsgi())
421 456 return app(environ, start_response)
422 457 return _hg_proxy
423 458
424 459 def git_proxy(self):
425 460 @wsgiapp
426 461 def _git_proxy(environ, start_response):
427 462 app = WsgiProxy(self.remote_wsgi.GitRemoteWsgi())
428 463 return app(environ, start_response)
429 464 return _git_proxy
430 465
431 466 def hg_stream(self):
432 467 if self._use_echo_app:
433 468 @wsgiapp
434 469 def _hg_stream(environ, start_response):
435 470 app = EchoApp('fake_path', 'fake_name', None)
436 471 return app(environ, start_response)
437 472 return _hg_stream
438 473 else:
439 474 @wsgiapp
440 475 def _hg_stream(environ, start_response):
441 476 log.debug('http-app: handling hg stream')
442 477 repo_path = environ['HTTP_X_RC_REPO_PATH']
443 478 repo_name = environ['HTTP_X_RC_REPO_NAME']
444 479 packed_config = base64.b64decode(
445 480 environ['HTTP_X_RC_REPO_CONFIG'])
446 481 config = msgpack.unpackb(packed_config)
447 482 app = scm_app.create_hg_wsgi_app(
448 483 repo_path, repo_name, config)
449 484
450 485 # Consistent path information for hgweb
451 486 environ['PATH_INFO'] = environ['HTTP_X_RC_PATH_INFO']
452 487 environ['REPO_NAME'] = repo_name
453 488 self.set_env_from_config(environ, config)
454 489
455 490 log.debug('http-app: starting app handler '
456 491 'with %s and process request', app)
457 492 return app(environ, ResponseFilter(start_response))
458 493 return _hg_stream
459 494
460 495 def git_stream(self):
461 496 if self._use_echo_app:
462 497 @wsgiapp
463 498 def _git_stream(environ, start_response):
464 499 app = EchoApp('fake_path', 'fake_name', None)
465 500 return app(environ, start_response)
466 501 return _git_stream
467 502 else:
468 503 @wsgiapp
469 504 def _git_stream(environ, start_response):
470 505 log.debug('http-app: handling git stream')
471 506 repo_path = environ['HTTP_X_RC_REPO_PATH']
472 507 repo_name = environ['HTTP_X_RC_REPO_NAME']
473 508 packed_config = base64.b64decode(
474 509 environ['HTTP_X_RC_REPO_CONFIG'])
475 510 config = msgpack.unpackb(packed_config)
476 511
477 512 environ['PATH_INFO'] = environ['HTTP_X_RC_PATH_INFO']
478 513 self.set_env_from_config(environ, config)
479 514
480 515 content_type = environ.get('CONTENT_TYPE', '')
481 516
482 517 path = environ['PATH_INFO']
483 518 is_lfs_request = GIT_LFS_CONTENT_TYPE in content_type
484 519 log.debug(
485 520 'LFS: Detecting if request `%s` is LFS server path based '
486 521 'on content type:`%s`, is_lfs:%s',
487 522 path, content_type, is_lfs_request)
488 523
489 524 if not is_lfs_request:
490 525 # fallback detection by path
491 526 if GIT_LFS_PROTO_PAT.match(path):
492 527 is_lfs_request = True
493 528 log.debug(
494 529 'LFS: fallback detection by path of: `%s`, is_lfs:%s',
495 530 path, is_lfs_request)
496 531
497 532 if is_lfs_request:
498 533 app = scm_app.create_git_lfs_wsgi_app(
499 534 repo_path, repo_name, config)
500 535 else:
501 536 app = scm_app.create_git_wsgi_app(
502 537 repo_path, repo_name, config)
503 538
504 539 log.debug('http-app: starting app handler '
505 540 'with %s and process request', app)
506 541
507 542 return app(environ, start_response)
508 543
509 544 return _git_stream
510 545
511 546 def is_vcs_view(self, context, request):
512 547 """
513 548 View predicate that returns true if given backend is supported by
514 549 defined remotes.
515 550 """
516 551 backend = request.matchdict.get('backend')
517 552 return backend in self._remotes
518 553
519 554 def handle_vcs_exception(self, exception, request):
520 555 _vcs_kind = getattr(exception, '_vcs_kind', '')
521 556 if _vcs_kind == 'repo_locked':
522 557 # Get custom repo-locked status code if present.
523 558 status_code = request.headers.get('X-RC-Locked-Status-Code')
524 559 return HTTPRepoLocked(
525 560 title=exception.message, status_code=status_code)
526 561
527 562 elif _vcs_kind == 'repo_branch_protected':
528 563 # Get custom repo-branch-protected status code if present.
529 564 return HTTPRepoBranchProtected(title=exception.message)
530 565
531 566 exc_info = request.exc_info
532 567 store_exception(id(exc_info), exc_info)
533 568
534 569 traceback_info = 'unavailable'
535 570 if request.exc_info:
536 571 exc_type, exc_value, exc_tb = request.exc_info
537 572 traceback_info = ''.join(traceback.format_exception(exc_type, exc_value, exc_tb))
538 573
539 574 log.error(
540 575 'error occurred handling this request for path: %s, \n tb: %s',
541 576 request.path, traceback_info)
542 577 raise exception
543 578
544 579
545 580 class ResponseFilter(object):
546 581
547 582 def __init__(self, start_response):
548 583 self._start_response = start_response
549 584
550 585 def __call__(self, status, response_headers, exc_info=None):
551 586 headers = tuple(
552 587 (h, v) for h, v in response_headers
553 588 if not wsgiref.util.is_hop_by_hop(h))
554 589 return self._start_response(status, headers, exc_info)
555 590
556 591
557 592 def main(global_config, **settings):
558 593 if MercurialFactory:
559 594 hgpatches.patch_largefiles_capabilities()
560 595 hgpatches.patch_subrepo_type_mapping()
561 596
562 597 app = HTTPApplication(settings=settings, global_config=global_config)
563 598 return app.wsgi_app()
@@ -1,146 +1,151 b''
1 1 # -*- coding: utf-8 -*-
2 2
3 3 # RhodeCode VCSServer provides access to different vcs backends via network.
4 4 # Copyright (C) 2014-2018 RhodeCode GmbH
5 5 #
6 6 # This program is free software; you can redistribute it and/or modify
7 7 # it under the terms of the GNU General Public License as published by
8 8 # the Free Software Foundation; either version 3 of the License, or
9 9 # (at your option) any later version.
10 10 #
11 11 # This program is distributed in the hope that it will be useful,
12 12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
13 13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 14 # GNU General Public License for more details.
15 15 #
16 16 # You should have received a copy of the GNU General Public License
17 17 # along with this program; if not, write to the Free Software Foundation,
18 18 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19 19
20 20
21 21 import os
22 22 import time
23 23 import datetime
24 24 import msgpack
25 25 import logging
26 26 import traceback
27 27 import tempfile
28 28
29 29
30 30 log = logging.getLogger(__name__)
31 31
32 32 # NOTE: Any changes should be synced with exc_tracking at rhodecode.lib.exc_tracking
33 33 global_prefix = 'vcsserver'
34 exc_store_dir_name = 'rc_exception_store_v1'
34 35
35 36
36 37 def exc_serialize(exc_id, tb, exc_type):
37 38
38 39 data = {
39 40 'version': 'v1',
40 41 'exc_id': exc_id,
41 42 'exc_utc_date': datetime.datetime.utcnow().isoformat(),
42 43 'exc_timestamp': repr(time.time()),
43 44 'exc_message': tb,
44 45 'exc_type': exc_type,
45 46 }
46 47 return msgpack.packb(data), data
47 48
48 49
49 50 def exc_unserialize(tb):
50 51 return msgpack.unpackb(tb)
51 52
52 53
53 54 def get_exc_store():
54 55 """
55 56 Get and create exception store if it's not existing
56 57 """
57 exc_store_dir = 'rc_exception_store_v1'
58 # fallback
59 _exc_store_path = os.path.join(tempfile.gettempdir(), exc_store_dir)
58 import vcsserver as app
60 59
61 exc_store_dir = '' # TODO: need a persistent cross instance store here
62 if exc_store_dir:
63 _exc_store_path = os.path.join(exc_store_dir, exc_store_dir)
60 exc_store_dir = app.CONFIG.get('exception_store_path', '') or tempfile.gettempdir()
61 _exc_store_path = os.path.join(exc_store_dir, exc_store_dir_name)
64 62
65 63 _exc_store_path = os.path.abspath(_exc_store_path)
66 64 if not os.path.isdir(_exc_store_path):
67 65 os.makedirs(_exc_store_path)
68 66 log.debug('Initializing exceptions store at %s', _exc_store_path)
69 67 return _exc_store_path
70 68
71 69
72 70 def _store_exception(exc_id, exc_info, prefix):
73 71 exc_type, exc_value, exc_traceback = exc_info
74 72 tb = ''.join(traceback.format_exception(
75 73 exc_type, exc_value, exc_traceback, None))
76 74
77 75 exc_type_name = exc_type.__name__
78 76 exc_store_path = get_exc_store()
79 77 exc_data, org_data = exc_serialize(exc_id, tb, exc_type_name)
80 78 exc_pref_id = '{}_{}_{}'.format(exc_id, prefix, org_data['exc_timestamp'])
81 79 if not os.path.isdir(exc_store_path):
82 80 os.makedirs(exc_store_path)
83 81 stored_exc_path = os.path.join(exc_store_path, exc_pref_id)
84 82 with open(stored_exc_path, 'wb') as f:
85 83 f.write(exc_data)
86 84 log.debug('Stored generated exception %s as: %s', exc_id, stored_exc_path)
87 85
88 86
89 87 def store_exception(exc_id, exc_info, prefix=global_prefix):
88 """
89 Example usage::
90
91 exc_info = sys.exc_info()
92 store_exception(id(exc_info), exc_info)
93 """
94
90 95 try:
91 96 _store_exception(exc_id=exc_id, exc_info=exc_info, prefix=prefix)
92 97 except Exception:
93 98 log.exception('Failed to store exception `%s` information', exc_id)
94 99 # there's no way this can fail, it will crash server badly if it does.
95 100 pass
96 101
97 102
98 103 def _find_exc_file(exc_id, prefix=global_prefix):
99 104 exc_store_path = get_exc_store()
100 105 if prefix:
101 106 exc_id = '{}_{}'.format(exc_id, prefix)
102 107 else:
103 108 # search without a prefix
104 109 exc_id = '{}'.format(exc_id)
105 110
106 111 # we need to search the store for such start pattern as above
107 112 for fname in os.listdir(exc_store_path):
108 113 if fname.startswith(exc_id):
109 114 exc_id = os.path.join(exc_store_path, fname)
110 115 break
111 116 continue
112 117 else:
113 118 exc_id = None
114 119
115 120 return exc_id
116 121
117 122
118 123 def _read_exception(exc_id, prefix):
119 124 exc_id_file_path = _find_exc_file(exc_id=exc_id, prefix=prefix)
120 125 if exc_id_file_path:
121 126 with open(exc_id_file_path, 'rb') as f:
122 127 return exc_unserialize(f.read())
123 128 else:
124 129 log.debug('Exception File `%s` not found', exc_id_file_path)
125 130 return None
126 131
127 132
128 133 def read_exception(exc_id, prefix=global_prefix):
129 134 try:
130 135 return _read_exception(exc_id=exc_id, prefix=prefix)
131 136 except Exception:
132 137 log.exception('Failed to read exception `%s` information', exc_id)
133 138 # there's no way this can fail, it will crash server badly if it does.
134 139 return None
135 140
136 141
137 142 def delete_exception(exc_id, prefix=global_prefix):
138 143 try:
139 144 exc_id_file_path = _find_exc_file(exc_id, prefix=prefix)
140 145 if exc_id_file_path:
141 146 os.remove(exc_id_file_path)
142 147
143 148 except Exception:
144 149 log.exception('Failed to remove exception `%s` information', exc_id)
145 150 # there's no way this can fail, it will crash server badly if it does.
146 151 pass
@@ -1,57 +1,57 b''
1 1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 2 # Copyright (C) 2014-2018 RhodeCode GmbH
3 3 #
4 4 # This program is free software; you can redistribute it and/or modify
5 5 # it under the terms of the GNU General Public License as published by
6 6 # the Free Software Foundation; either version 3 of the License, or
7 7 # (at your option) any later version.
8 8 #
9 9 # This program is distributed in the hope that it will be useful,
10 10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 12 # GNU General Public License for more details.
13 13 #
14 14 # You should have received a copy of the GNU General Public License
15 15 # along with this program; if not, write to the Free Software Foundation,
16 16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17 17
18 18 import mock
19 19 import pytest
20 20
21 21 from vcsserver import http_main
22 22 from vcsserver.base import obfuscate_qs
23 23
24 24
25 25 @mock.patch('vcsserver.http_main.VCS', mock.Mock())
26 26 @mock.patch('vcsserver.hgpatches.patch_largefiles_capabilities')
27 27 def test_applies_largefiles_patch(patch_largefiles_capabilities):
28 http_main.main([])
28 http_main.main({})
29 29 patch_largefiles_capabilities.assert_called_once_with()
30 30
31 31
32 32 @mock.patch('vcsserver.http_main.VCS', mock.Mock())
33 33 @mock.patch('vcsserver.http_main.MercurialFactory', None)
34 34 @mock.patch(
35 35 'vcsserver.hgpatches.patch_largefiles_capabilities',
36 36 mock.Mock(side_effect=Exception("Must not be called")))
37 37 def test_applies_largefiles_patch_only_if_mercurial_is_available():
38 http_main.main([])
38 http_main.main({})
39 39
40 40
41 41 @pytest.mark.parametrize('given, expected', [
42 42 ('bad', 'bad'),
43 43 ('query&foo=bar', 'query&foo=bar'),
44 44 ('equery&auth_token=bar', 'equery&auth_token=*****'),
45 45 ('a;b;c;query&foo=bar&auth_token=secret',
46 46 'a&b&c&query&foo=bar&auth_token=*****'),
47 47 ('', ''),
48 48 (None, None),
49 49 ('foo=bar', 'foo=bar'),
50 50 ('auth_token=secret', 'auth_token=*****'),
51 51 ('auth_token=secret&api_key=secret2',
52 52 'auth_token=*****&api_key=*****'),
53 53 ('auth_token=secret&api_key=secret2&param=value',
54 54 'auth_token=*****&api_key=*****&param=value'),
55 55 ])
56 56 def test_obfuscate_qs(given, expected):
57 57 assert expected == obfuscate_qs(given)
General Comments 0
You need to be logged in to leave comments. Login now