##// END OF EJS Templates
http-app: refactor new usage of main http api endpoints
super-admin -
r1123:f468f658 python3
parent child Browse files
Show More
@@ -1,773 +1,778 b''
1 1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 2 # Copyright (C) 2014-2020 RhodeCode GmbH
3 3 #
4 4 # This program is free software; you can redistribute it and/or modify
5 5 # it under the terms of the GNU General Public License as published by
6 6 # the Free Software Foundation; either version 3 of the License, or
7 7 # (at your option) any later version.
8 8 #
9 9 # This program is distributed in the hope that it will be useful,
10 10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 12 # GNU General Public License for more details.
13 13 #
14 14 # You should have received a copy of the GNU General Public License
15 15 # along with this program; if not, write to the Free Software Foundation,
16 16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17 17
18 18 import io
19 19 import os
20 20 import sys
21 import base64
22 21 import locale
23 22 import logging
24 23 import uuid
25 24 import time
26 25 import wsgiref.util
27 26 import traceback
28 27 import tempfile
29 28 import psutil
30 29
31 30 from itertools import chain
32 31
33 32 import msgpack
34 33 import configparser
35 34
36 35 from pyramid.config import Configurator
37 36 from pyramid.wsgi import wsgiapp
38 37 from pyramid.response import Response
39 38
40 from vcsserver.base import BinaryEnvelope
39 from vcsserver.base import BytesEnvelope, BinaryEnvelope
41 40 from vcsserver.lib.rc_json import json
42 41 from vcsserver.config.settings_maker import SettingsMaker
43 from vcsserver.str_utils import safe_int, safe_bytes, safe_str
42 from vcsserver.str_utils import safe_int
44 43 from vcsserver.lib.statsd_client import StatsdClient
44 from vcsserver.tweens.request_wrapper import get_call_context, get_headers_call_context
45 45
46 46 log = logging.getLogger(__name__)
47 47
48 48 # due to Mercurial/glibc2.27 problems we need to detect if locale settings are
49 49 # causing problems and "fix" it in case they do and fallback to LC_ALL = C
50 50
51 51 try:
52 52 locale.setlocale(locale.LC_ALL, '')
53 53 except locale.Error as e:
54 54 log.error(
55 55 'LOCALE ERROR: failed to set LC_ALL, fallback to LC_ALL=C, org error: %s', e)
56 56 os.environ['LC_ALL'] = 'C'
57 57
58 58
59 59 import vcsserver
60 60 from vcsserver import remote_wsgi, scm_app, settings, hgpatches
61 61 from vcsserver.git_lfs.app import GIT_LFS_CONTENT_TYPE, GIT_LFS_PROTO_PAT
62 62 from vcsserver.echo_stub import remote_wsgi as remote_wsgi_stub
63 63 from vcsserver.echo_stub.echo_app import EchoApp
64 64 from vcsserver.exceptions import HTTPRepoLocked, HTTPRepoBranchProtected
65 65 from vcsserver.lib.exc_tracking import store_exception
66 66 from vcsserver.server import VcsServer
67 67
68 68 strict_vcs = True
69 69
70 70 git_import_err = None
71 71 try:
72 72 from vcsserver.remote.git import GitFactory, GitRemote
73 73 except ImportError as e:
74 74 GitFactory = None
75 75 GitRemote = None
76 76 git_import_err = e
77 77 if strict_vcs:
78 78 raise
79 79
80 80
81 81 hg_import_err = None
82 82 try:
83 83 from vcsserver.remote.hg import MercurialFactory, HgRemote
84 84 except ImportError as e:
85 85 MercurialFactory = None
86 86 HgRemote = None
87 87 hg_import_err = e
88 88 if strict_vcs:
89 89 raise
90 90
91 91
92 92 svn_import_err = None
93 93 try:
94 94 from vcsserver.remote.svn import SubversionFactory, SvnRemote
95 95 except ImportError as e:
96 96 SubversionFactory = None
97 97 SvnRemote = None
98 98 svn_import_err = e
99 99 if strict_vcs:
100 100 raise
101 101
102 102
103 103 def _is_request_chunked(environ):
104 104 stream = environ.get('HTTP_TRANSFER_ENCODING', '') == 'chunked'
105 105 return stream
106 106
107 107
108 108 def log_max_fd():
109 109 try:
110 110 maxfd = psutil.Process().rlimit(psutil.RLIMIT_NOFILE)[1]
111 111 log.info('Max file descriptors value: %s', maxfd)
112 112 except Exception:
113 113 pass
114 114
115 115
116 116 class VCS(object):
117 117 def __init__(self, locale_conf=None, cache_config=None):
118 118 self.locale = locale_conf
119 119 self.cache_config = cache_config
120 120 self._configure_locale()
121 121
122 122 log_max_fd()
123 123
124 124 if GitFactory and GitRemote:
125 125 git_factory = GitFactory()
126 126 self._git_remote = GitRemote(git_factory)
127 127 else:
128 128 log.error("Git client import failed: %s", git_import_err)
129 129
130 130 if MercurialFactory and HgRemote:
131 131 hg_factory = MercurialFactory()
132 132 self._hg_remote = HgRemote(hg_factory)
133 133 else:
134 134 log.error("Mercurial client import failed: %s", hg_import_err)
135 135
136 136 if SubversionFactory and SvnRemote:
137 137 svn_factory = SubversionFactory()
138 138
139 139 # hg factory is used for svn url validation
140 140 hg_factory = MercurialFactory()
141 141 self._svn_remote = SvnRemote(svn_factory, hg_factory=hg_factory)
142 142 else:
143 143 log.error("Subversion client import failed: %s", svn_import_err)
144 144
145 145 self._vcsserver = VcsServer()
146 146
147 147 def _configure_locale(self):
148 148 if self.locale:
149 149 log.info('Settings locale: `LC_ALL` to %s', self.locale)
150 150 else:
151 151 log.info('Configuring locale subsystem based on environment variables')
152 152 try:
153 153 # If self.locale is the empty string, then the locale
154 154 # module will use the environment variables. See the
155 155 # documentation of the package `locale`.
156 156 locale.setlocale(locale.LC_ALL, self.locale)
157 157
158 158 language_code, encoding = locale.getlocale()
159 159 log.info(
160 160 'Locale set to language code "%s" with encoding "%s".',
161 161 language_code, encoding)
162 162 except locale.Error:
163 163 log.exception('Cannot set locale, not configuring the locale system')
164 164
165 165
166 166 class WsgiProxy(object):
167 167 def __init__(self, wsgi):
168 168 self.wsgi = wsgi
169 169
170 170 def __call__(self, environ, start_response):
171 171 input_data = environ['wsgi.input'].read()
172 172 input_data = msgpack.unpackb(input_data)
173 173
174 174 error = None
175 175 try:
176 176 data, status, headers = self.wsgi.handle(
177 177 input_data['environment'], input_data['input_data'],
178 178 *input_data['args'], **input_data['kwargs'])
179 179 except Exception as e:
180 180 data, status, headers = [], None, None
181 181 error = {
182 182 'message': str(e),
183 183 '_vcs_kind': getattr(e, '_vcs_kind', None)
184 184 }
185 185
186 186 start_response(200, {})
187 187 return self._iterator(error, status, headers, data)
188 188
189 189 def _iterator(self, error, status, headers, data):
190 190 initial_data = [
191 191 error,
192 192 status,
193 193 headers,
194 194 ]
195 195
196 196 for d in chain(initial_data, data):
197 197 yield msgpack.packb(d)
198 198
199 199
200 200 def not_found(request):
201 201 return {'status': '404 NOT FOUND'}
202 202
203 203
204 204 class VCSViewPredicate(object):
205 205 def __init__(self, val, config):
206 206 self.remotes = val
207 207
208 208 def text(self):
209 209 return f'vcs view method = {list(self.remotes.keys())}'
210 210
211 211 phash = text
212 212
213 213 def __call__(self, context, request):
214 214 """
215 215 View predicate that returns true if given backend is supported by
216 216 defined remotes.
217 217 """
218 218 backend = request.matchdict.get('backend')
219 219 return backend in self.remotes
220 220
221 221
222 222 class HTTPApplication(object):
223 223 ALLOWED_EXCEPTIONS = ('KeyError', 'URLError')
224 224
225 225 remote_wsgi = remote_wsgi
226 226 _use_echo_app = False
227 227
228 228 def __init__(self, settings=None, global_config=None):
229 229
230 230 self.config = Configurator(settings=settings)
231 231 # Init our statsd at very start
232 232 self.config.registry.statsd = StatsdClient.statsd
233 233 self.config.registry.vcs_call_context = {}
234 234
235 235 self.global_config = global_config
236 236 self.config.include('vcsserver.lib.rc_cache')
237 self.config.include('vcsserver.lib.rc_cache.archive_cache')
237 238
238 239 settings_locale = settings.get('locale', '') or 'en_US.UTF-8'
239 240 vcs = VCS(locale_conf=settings_locale, cache_config=settings)
240 241 self._remotes = {
241 242 'hg': vcs._hg_remote,
242 243 'git': vcs._git_remote,
243 244 'svn': vcs._svn_remote,
244 245 'server': vcs._vcsserver,
245 246 }
246 247 if settings.get('dev.use_echo_app', 'false').lower() == 'true':
247 248 self._use_echo_app = True
248 249 log.warning("Using EchoApp for VCS operations.")
249 250 self.remote_wsgi = remote_wsgi_stub
250 251
251 252 self._configure_settings(global_config, settings)
252 253
253 254 self._configure()
254 255
255 256 def _configure_settings(self, global_config, app_settings):
256 257 """
257 258 Configure the settings module.
258 259 """
259 260 settings_merged = global_config.copy()
260 261 settings_merged.update(app_settings)
261 262
262 263 git_path = app_settings.get('git_path', None)
263 264 if git_path:
264 265 settings.GIT_EXECUTABLE = git_path
265 266 binary_dir = app_settings.get('core.binary_dir', None)
266 267 if binary_dir:
267 268 settings.BINARY_DIR = binary_dir
268 269
269 270 # Store the settings to make them available to other modules.
270 271 vcsserver.PYRAMID_SETTINGS = settings_merged
271 272 vcsserver.CONFIG = settings_merged
272 273
273 274 def _configure(self):
274 275 self.config.add_renderer(name='msgpack', factory=self._msgpack_renderer_factory)
275 276
276 277 self.config.add_route('service', '/_service')
277 278 self.config.add_route('status', '/status')
278 279 self.config.add_route('hg_proxy', '/proxy/hg')
279 280 self.config.add_route('git_proxy', '/proxy/git')
280 281
281 282 # rpc methods
282 283 self.config.add_route('vcs', '/{backend}')
283 284
284 285 # streaming rpc remote methods
285 286 self.config.add_route('vcs_stream', '/{backend}/stream')
286 287
287 288 # vcs operations clone/push as streaming
288 289 self.config.add_route('stream_git', '/stream/git/*repo_name')
289 290 self.config.add_route('stream_hg', '/stream/hg/*repo_name')
290 291
291 292 self.config.add_view(self.status_view, route_name='status', renderer='json')
292 293 self.config.add_view(self.service_view, route_name='service', renderer='msgpack')
293 294
294 295 self.config.add_view(self.hg_proxy(), route_name='hg_proxy')
295 296 self.config.add_view(self.git_proxy(), route_name='git_proxy')
296 297 self.config.add_view(self.vcs_view, route_name='vcs', renderer='msgpack',
297 298 vcs_view=self._remotes)
298 299 self.config.add_view(self.vcs_stream_view, route_name='vcs_stream',
299 300 vcs_view=self._remotes)
300 301
301 302 self.config.add_view(self.hg_stream(), route_name='stream_hg')
302 303 self.config.add_view(self.git_stream(), route_name='stream_git')
303 304
304 305 self.config.add_view_predicate('vcs_view', VCSViewPredicate)
305 306
306 307 self.config.add_notfound_view(not_found, renderer='json')
307 308
308 309 self.config.add_view(self.handle_vcs_exception, context=Exception)
309 310
310 311 self.config.add_tween(
311 312 'vcsserver.tweens.request_wrapper.RequestWrapperTween',
312 313 )
313 314 self.config.add_request_method(
314 315 'vcsserver.lib.request_counter.get_request_counter',
315 316 'request_count')
316 317
317 318 def wsgi_app(self):
318 319 return self.config.make_wsgi_app()
319 320
320 321 def _vcs_view_params(self, request):
321 322 remote = self._remotes[request.matchdict['backend']]
322 323 payload = msgpack.unpackb(request.body, use_list=True)
323 324
324 325 method = payload.get('method')
325 326 params = payload['params']
326 327 wire = params.get('wire')
327 328 args = params.get('args')
328 329 kwargs = params.get('kwargs')
329 330 context_uid = None
330 331
331 332 request.registry.vcs_call_context = {
332 333 'method': method,
333 'repo_name': payload.get('_repo_name')
334 'repo_name': payload.get('_repo_name'),
334 335 }
335 336
336 337 if wire:
337 338 try:
338 339 wire['context'] = context_uid = uuid.UUID(wire['context'])
339 340 except KeyError:
340 341 pass
341 342 args.insert(0, wire)
342 343 repo_state_uid = wire.get('repo_state_uid') if wire else None
343 344
344 345 # NOTE(marcink): trading complexity for slight performance
345 346 if log.isEnabledFor(logging.DEBUG):
346 347 # also we SKIP printing out any of those methods args since they maybe excessive
347 348 just_args_methods = {
348 'commitctx': ('content', 'removed', 'updated')
349 'commitctx': ('content', 'removed', 'updated'),
350 'commit': ('content', 'removed', 'updated')
349 351 }
350 352 if method in just_args_methods:
351 353 skip_args = just_args_methods[method]
352 354 call_args = ''
353 355 call_kwargs = {}
354 356 for k in kwargs:
355 357 if k in skip_args:
356 358 # replace our skip key with dummy
357 359 call_kwargs[k] = f'RemovedParam({k})'
358 360 else:
359 361 call_kwargs[k] = kwargs[k]
360 362 else:
361 363 call_args = args[1:]
362 364 call_kwargs = kwargs
363 365
364 366 log.debug('Method requested:`%s` with args:%s kwargs:%s context_uid: %s, repo_state_uid:%s',
365 367 method, call_args, call_kwargs, context_uid, repo_state_uid)
366 368
367 369 statsd = request.registry.statsd
368 370 if statsd:
369 371 statsd.incr(
370 372 'vcsserver_method_total', tags=[
371 373 f"method:{method}",
372 374 ])
373 375 return payload, remote, method, args, kwargs
374 376
375 377 def vcs_view(self, request):
376 378
377 379 payload, remote, method, args, kwargs = self._vcs_view_params(request)
378 380 payload_id = payload.get('id')
379 381
380 382 try:
381 383 resp = getattr(remote, method)(*args, **kwargs)
382 384 except Exception as e:
383 385 exc_info = list(sys.exc_info())
384 386 exc_type, exc_value, exc_traceback = exc_info
385 387
386 388 org_exc = getattr(e, '_org_exc', None)
387 389 org_exc_name = None
388 390 org_exc_tb = ''
389 391 if org_exc:
390 392 org_exc_name = org_exc.__class__.__name__
391 393 org_exc_tb = getattr(e, '_org_exc_tb', '')
392 394 # replace our "faked" exception with our org
393 395 exc_info[0] = org_exc.__class__
394 396 exc_info[1] = org_exc
395 397
396 398 should_store_exc = True
397 399 if org_exc:
398 400 def get_exc_fqn(_exc_obj):
399 401 module_name = getattr(org_exc.__class__, '__module__', 'UNKNOWN')
400 402 return module_name + '.' + org_exc_name
401 403
402 404 exc_fqn = get_exc_fqn(org_exc)
403 405
404 406 if exc_fqn in ['mercurial.error.RepoLookupError',
405 407 'vcsserver.exceptions.RefNotFoundException']:
406 408 should_store_exc = False
407 409
408 410 if should_store_exc:
409 411 store_exception(id(exc_info), exc_info, request_path=request.path)
410 412
411 413 tb_info = ''.join(
412 414 traceback.format_exception(exc_type, exc_value, exc_traceback))
413 415
414 416 type_ = e.__class__.__name__
415 417 if type_ not in self.ALLOWED_EXCEPTIONS:
416 418 type_ = None
417 419
418 420 resp = {
419 421 'id': payload_id,
420 422 'error': {
421 423 'message': str(e),
422 424 'traceback': tb_info,
423 425 'org_exc': org_exc_name,
424 426 'org_exc_tb': org_exc_tb,
425 427 'type': type_
426 428 }
427 429 }
428 430
429 431 try:
430 432 resp['error']['_vcs_kind'] = getattr(e, '_vcs_kind', None)
431 433 except AttributeError:
432 434 pass
433 435 else:
434 436 resp = {
435 437 'id': payload_id,
436 438 'result': resp
437 439 }
438 440 log.debug('Serving data for method %s', method)
439 441 return resp
440 442
441 443 def vcs_stream_view(self, request):
442 444 payload, remote, method, args, kwargs = self._vcs_view_params(request)
443 445 # this method has a stream: marker we remove it here
444 446 method = method.split('stream:')[-1]
445 447 chunk_size = safe_int(payload.get('chunk_size')) or 4096
446 448
447 449 try:
448 450 resp = getattr(remote, method)(*args, **kwargs)
449 451 except Exception as e:
450 452 raise
451 453
452 454 def get_chunked_data(method_resp):
453 455 stream = io.BytesIO(method_resp)
454 456 while 1:
455 457 chunk = stream.read(chunk_size)
456 458 if not chunk:
457 459 break
458 460 yield chunk
459 461
460 462 response = Response(app_iter=get_chunked_data(resp))
461 463 response.content_type = 'application/octet-stream'
462 464
463 465 return response
464 466
465 467 def status_view(self, request):
466 468 import vcsserver
467 469 return {'status': 'OK', 'vcsserver_version': vcsserver.__version__,
468 470 'pid': os.getpid()}
469 471
470 472 def service_view(self, request):
471 473 import vcsserver
472 474
473 475 payload = msgpack.unpackb(request.body, use_list=True)
474 476 server_config, app_config = {}, {}
475 477
476 478 try:
477 479 path = self.global_config['__file__']
478 480 config = configparser.RawConfigParser()
479 481
480 482 config.read(path)
481 483
482 484 if config.has_section('server:main'):
483 485 server_config = dict(config.items('server:main'))
484 486 if config.has_section('app:main'):
485 487 app_config = dict(config.items('app:main'))
486 488
487 489 except Exception:
488 490 log.exception('Failed to read .ini file for display')
489 491
490 492 environ = list(os.environ.items())
491 493
492 494 resp = {
493 495 'id': payload.get('id'),
494 496 'result': dict(
495 497 version=vcsserver.__version__,
496 498 config=server_config,
497 499 app_config=app_config,
498 500 environ=environ,
499 501 payload=payload,
500 502 )
501 503 }
502 504 return resp
503 505
504 506 def _msgpack_renderer_factory(self, info):
505 507
506 508 def _render(value, system):
507 509 bin_type = False
508 510 res = value.get('result')
509 if res and isinstance(res, BinaryEnvelope):
511 if isinstance(res, BytesEnvelope):
512 log.debug('Result is wrapped in BytesEnvelope type')
513 bin_type = True
514 elif isinstance(res, BinaryEnvelope):
510 515 log.debug('Result is wrapped in BinaryEnvelope type')
511 value['result'] = res.value
512 bin_type = res.bin_type
516 value['result'] = res.val
517 bin_type = True
513 518
514 519 request = system.get('request')
515 520 if request is not None:
516 521 response = request.response
517 522 ct = response.content_type
518 523 if ct == response.default_content_type:
519 524 response.content_type = 'application/x-msgpack'
520 525 if bin_type:
521 526 response.content_type = 'application/x-msgpack-bin'
522 527
523 528 return msgpack.packb(value, use_bin_type=bin_type)
524 529 return _render
525 530
526 531 def set_env_from_config(self, environ, config):
527 532 dict_conf = {}
528 533 try:
529 534 for elem in config:
530 535 if elem[0] == 'rhodecode':
531 536 dict_conf = json.loads(elem[2])
532 537 break
533 538 except Exception:
534 539 log.exception('Failed to fetch SCM CONFIG')
535 540 return
536 541
537 542 username = dict_conf.get('username')
538 543 if username:
539 544 environ['REMOTE_USER'] = username
540 545 # mercurial specific, some extension api rely on this
541 546 environ['HGUSER'] = username
542 547
543 548 ip = dict_conf.get('ip')
544 549 if ip:
545 550 environ['REMOTE_HOST'] = ip
546 551
547 552 if _is_request_chunked(environ):
548 553 # set the compatibility flag for webob
549 554 environ['wsgi.input_terminated'] = True
550 555
551 556 def hg_proxy(self):
552 557 @wsgiapp
553 558 def _hg_proxy(environ, start_response):
554 559 app = WsgiProxy(self.remote_wsgi.HgRemoteWsgi())
555 560 return app(environ, start_response)
556 561 return _hg_proxy
557 562
558 563 def git_proxy(self):
559 564 @wsgiapp
560 565 def _git_proxy(environ, start_response):
561 566 app = WsgiProxy(self.remote_wsgi.GitRemoteWsgi())
562 567 return app(environ, start_response)
563 568 return _git_proxy
564 569
565 570 def hg_stream(self):
566 571 if self._use_echo_app:
567 572 @wsgiapp
568 573 def _hg_stream(environ, start_response):
569 574 app = EchoApp('fake_path', 'fake_name', None)
570 575 return app(environ, start_response)
571 576 return _hg_stream
572 577 else:
573 578 @wsgiapp
574 579 def _hg_stream(environ, start_response):
575 580 log.debug('http-app: handling hg stream')
576
577 packed_cc = base64.b64decode(environ['HTTP_X_RC_VCS_STREAM_CALL_CONTEXT'])
578 call_context = msgpack.unpackb(packed_cc)
581 call_context = get_headers_call_context(environ)
579 582
580 583 repo_path = call_context['repo_path']
581 584 repo_name = call_context['repo_name']
582 585 config = call_context['repo_config']
583 586
584 587 app = scm_app.create_hg_wsgi_app(
585 588 repo_path, repo_name, config)
586 589
587 590 # Consistent path information for hgweb
588 591 environ['PATH_INFO'] = call_context['path_info']
589 592 environ['REPO_NAME'] = repo_name
590 593 self.set_env_from_config(environ, config)
591 594
592 595 log.debug('http-app: starting app handler '
593 596 'with %s and process request', app)
594 597 return app(environ, ResponseFilter(start_response))
595 598 return _hg_stream
596 599
597 600 def git_stream(self):
598 601 if self._use_echo_app:
599 602 @wsgiapp
600 603 def _git_stream(environ, start_response):
601 604 app = EchoApp('fake_path', 'fake_name', None)
602 605 return app(environ, start_response)
603 606 return _git_stream
604 607 else:
605 608 @wsgiapp
606 609 def _git_stream(environ, start_response):
607 610 log.debug('http-app: handling git stream')
608 611
609 packed_cc = base64.b64decode(environ['HTTP_X_RC_VCS_STREAM_CALL_CONTEXT'])
610 call_context = msgpack.unpackb(packed_cc)
612 call_context = get_headers_call_context(environ)
611 613
612 614 repo_path = call_context['repo_path']
613 615 repo_name = call_context['repo_name']
614 616 config = call_context['repo_config']
615 617
616 618 environ['PATH_INFO'] = call_context['path_info']
617 619 self.set_env_from_config(environ, config)
618 620
619 621 content_type = environ.get('CONTENT_TYPE', '')
620 622
621 623 path = environ['PATH_INFO']
622 624 is_lfs_request = GIT_LFS_CONTENT_TYPE in content_type
623 625 log.debug(
624 626 'LFS: Detecting if request `%s` is LFS server path based '
625 627 'on content type:`%s`, is_lfs:%s',
626 628 path, content_type, is_lfs_request)
627 629
628 630 if not is_lfs_request:
629 631 # fallback detection by path
630 632 if GIT_LFS_PROTO_PAT.match(path):
631 633 is_lfs_request = True
632 634 log.debug(
633 635 'LFS: fallback detection by path of: `%s`, is_lfs:%s',
634 636 path, is_lfs_request)
635 637
636 638 if is_lfs_request:
637 639 app = scm_app.create_git_lfs_wsgi_app(
638 640 repo_path, repo_name, config)
639 641 else:
640 642 app = scm_app.create_git_wsgi_app(
641 643 repo_path, repo_name, config)
642 644
643 645 log.debug('http-app: starting app handler '
644 646 'with %s and process request', app)
645 647
646 648 return app(environ, start_response)
647 649
648 650 return _git_stream
649 651
650 652 def handle_vcs_exception(self, exception, request):
651 653 _vcs_kind = getattr(exception, '_vcs_kind', '')
654
652 655 if _vcs_kind == 'repo_locked':
653 # Get custom repo-locked status code if present.
654 status_code = request.headers.get('X-RC-Locked-Status-Code')
656 headers_call_context = get_headers_call_context(request.environ)
657 status_code = safe_int(headers_call_context['locked_status_code'])
658
655 659 return HTTPRepoLocked(
656 title=str(exception), status_code=status_code)
660 title=str(exception), status_code=status_code, headers=[('X-Rc-Locked', '1')])
657 661
658 662 elif _vcs_kind == 'repo_branch_protected':
659 663 # Get custom repo-branch-protected status code if present.
660 return HTTPRepoBranchProtected(title=str(exception))
664 return HTTPRepoBranchProtected(
665 title=str(exception), headers=[('X-Rc-Branch-Protection', '1')])
661 666
662 667 exc_info = request.exc_info
663 668 store_exception(id(exc_info), exc_info)
664 669
665 670 traceback_info = 'unavailable'
666 671 if request.exc_info:
667 672 exc_type, exc_value, exc_tb = request.exc_info
668 673 traceback_info = ''.join(traceback.format_exception(exc_type, exc_value, exc_tb))
669 674
670 675 log.error(
671 676 'error occurred handling this request for path: %s, \n tb: %s',
672 677 request.path, traceback_info)
673 678
674 679 statsd = request.registry.statsd
675 680 if statsd:
676 681 exc_type = f"{exception.__class__.__module__}.{exception.__class__.__name__}"
677 682 statsd.incr('vcsserver_exception_total',
678 683 tags=[f"type:{exc_type}"])
679 684 raise exception
680 685
681 686
682 687 class ResponseFilter(object):
683 688
684 689 def __init__(self, start_response):
685 690 self._start_response = start_response
686 691
687 692 def __call__(self, status, response_headers, exc_info=None):
688 693 headers = tuple(
689 694 (h, v) for h, v in response_headers
690 695 if not wsgiref.util.is_hop_by_hop(h))
691 696 return self._start_response(status, headers, exc_info)
692 697
693 698
694 699 def sanitize_settings_and_apply_defaults(global_config, settings):
695 700 global_settings_maker = SettingsMaker(global_config)
696 701 settings_maker = SettingsMaker(settings)
697 702
698 703 settings_maker.make_setting('logging.autoconfigure', False, parser='bool')
699 704
700 705 logging_conf = os.path.join(os.path.dirname(global_config.get('__file__')), 'logging.ini')
701 706 settings_maker.enable_logging(logging_conf)
702 707
703 708 # Default includes, possible to change as a user
704 709 pyramid_includes = settings_maker.make_setting('pyramid.includes', [], parser='list:newline')
705 710 log.debug("Using the following pyramid.includes: %s", pyramid_includes)
706 711
707 712 settings_maker.make_setting('__file__', global_config.get('__file__'))
708 713
709 714 settings_maker.make_setting('pyramid.default_locale_name', 'en')
710 715 settings_maker.make_setting('locale', 'en_US.UTF-8')
711 716
712 717 settings_maker.make_setting('core.binary_dir', '')
713 718
714 719 temp_store = tempfile.gettempdir()
715 720 default_cache_dir = os.path.join(temp_store, 'rc_cache')
716 721 # save default, cache dir, and use it for all backends later.
717 722 default_cache_dir = settings_maker.make_setting(
718 723 'cache_dir',
719 724 default=default_cache_dir, default_when_empty=True,
720 725 parser='dir:ensured')
721 726
722 727 # exception store cache
723 728 settings_maker.make_setting(
724 729 'exception_tracker.store_path',
725 730 default=os.path.join(default_cache_dir, 'exc_store'), default_when_empty=True,
726 731 parser='dir:ensured'
727 732 )
728 733
729 734 # repo_object cache defaults
730 735 settings_maker.make_setting(
731 736 'rc_cache.repo_object.backend',
732 737 default='dogpile.cache.rc.file_namespace',
733 738 parser='string')
734 739 settings_maker.make_setting(
735 740 'rc_cache.repo_object.expiration_time',
736 741 default=30 * 24 * 60 * 60, # 30days
737 742 parser='int')
738 743 settings_maker.make_setting(
739 744 'rc_cache.repo_object.arguments.filename',
740 745 default=os.path.join(default_cache_dir, 'vcsserver_cache_repo_object.db'),
741 746 parser='string')
742 747
743 748 # statsd
744 749 settings_maker.make_setting('statsd.enabled', False, parser='bool')
745 750 settings_maker.make_setting('statsd.statsd_host', 'statsd-exporter', parser='string')
746 751 settings_maker.make_setting('statsd.statsd_port', 9125, parser='int')
747 752 settings_maker.make_setting('statsd.statsd_prefix', '')
748 753 settings_maker.make_setting('statsd.statsd_ipv6', False, parser='bool')
749 754
750 755 settings_maker.env_expand()
751 756
752 757
753 758 def main(global_config, **settings):
754 759 start_time = time.time()
755 760 log.info('Pyramid app config starting')
756 761
757 762 if MercurialFactory:
758 763 hgpatches.patch_largefiles_capabilities()
759 764 hgpatches.patch_subrepo_type_mapping()
760 765
761 766 # Fill in and sanitize the defaults & do ENV expansion
762 767 sanitize_settings_and_apply_defaults(global_config, settings)
763 768
764 769 # init and bootstrap StatsdClient
765 770 StatsdClient.setup(settings)
766 771
767 772 pyramid_app = HTTPApplication(settings=settings, global_config=global_config).wsgi_app()
768 773 total_time = time.time() - start_time
769 774 log.info('Pyramid app `%s` created and configured in %.2fs',
770 getattr(pyramid_app, 'func_name', 'pyramid_app'), total_time)
775 pyramid_app.__class__, total_time)
771 776 return pyramid_app
772 777
773 778
General Comments 0
You need to be logged in to leave comments. Login now