##// END OF EJS Templates
core: unify startup log for pyramid
super-admin -
r1125:f619b34f python3
parent child Browse files
Show More
@@ -1,40 +1,41 b''
1 1 syntax: glob
2
2 3 *.orig
3 4 *.pyc
4 5 *.swp
5 6 *.sqlite
6 7 *.tox
7 8 *.egg-info
8 9 *.egg
9 10 *.eggs
10 11 *.idea
11 12 .DS_Store*
12 13
13 14
14 15 syntax: regexp
15 16
16 17 #.filename
17 18 ^\.settings$
18 19 ^\.project$
19 20 ^\.pydevproject$
20 21 ^\.coverage$
21 22 ^\.cache.*$
22 23 ^\.venv.*$
23 24 ^\.ruff_cache.*$
24 25 ^\.rhodecode$
25 26
26 27
27 28 ^.dev
28 29 ^build/
29 30 ^coverage\.xml$
30 31 ^data$
31 32 ^dev.ini$
32 33 ^acceptance_tests/dev.*\.ini$
33 34 ^dist/
34 35 ^fabfile.py
35 36 ^htmlcov
36 37 ^junit\.xml$
37 38 ^node_modules/
38 39 ^pylint.log$
39 40 ^build$
40 41 ^result$
@@ -1,778 +1,777 b''
1 1 # RhodeCode VCSServer provides access to different vcs backends via network.
2 2 # Copyright (C) 2014-2020 RhodeCode GmbH
3 3 #
4 4 # This program is free software; you can redistribute it and/or modify
5 5 # it under the terms of the GNU General Public License as published by
6 6 # the Free Software Foundation; either version 3 of the License, or
7 7 # (at your option) any later version.
8 8 #
9 9 # This program is distributed in the hope that it will be useful,
10 10 # but WITHOUT ANY WARRANTY; without even the implied warranty of
11 11 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12 12 # GNU General Public License for more details.
13 13 #
14 14 # You should have received a copy of the GNU General Public License
15 15 # along with this program; if not, write to the Free Software Foundation,
16 16 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17 17
18 18 import io
19 19 import os
20 20 import sys
21 21 import locale
22 22 import logging
23 23 import uuid
24 24 import time
25 25 import wsgiref.util
26 26 import traceback
27 27 import tempfile
28 28 import psutil
29 29
30 30 from itertools import chain
31 31
32 32 import msgpack
33 33 import configparser
34 34
35 35 from pyramid.config import Configurator
36 36 from pyramid.wsgi import wsgiapp
37 37 from pyramid.response import Response
38 38
39 39 from vcsserver.base import BytesEnvelope, BinaryEnvelope
40 40 from vcsserver.lib.rc_json import json
41 41 from vcsserver.config.settings_maker import SettingsMaker
42 42 from vcsserver.str_utils import safe_int
43 43 from vcsserver.lib.statsd_client import StatsdClient
44 44 from vcsserver.tweens.request_wrapper import get_call_context, get_headers_call_context
45 45
46 46 log = logging.getLogger(__name__)
47 47
48 48 # due to Mercurial/glibc2.27 problems we need to detect if locale settings are
49 49 # causing problems and "fix" it in case they do and fallback to LC_ALL = C
50 50
51 51 try:
52 52 locale.setlocale(locale.LC_ALL, '')
53 53 except locale.Error as e:
54 54 log.error(
55 55 'LOCALE ERROR: failed to set LC_ALL, fallback to LC_ALL=C, org error: %s', e)
56 56 os.environ['LC_ALL'] = 'C'
57 57
58 58
59 59 import vcsserver
60 60 from vcsserver import remote_wsgi, scm_app, settings, hgpatches
61 61 from vcsserver.git_lfs.app import GIT_LFS_CONTENT_TYPE, GIT_LFS_PROTO_PAT
62 62 from vcsserver.echo_stub import remote_wsgi as remote_wsgi_stub
63 63 from vcsserver.echo_stub.echo_app import EchoApp
64 64 from vcsserver.exceptions import HTTPRepoLocked, HTTPRepoBranchProtected
65 65 from vcsserver.lib.exc_tracking import store_exception
66 66 from vcsserver.server import VcsServer
67 67
68 68 strict_vcs = True
69 69
70 70 git_import_err = None
71 71 try:
72 72 from vcsserver.remote.git import GitFactory, GitRemote
73 73 except ImportError as e:
74 74 GitFactory = None
75 75 GitRemote = None
76 76 git_import_err = e
77 77 if strict_vcs:
78 78 raise
79 79
80 80
81 81 hg_import_err = None
82 82 try:
83 83 from vcsserver.remote.hg import MercurialFactory, HgRemote
84 84 except ImportError as e:
85 85 MercurialFactory = None
86 86 HgRemote = None
87 87 hg_import_err = e
88 88 if strict_vcs:
89 89 raise
90 90
91 91
92 92 svn_import_err = None
93 93 try:
94 94 from vcsserver.remote.svn import SubversionFactory, SvnRemote
95 95 except ImportError as e:
96 96 SubversionFactory = None
97 97 SvnRemote = None
98 98 svn_import_err = e
99 99 if strict_vcs:
100 100 raise
101 101
102 102
103 103 def _is_request_chunked(environ):
104 104 stream = environ.get('HTTP_TRANSFER_ENCODING', '') == 'chunked'
105 105 return stream
106 106
107 107
108 108 def log_max_fd():
109 109 try:
110 110 maxfd = psutil.Process().rlimit(psutil.RLIMIT_NOFILE)[1]
111 111 log.info('Max file descriptors value: %s', maxfd)
112 112 except Exception:
113 113 pass
114 114
115 115
116 116 class VCS(object):
117 117 def __init__(self, locale_conf=None, cache_config=None):
118 118 self.locale = locale_conf
119 119 self.cache_config = cache_config
120 120 self._configure_locale()
121 121
122 122 log_max_fd()
123 123
124 124 if GitFactory and GitRemote:
125 125 git_factory = GitFactory()
126 126 self._git_remote = GitRemote(git_factory)
127 127 else:
128 128 log.error("Git client import failed: %s", git_import_err)
129 129
130 130 if MercurialFactory and HgRemote:
131 131 hg_factory = MercurialFactory()
132 132 self._hg_remote = HgRemote(hg_factory)
133 133 else:
134 134 log.error("Mercurial client import failed: %s", hg_import_err)
135 135
136 136 if SubversionFactory and SvnRemote:
137 137 svn_factory = SubversionFactory()
138 138
139 139 # hg factory is used for svn url validation
140 140 hg_factory = MercurialFactory()
141 141 self._svn_remote = SvnRemote(svn_factory, hg_factory=hg_factory)
142 142 else:
143 143 log.error("Subversion client import failed: %s", svn_import_err)
144 144
145 145 self._vcsserver = VcsServer()
146 146
147 147 def _configure_locale(self):
148 148 if self.locale:
149 149 log.info('Settings locale: `LC_ALL` to %s', self.locale)
150 150 else:
151 151 log.info('Configuring locale subsystem based on environment variables')
152 152 try:
153 153 # If self.locale is the empty string, then the locale
154 154 # module will use the environment variables. See the
155 155 # documentation of the package `locale`.
156 156 locale.setlocale(locale.LC_ALL, self.locale)
157 157
158 158 language_code, encoding = locale.getlocale()
159 159 log.info(
160 160 'Locale set to language code "%s" with encoding "%s".',
161 161 language_code, encoding)
162 162 except locale.Error:
163 163 log.exception('Cannot set locale, not configuring the locale system')
164 164
165 165
166 166 class WsgiProxy(object):
167 167 def __init__(self, wsgi):
168 168 self.wsgi = wsgi
169 169
170 170 def __call__(self, environ, start_response):
171 171 input_data = environ['wsgi.input'].read()
172 172 input_data = msgpack.unpackb(input_data)
173 173
174 174 error = None
175 175 try:
176 176 data, status, headers = self.wsgi.handle(
177 177 input_data['environment'], input_data['input_data'],
178 178 *input_data['args'], **input_data['kwargs'])
179 179 except Exception as e:
180 180 data, status, headers = [], None, None
181 181 error = {
182 182 'message': str(e),
183 183 '_vcs_kind': getattr(e, '_vcs_kind', None)
184 184 }
185 185
186 186 start_response(200, {})
187 187 return self._iterator(error, status, headers, data)
188 188
189 189 def _iterator(self, error, status, headers, data):
190 190 initial_data = [
191 191 error,
192 192 status,
193 193 headers,
194 194 ]
195 195
196 196 for d in chain(initial_data, data):
197 197 yield msgpack.packb(d)
198 198
199 199
200 200 def not_found(request):
201 201 return {'status': '404 NOT FOUND'}
202 202
203 203
204 204 class VCSViewPredicate(object):
205 205 def __init__(self, val, config):
206 206 self.remotes = val
207 207
208 208 def text(self):
209 209 return f'vcs view method = {list(self.remotes.keys())}'
210 210
211 211 phash = text
212 212
213 213 def __call__(self, context, request):
214 214 """
215 215 View predicate that returns true if given backend is supported by
216 216 defined remotes.
217 217 """
218 218 backend = request.matchdict.get('backend')
219 219 return backend in self.remotes
220 220
221 221
222 222 class HTTPApplication(object):
223 223 ALLOWED_EXCEPTIONS = ('KeyError', 'URLError')
224 224
225 225 remote_wsgi = remote_wsgi
226 226 _use_echo_app = False
227 227
228 228 def __init__(self, settings=None, global_config=None):
229 229
230 230 self.config = Configurator(settings=settings)
231 231 # Init our statsd at very start
232 232 self.config.registry.statsd = StatsdClient.statsd
233 233 self.config.registry.vcs_call_context = {}
234 234
235 235 self.global_config = global_config
236 236 self.config.include('vcsserver.lib.rc_cache')
237 237 self.config.include('vcsserver.lib.rc_cache.archive_cache')
238 238
239 239 settings_locale = settings.get('locale', '') or 'en_US.UTF-8'
240 240 vcs = VCS(locale_conf=settings_locale, cache_config=settings)
241 241 self._remotes = {
242 242 'hg': vcs._hg_remote,
243 243 'git': vcs._git_remote,
244 244 'svn': vcs._svn_remote,
245 245 'server': vcs._vcsserver,
246 246 }
247 247 if settings.get('dev.use_echo_app', 'false').lower() == 'true':
248 248 self._use_echo_app = True
249 249 log.warning("Using EchoApp for VCS operations.")
250 250 self.remote_wsgi = remote_wsgi_stub
251 251
252 252 self._configure_settings(global_config, settings)
253 253
254 254 self._configure()
255 255
256 256 def _configure_settings(self, global_config, app_settings):
257 257 """
258 258 Configure the settings module.
259 259 """
260 260 settings_merged = global_config.copy()
261 261 settings_merged.update(app_settings)
262 262
263 263 git_path = app_settings.get('git_path', None)
264 264 if git_path:
265 265 settings.GIT_EXECUTABLE = git_path
266 266 binary_dir = app_settings.get('core.binary_dir', None)
267 267 if binary_dir:
268 268 settings.BINARY_DIR = binary_dir
269 269
270 270 # Store the settings to make them available to other modules.
271 271 vcsserver.PYRAMID_SETTINGS = settings_merged
272 272 vcsserver.CONFIG = settings_merged
273 273
274 274 def _configure(self):
275 275 self.config.add_renderer(name='msgpack', factory=self._msgpack_renderer_factory)
276 276
277 277 self.config.add_route('service', '/_service')
278 278 self.config.add_route('status', '/status')
279 279 self.config.add_route('hg_proxy', '/proxy/hg')
280 280 self.config.add_route('git_proxy', '/proxy/git')
281 281
282 282 # rpc methods
283 283 self.config.add_route('vcs', '/{backend}')
284 284
285 285 # streaming rpc remote methods
286 286 self.config.add_route('vcs_stream', '/{backend}/stream')
287 287
288 288 # vcs operations clone/push as streaming
289 289 self.config.add_route('stream_git', '/stream/git/*repo_name')
290 290 self.config.add_route('stream_hg', '/stream/hg/*repo_name')
291 291
292 292 self.config.add_view(self.status_view, route_name='status', renderer='json')
293 293 self.config.add_view(self.service_view, route_name='service', renderer='msgpack')
294 294
295 295 self.config.add_view(self.hg_proxy(), route_name='hg_proxy')
296 296 self.config.add_view(self.git_proxy(), route_name='git_proxy')
297 297 self.config.add_view(self.vcs_view, route_name='vcs', renderer='msgpack',
298 298 vcs_view=self._remotes)
299 299 self.config.add_view(self.vcs_stream_view, route_name='vcs_stream',
300 300 vcs_view=self._remotes)
301 301
302 302 self.config.add_view(self.hg_stream(), route_name='stream_hg')
303 303 self.config.add_view(self.git_stream(), route_name='stream_git')
304 304
305 305 self.config.add_view_predicate('vcs_view', VCSViewPredicate)
306 306
307 307 self.config.add_notfound_view(not_found, renderer='json')
308 308
309 309 self.config.add_view(self.handle_vcs_exception, context=Exception)
310 310
311 311 self.config.add_tween(
312 312 'vcsserver.tweens.request_wrapper.RequestWrapperTween',
313 313 )
314 314 self.config.add_request_method(
315 315 'vcsserver.lib.request_counter.get_request_counter',
316 316 'request_count')
317 317
318 318 def wsgi_app(self):
319 319 return self.config.make_wsgi_app()
320 320
321 321 def _vcs_view_params(self, request):
322 322 remote = self._remotes[request.matchdict['backend']]
323 323 payload = msgpack.unpackb(request.body, use_list=True)
324 324
325 325 method = payload.get('method')
326 326 params = payload['params']
327 327 wire = params.get('wire')
328 328 args = params.get('args')
329 329 kwargs = params.get('kwargs')
330 330 context_uid = None
331 331
332 332 request.registry.vcs_call_context = {
333 333 'method': method,
334 334 'repo_name': payload.get('_repo_name'),
335 335 }
336 336
337 337 if wire:
338 338 try:
339 339 wire['context'] = context_uid = uuid.UUID(wire['context'])
340 340 except KeyError:
341 341 pass
342 342 args.insert(0, wire)
343 343 repo_state_uid = wire.get('repo_state_uid') if wire else None
344 344
345 345 # NOTE(marcink): trading complexity for slight performance
346 346 if log.isEnabledFor(logging.DEBUG):
347 347 # also we SKIP printing out any of those methods args since they maybe excessive
348 348 just_args_methods = {
349 349 'commitctx': ('content', 'removed', 'updated'),
350 350 'commit': ('content', 'removed', 'updated')
351 351 }
352 352 if method in just_args_methods:
353 353 skip_args = just_args_methods[method]
354 354 call_args = ''
355 355 call_kwargs = {}
356 356 for k in kwargs:
357 357 if k in skip_args:
358 358 # replace our skip key with dummy
359 359 call_kwargs[k] = f'RemovedParam({k})'
360 360 else:
361 361 call_kwargs[k] = kwargs[k]
362 362 else:
363 363 call_args = args[1:]
364 364 call_kwargs = kwargs
365 365
366 366 log.debug('Method requested:`%s` with args:%s kwargs:%s context_uid: %s, repo_state_uid:%s',
367 367 method, call_args, call_kwargs, context_uid, repo_state_uid)
368 368
369 369 statsd = request.registry.statsd
370 370 if statsd:
371 371 statsd.incr(
372 372 'vcsserver_method_total', tags=[
373 373 f"method:{method}",
374 374 ])
375 375 return payload, remote, method, args, kwargs
376 376
377 377 def vcs_view(self, request):
378 378
379 379 payload, remote, method, args, kwargs = self._vcs_view_params(request)
380 380 payload_id = payload.get('id')
381 381
382 382 try:
383 383 resp = getattr(remote, method)(*args, **kwargs)
384 384 except Exception as e:
385 385 exc_info = list(sys.exc_info())
386 386 exc_type, exc_value, exc_traceback = exc_info
387 387
388 388 org_exc = getattr(e, '_org_exc', None)
389 389 org_exc_name = None
390 390 org_exc_tb = ''
391 391 if org_exc:
392 392 org_exc_name = org_exc.__class__.__name__
393 393 org_exc_tb = getattr(e, '_org_exc_tb', '')
394 394 # replace our "faked" exception with our org
395 395 exc_info[0] = org_exc.__class__
396 396 exc_info[1] = org_exc
397 397
398 398 should_store_exc = True
399 399 if org_exc:
400 400 def get_exc_fqn(_exc_obj):
401 401 module_name = getattr(org_exc.__class__, '__module__', 'UNKNOWN')
402 402 return module_name + '.' + org_exc_name
403 403
404 404 exc_fqn = get_exc_fqn(org_exc)
405 405
406 406 if exc_fqn in ['mercurial.error.RepoLookupError',
407 407 'vcsserver.exceptions.RefNotFoundException']:
408 408 should_store_exc = False
409 409
410 410 if should_store_exc:
411 411 store_exception(id(exc_info), exc_info, request_path=request.path)
412 412
413 413 tb_info = ''.join(
414 414 traceback.format_exception(exc_type, exc_value, exc_traceback))
415 415
416 416 type_ = e.__class__.__name__
417 417 if type_ not in self.ALLOWED_EXCEPTIONS:
418 418 type_ = None
419 419
420 420 resp = {
421 421 'id': payload_id,
422 422 'error': {
423 423 'message': str(e),
424 424 'traceback': tb_info,
425 425 'org_exc': org_exc_name,
426 426 'org_exc_tb': org_exc_tb,
427 427 'type': type_
428 428 }
429 429 }
430 430
431 431 try:
432 432 resp['error']['_vcs_kind'] = getattr(e, '_vcs_kind', None)
433 433 except AttributeError:
434 434 pass
435 435 else:
436 436 resp = {
437 437 'id': payload_id,
438 438 'result': resp
439 439 }
440 440 log.debug('Serving data for method %s', method)
441 441 return resp
442 442
443 443 def vcs_stream_view(self, request):
444 444 payload, remote, method, args, kwargs = self._vcs_view_params(request)
445 445 # this method has a stream: marker we remove it here
446 446 method = method.split('stream:')[-1]
447 447 chunk_size = safe_int(payload.get('chunk_size')) or 4096
448 448
449 449 try:
450 450 resp = getattr(remote, method)(*args, **kwargs)
451 451 except Exception as e:
452 452 raise
453 453
454 454 def get_chunked_data(method_resp):
455 455 stream = io.BytesIO(method_resp)
456 456 while 1:
457 457 chunk = stream.read(chunk_size)
458 458 if not chunk:
459 459 break
460 460 yield chunk
461 461
462 462 response = Response(app_iter=get_chunked_data(resp))
463 463 response.content_type = 'application/octet-stream'
464 464
465 465 return response
466 466
467 467 def status_view(self, request):
468 468 import vcsserver
469 469 return {'status': 'OK', 'vcsserver_version': vcsserver.__version__,
470 470 'pid': os.getpid()}
471 471
472 472 def service_view(self, request):
473 473 import vcsserver
474 474
475 475 payload = msgpack.unpackb(request.body, use_list=True)
476 476 server_config, app_config = {}, {}
477 477
478 478 try:
479 479 path = self.global_config['__file__']
480 480 config = configparser.RawConfigParser()
481 481
482 482 config.read(path)
483 483
484 484 if config.has_section('server:main'):
485 485 server_config = dict(config.items('server:main'))
486 486 if config.has_section('app:main'):
487 487 app_config = dict(config.items('app:main'))
488 488
489 489 except Exception:
490 490 log.exception('Failed to read .ini file for display')
491 491
492 492 environ = list(os.environ.items())
493 493
494 494 resp = {
495 495 'id': payload.get('id'),
496 496 'result': dict(
497 497 version=vcsserver.__version__,
498 498 config=server_config,
499 499 app_config=app_config,
500 500 environ=environ,
501 501 payload=payload,
502 502 )
503 503 }
504 504 return resp
505 505
506 506 def _msgpack_renderer_factory(self, info):
507 507
508 508 def _render(value, system):
509 509 bin_type = False
510 510 res = value.get('result')
511 511 if isinstance(res, BytesEnvelope):
512 512 log.debug('Result is wrapped in BytesEnvelope type')
513 513 bin_type = True
514 514 elif isinstance(res, BinaryEnvelope):
515 515 log.debug('Result is wrapped in BinaryEnvelope type')
516 516 value['result'] = res.val
517 517 bin_type = True
518 518
519 519 request = system.get('request')
520 520 if request is not None:
521 521 response = request.response
522 522 ct = response.content_type
523 523 if ct == response.default_content_type:
524 524 response.content_type = 'application/x-msgpack'
525 525 if bin_type:
526 526 response.content_type = 'application/x-msgpack-bin'
527 527
528 528 return msgpack.packb(value, use_bin_type=bin_type)
529 529 return _render
530 530
531 531 def set_env_from_config(self, environ, config):
532 532 dict_conf = {}
533 533 try:
534 534 for elem in config:
535 535 if elem[0] == 'rhodecode':
536 536 dict_conf = json.loads(elem[2])
537 537 break
538 538 except Exception:
539 539 log.exception('Failed to fetch SCM CONFIG')
540 540 return
541 541
542 542 username = dict_conf.get('username')
543 543 if username:
544 544 environ['REMOTE_USER'] = username
545 545 # mercurial specific, some extension api rely on this
546 546 environ['HGUSER'] = username
547 547
548 548 ip = dict_conf.get('ip')
549 549 if ip:
550 550 environ['REMOTE_HOST'] = ip
551 551
552 552 if _is_request_chunked(environ):
553 553 # set the compatibility flag for webob
554 554 environ['wsgi.input_terminated'] = True
555 555
556 556 def hg_proxy(self):
557 557 @wsgiapp
558 558 def _hg_proxy(environ, start_response):
559 559 app = WsgiProxy(self.remote_wsgi.HgRemoteWsgi())
560 560 return app(environ, start_response)
561 561 return _hg_proxy
562 562
563 563 def git_proxy(self):
564 564 @wsgiapp
565 565 def _git_proxy(environ, start_response):
566 566 app = WsgiProxy(self.remote_wsgi.GitRemoteWsgi())
567 567 return app(environ, start_response)
568 568 return _git_proxy
569 569
570 570 def hg_stream(self):
571 571 if self._use_echo_app:
572 572 @wsgiapp
573 573 def _hg_stream(environ, start_response):
574 574 app = EchoApp('fake_path', 'fake_name', None)
575 575 return app(environ, start_response)
576 576 return _hg_stream
577 577 else:
578 578 @wsgiapp
579 579 def _hg_stream(environ, start_response):
580 580 log.debug('http-app: handling hg stream')
581 581 call_context = get_headers_call_context(environ)
582 582
583 583 repo_path = call_context['repo_path']
584 584 repo_name = call_context['repo_name']
585 585 config = call_context['repo_config']
586 586
587 587 app = scm_app.create_hg_wsgi_app(
588 588 repo_path, repo_name, config)
589 589
590 590 # Consistent path information for hgweb
591 591 environ['PATH_INFO'] = call_context['path_info']
592 592 environ['REPO_NAME'] = repo_name
593 593 self.set_env_from_config(environ, config)
594 594
595 595 log.debug('http-app: starting app handler '
596 596 'with %s and process request', app)
597 597 return app(environ, ResponseFilter(start_response))
598 598 return _hg_stream
599 599
600 600 def git_stream(self):
601 601 if self._use_echo_app:
602 602 @wsgiapp
603 603 def _git_stream(environ, start_response):
604 604 app = EchoApp('fake_path', 'fake_name', None)
605 605 return app(environ, start_response)
606 606 return _git_stream
607 607 else:
608 608 @wsgiapp
609 609 def _git_stream(environ, start_response):
610 610 log.debug('http-app: handling git stream')
611 611
612 612 call_context = get_headers_call_context(environ)
613 613
614 614 repo_path = call_context['repo_path']
615 615 repo_name = call_context['repo_name']
616 616 config = call_context['repo_config']
617 617
618 618 environ['PATH_INFO'] = call_context['path_info']
619 619 self.set_env_from_config(environ, config)
620 620
621 621 content_type = environ.get('CONTENT_TYPE', '')
622 622
623 623 path = environ['PATH_INFO']
624 624 is_lfs_request = GIT_LFS_CONTENT_TYPE in content_type
625 625 log.debug(
626 626 'LFS: Detecting if request `%s` is LFS server path based '
627 627 'on content type:`%s`, is_lfs:%s',
628 628 path, content_type, is_lfs_request)
629 629
630 630 if not is_lfs_request:
631 631 # fallback detection by path
632 632 if GIT_LFS_PROTO_PAT.match(path):
633 633 is_lfs_request = True
634 634 log.debug(
635 635 'LFS: fallback detection by path of: `%s`, is_lfs:%s',
636 636 path, is_lfs_request)
637 637
638 638 if is_lfs_request:
639 639 app = scm_app.create_git_lfs_wsgi_app(
640 640 repo_path, repo_name, config)
641 641 else:
642 642 app = scm_app.create_git_wsgi_app(
643 643 repo_path, repo_name, config)
644 644
645 645 log.debug('http-app: starting app handler '
646 646 'with %s and process request', app)
647 647
648 648 return app(environ, start_response)
649 649
650 650 return _git_stream
651 651
652 652 def handle_vcs_exception(self, exception, request):
653 653 _vcs_kind = getattr(exception, '_vcs_kind', '')
654 654
655 655 if _vcs_kind == 'repo_locked':
656 656 headers_call_context = get_headers_call_context(request.environ)
657 657 status_code = safe_int(headers_call_context['locked_status_code'])
658 658
659 659 return HTTPRepoLocked(
660 660 title=str(exception), status_code=status_code, headers=[('X-Rc-Locked', '1')])
661 661
662 662 elif _vcs_kind == 'repo_branch_protected':
663 663 # Get custom repo-branch-protected status code if present.
664 664 return HTTPRepoBranchProtected(
665 665 title=str(exception), headers=[('X-Rc-Branch-Protection', '1')])
666 666
667 667 exc_info = request.exc_info
668 668 store_exception(id(exc_info), exc_info)
669 669
670 670 traceback_info = 'unavailable'
671 671 if request.exc_info:
672 672 exc_type, exc_value, exc_tb = request.exc_info
673 673 traceback_info = ''.join(traceback.format_exception(exc_type, exc_value, exc_tb))
674 674
675 675 log.error(
676 676 'error occurred handling this request for path: %s, \n tb: %s',
677 677 request.path, traceback_info)
678 678
679 679 statsd = request.registry.statsd
680 680 if statsd:
681 681 exc_type = f"{exception.__class__.__module__}.{exception.__class__.__name__}"
682 682 statsd.incr('vcsserver_exception_total',
683 683 tags=[f"type:{exc_type}"])
684 684 raise exception
685 685
686 686
687 687 class ResponseFilter(object):
688 688
689 689 def __init__(self, start_response):
690 690 self._start_response = start_response
691 691
692 692 def __call__(self, status, response_headers, exc_info=None):
693 693 headers = tuple(
694 694 (h, v) for h, v in response_headers
695 695 if not wsgiref.util.is_hop_by_hop(h))
696 696 return self._start_response(status, headers, exc_info)
697 697
698 698
699 699 def sanitize_settings_and_apply_defaults(global_config, settings):
700 700 global_settings_maker = SettingsMaker(global_config)
701 701 settings_maker = SettingsMaker(settings)
702 702
703 703 settings_maker.make_setting('logging.autoconfigure', False, parser='bool')
704 704
705 705 logging_conf = os.path.join(os.path.dirname(global_config.get('__file__')), 'logging.ini')
706 706 settings_maker.enable_logging(logging_conf)
707 707
708 708 # Default includes, possible to change as a user
709 709 pyramid_includes = settings_maker.make_setting('pyramid.includes', [], parser='list:newline')
710 710 log.debug("Using the following pyramid.includes: %s", pyramid_includes)
711 711
712 712 settings_maker.make_setting('__file__', global_config.get('__file__'))
713 713
714 714 settings_maker.make_setting('pyramid.default_locale_name', 'en')
715 715 settings_maker.make_setting('locale', 'en_US.UTF-8')
716 716
717 717 settings_maker.make_setting('core.binary_dir', '')
718 718
719 719 temp_store = tempfile.gettempdir()
720 720 default_cache_dir = os.path.join(temp_store, 'rc_cache')
721 721 # save default, cache dir, and use it for all backends later.
722 722 default_cache_dir = settings_maker.make_setting(
723 723 'cache_dir',
724 724 default=default_cache_dir, default_when_empty=True,
725 725 parser='dir:ensured')
726 726
727 727 # exception store cache
728 728 settings_maker.make_setting(
729 729 'exception_tracker.store_path',
730 730 default=os.path.join(default_cache_dir, 'exc_store'), default_when_empty=True,
731 731 parser='dir:ensured'
732 732 )
733 733
734 734 # repo_object cache defaults
735 735 settings_maker.make_setting(
736 736 'rc_cache.repo_object.backend',
737 737 default='dogpile.cache.rc.file_namespace',
738 738 parser='string')
739 739 settings_maker.make_setting(
740 740 'rc_cache.repo_object.expiration_time',
741 741 default=30 * 24 * 60 * 60, # 30days
742 742 parser='int')
743 743 settings_maker.make_setting(
744 744 'rc_cache.repo_object.arguments.filename',
745 745 default=os.path.join(default_cache_dir, 'vcsserver_cache_repo_object.db'),
746 746 parser='string')
747 747
748 748 # statsd
749 749 settings_maker.make_setting('statsd.enabled', False, parser='bool')
750 750 settings_maker.make_setting('statsd.statsd_host', 'statsd-exporter', parser='string')
751 751 settings_maker.make_setting('statsd.statsd_port', 9125, parser='int')
752 752 settings_maker.make_setting('statsd.statsd_prefix', '')
753 753 settings_maker.make_setting('statsd.statsd_ipv6', False, parser='bool')
754 754
755 755 settings_maker.env_expand()
756 756
757 757
758 758 def main(global_config, **settings):
759 759 start_time = time.time()
760 760 log.info('Pyramid app config starting')
761 761
762 762 if MercurialFactory:
763 763 hgpatches.patch_largefiles_capabilities()
764 764 hgpatches.patch_subrepo_type_mapping()
765 765
766 766 # Fill in and sanitize the defaults & do ENV expansion
767 767 sanitize_settings_and_apply_defaults(global_config, settings)
768 768
769 769 # init and bootstrap StatsdClient
770 770 StatsdClient.setup(settings)
771 771
772 772 pyramid_app = HTTPApplication(settings=settings, global_config=global_config).wsgi_app()
773 773 total_time = time.time() - start_time
774 log.info('Pyramid app `%s` created and configured in %.2fs',
775 pyramid_app.__class__, total_time)
774 log.info('Pyramid app created and configured in %.2fs', total_time)
776 775 return pyramid_app
777 776
778 777
General Comments 0
You need to be logged in to leave comments. Login now