http_main.py
774 lines
| 26.2 KiB
| text/x-python
|
PythonLexer
/ vcsserver / http_main.py
r0 | # RhodeCode VCSServer provides access to different vcs backends via network. | |||
r1126 | # Copyright (C) 2014-2023 RhodeCode GmbH | |||
r0 | # | |||
# This program is free software; you can redistribute it and/or modify | ||||
# it under the terms of the GNU General Public License as published by | ||||
# the Free Software Foundation; either version 3 of the License, or | ||||
# (at your option) any later version. | ||||
# | ||||
# This program is distributed in the hope that it will be useful, | ||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of | ||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||||
# GNU General Public License for more details. | ||||
# | ||||
# You should have received a copy of the GNU General Public License | ||||
# along with this program; if not, write to the Free Software Foundation, | ||||
# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA | ||||
r1048 | import io | |||
r372 | import os | |||
r1148 | import platform | |||
r491 | import sys | |||
r0 | import locale | |||
import logging | ||||
import uuid | ||||
r1021 | import time | |||
r0 | import wsgiref.util | |||
r519 | import tempfile | |||
r913 | import psutil | |||
r1021 | ||||
r0 | from itertools import chain | |||
import msgpack | ||||
r1038 | import configparser | |||
r1041 | ||||
r0 | from pyramid.config import Configurator | |||
from pyramid.wsgi import wsgiapp | ||||
r768 | from pyramid.response import Response | |||
r0 | ||||
r1123 | from vcsserver.base import BytesEnvelope, BinaryEnvelope | |||
r1048 | from vcsserver.lib.rc_json import json | |||
r1021 | from vcsserver.config.settings_maker import SettingsMaker | |||
r1123 | from vcsserver.str_utils import safe_int | |||
r1005 | from vcsserver.lib.statsd_client import StatsdClient | |||
r1145 | from vcsserver.tweens.request_wrapper import get_headers_call_context | |||
r1021 | ||||
r519 | import vcsserver | |||
|
r35 | from vcsserver import remote_wsgi, scm_app, settings, hgpatches | ||
r180 | from vcsserver.git_lfs.app import GIT_LFS_CONTENT_TYPE, GIT_LFS_PROTO_PAT | |||
r0 | from vcsserver.echo_stub import remote_wsgi as remote_wsgi_stub | |||
from vcsserver.echo_stub.echo_app import EchoApp | ||||
r509 | from vcsserver.exceptions import HTTPRepoLocked, HTTPRepoBranchProtected | |||
r1144 | from vcsserver.lib.exc_tracking import store_exception, format_exc | |||
r0 | from vcsserver.server import VcsServer | |||
r1039 | strict_vcs = True | |||
r988 | ||||
git_import_err = None | ||||
r0 | try: | |||
r1145 | from vcsserver.remote.git_remote import GitFactory, GitRemote | |||
r988 | except ImportError as e: | |||
r0 | GitFactory = None | |||
GitRemote = None | ||||
r988 | git_import_err = e | |||
r1039 | if strict_vcs: | |||
raise | ||||
r180 | ||||
r988 | hg_import_err = None | |||
r0 | try: | |||
r1145 | from vcsserver.remote.hg_remote import MercurialFactory, HgRemote | |||
r988 | except ImportError as e: | |||
r0 | MercurialFactory = None | |||
HgRemote = None | ||||
r988 | hg_import_err = e | |||
r1039 | if strict_vcs: | |||
raise | ||||
r180 | ||||
r988 | svn_import_err = None | |||
r0 | try: | |||
r1145 | from vcsserver.remote.svn_remote import SubversionFactory, SvnRemote | |||
r988 | except ImportError as e: | |||
r0 | SubversionFactory = None | |||
SvnRemote = None | ||||
r988 | svn_import_err = e | |||
r1039 | if strict_vcs: | |||
raise | ||||
r0 | ||||
r1145 | log = logging.getLogger(__name__) | |||
# due to Mercurial/glibc2.27 problems we need to detect if locale settings are | ||||
# causing problems and "fix" it in case they do and fallback to LC_ALL = C | ||||
try: | ||||
locale.setlocale(locale.LC_ALL, '') | ||||
except locale.Error as e: | ||||
log.error( | ||||
'LOCALE ERROR: failed to set LC_ALL, fallback to LC_ALL=C, org error: %s', e) | ||||
os.environ['LC_ALL'] = 'C' | ||||
r507 | ||||
r332 | def _is_request_chunked(environ): | |||
stream = environ.get('HTTP_TRANSFER_ENCODING', '') == 'chunked' | ||||
return stream | ||||
r913 | def log_max_fd(): | |||
try: | ||||
maxfd = psutil.Process().rlimit(psutil.RLIMIT_NOFILE)[1] | ||||
log.info('Max file descriptors value: %s', maxfd) | ||||
except Exception: | ||||
pass | ||||
r1152 | class VCS: | |||
r768 | def __init__(self, locale_conf=None, cache_config=None): | |||
self.locale = locale_conf | ||||
r0 | self.cache_config = cache_config | |||
self._configure_locale() | ||||
r913 | log_max_fd() | |||
r893 | ||||
r0 | if GitFactory and GitRemote: | |||
r483 | git_factory = GitFactory() | |||
r0 | self._git_remote = GitRemote(git_factory) | |||
else: | ||||
r988 | log.error("Git client import failed: %s", git_import_err) | |||
r0 | ||||
if MercurialFactory and HgRemote: | ||||
r483 | hg_factory = MercurialFactory() | |||
r0 | self._hg_remote = HgRemote(hg_factory) | |||
else: | ||||
r988 | log.error("Mercurial client import failed: %s", hg_import_err) | |||
r0 | ||||
if SubversionFactory and SvnRemote: | ||||
r483 | svn_factory = SubversionFactory() | |||
r407 | # hg factory is used for svn url validation | |||
r483 | hg_factory = MercurialFactory() | |||
r0 | self._svn_remote = SvnRemote(svn_factory, hg_factory=hg_factory) | |||
else: | ||||
r988 | log.error("Subversion client import failed: %s", svn_import_err) | |||
r0 | ||||
self._vcsserver = VcsServer() | ||||
def _configure_locale(self): | ||||
if self.locale: | ||||
r541 | log.info('Settings locale: `LC_ALL` to %s', self.locale) | |||
r0 | else: | |||
r988 | log.info('Configuring locale subsystem based on environment variables') | |||
r0 | try: | |||
# If self.locale is the empty string, then the locale | ||||
# module will use the environment variables. See the | ||||
# documentation of the package `locale`. | ||||
locale.setlocale(locale.LC_ALL, self.locale) | ||||
language_code, encoding = locale.getlocale() | ||||
log.info( | ||||
'Locale set to language code "%s" with encoding "%s".', | ||||
language_code, encoding) | ||||
except locale.Error: | ||||
r988 | log.exception('Cannot set locale, not configuring the locale system') | |||
r0 | ||||
r1152 | class WsgiProxy: | |||
r0 | def __init__(self, wsgi): | |||
self.wsgi = wsgi | ||||
def __call__(self, environ, start_response): | ||||
input_data = environ['wsgi.input'].read() | ||||
input_data = msgpack.unpackb(input_data) | ||||
error = None | ||||
try: | ||||
data, status, headers = self.wsgi.handle( | ||||
input_data['environment'], input_data['input_data'], | ||||
*input_data['args'], **input_data['kwargs']) | ||||
except Exception as e: | ||||
data, status, headers = [], None, None | ||||
error = { | ||||
'message': str(e), | ||||
'_vcs_kind': getattr(e, '_vcs_kind', None) | ||||
} | ||||
start_response(200, {}) | ||||
return self._iterator(error, status, headers, data) | ||||
def _iterator(self, error, status, headers, data): | ||||
initial_data = [ | ||||
error, | ||||
status, | ||||
headers, | ||||
] | ||||
for d in chain(initial_data, data): | ||||
yield msgpack.packb(d) | ||||
r583 | def not_found(request): | |||
return {'status': '404 NOT FOUND'} | ||||
r1152 | class VCSViewPredicate: | |||
r583 | def __init__(self, val, config): | |||
self.remotes = val | ||||
def text(self): | ||||
r1114 | return f'vcs view method = {list(self.remotes.keys())}' | |||
r583 | ||||
phash = text | ||||
def __call__(self, context, request): | ||||
""" | ||||
View predicate that returns true if given backend is supported by | ||||
defined remotes. | ||||
""" | ||||
backend = request.matchdict.get('backend') | ||||
return backend in self.remotes | ||||
r1152 | class HTTPApplication: | |||
r0 | ALLOWED_EXCEPTIONS = ('KeyError', 'URLError') | |||
remote_wsgi = remote_wsgi | ||||
_use_echo_app = False | ||||
r173 | def __init__(self, settings=None, global_config=None): | |||
r483 | ||||
r0 | self.config = Configurator(settings=settings) | |||
r1005 | # Init our statsd at very start | |||
self.config.registry.statsd = StatsdClient.statsd | ||||
r1078 | self.config.registry.vcs_call_context = {} | |||
r1005 | ||||
r173 | self.global_config = global_config | |||
r483 | self.config.include('vcsserver.lib.rc_cache') | |||
r1123 | self.config.include('vcsserver.lib.rc_cache.archive_cache') | |||
r173 | ||||
r743 | settings_locale = settings.get('locale', '') or 'en_US.UTF-8' | |||
r768 | vcs = VCS(locale_conf=settings_locale, cache_config=settings) | |||
r0 | self._remotes = { | |||
'hg': vcs._hg_remote, | ||||
'git': vcs._git_remote, | ||||
'svn': vcs._svn_remote, | ||||
'server': vcs._vcsserver, | ||||
} | ||||
if settings.get('dev.use_echo_app', 'false').lower() == 'true': | ||||
self._use_echo_app = True | ||||
log.warning("Using EchoApp for VCS operations.") | ||||
self.remote_wsgi = remote_wsgi_stub | ||||
r519 | ||||
self._configure_settings(global_config, settings) | ||||
r920 | ||||
r0 | self._configure() | |||
r519 | def _configure_settings(self, global_config, app_settings): | |||
r0 | """ | |||
Configure the settings module. | ||||
""" | ||||
r519 | settings_merged = global_config.copy() | |||
settings_merged.update(app_settings) | ||||
r1214 | binary_dir = app_settings['core.binary_dir'] | |||
settings.BINARY_DIR = binary_dir | ||||
r519 | # Store the settings to make them available to other modules. | |||
vcsserver.PYRAMID_SETTINGS = settings_merged | ||||
vcsserver.CONFIG = settings_merged | ||||
r0 | def _configure(self): | |||
r583 | self.config.add_renderer(name='msgpack', factory=self._msgpack_renderer_factory) | |||
r0 | ||||
r102 | self.config.add_route('service', '/_service') | |||
r0 | self.config.add_route('status', '/status') | |||
self.config.add_route('hg_proxy', '/proxy/hg') | ||||
self.config.add_route('git_proxy', '/proxy/git') | ||||
r768 | ||||
# rpc methods | ||||
r0 | self.config.add_route('vcs', '/{backend}') | |||
r768 | ||||
# streaming rpc remote methods | ||||
self.config.add_route('vcs_stream', '/{backend}/stream') | ||||
# vcs operations clone/push as streaming | ||||
r0 | self.config.add_route('stream_git', '/stream/git/*repo_name') | |||
self.config.add_route('stream_hg', '/stream/hg/*repo_name') | ||||
r583 | self.config.add_view(self.status_view, route_name='status', renderer='json') | |||
self.config.add_view(self.service_view, route_name='service', renderer='msgpack') | ||||
r102 | ||||
r0 | self.config.add_view(self.hg_proxy(), route_name='hg_proxy') | |||
self.config.add_view(self.git_proxy(), route_name='git_proxy') | ||||
r583 | self.config.add_view(self.vcs_view, route_name='vcs', renderer='msgpack', | |||
vcs_view=self._remotes) | ||||
r768 | self.config.add_view(self.vcs_stream_view, route_name='vcs_stream', | |||
vcs_view=self._remotes) | ||||
r0 | ||||
self.config.add_view(self.hg_stream(), route_name='stream_hg') | ||||
self.config.add_view(self.git_stream(), route_name='stream_git') | ||||
r151 | ||||
r583 | self.config.add_view_predicate('vcs_view', VCSViewPredicate) | |||
self.config.add_notfound_view(not_found, renderer='json') | ||||
r151 | ||||
r178 | self.config.add_view(self.handle_vcs_exception, context=Exception) | |||
r150 | ||||
r154 | self.config.add_tween( | |||
r744 | 'vcsserver.tweens.request_wrapper.RequestWrapperTween', | |||
r154 | ) | |||
r756 | self.config.add_request_method( | |||
'vcsserver.lib.request_counter.get_request_counter', | ||||
'request_count') | ||||
r154 | ||||
r0 | def wsgi_app(self): | |||
return self.config.make_wsgi_app() | ||||
r768 | def _vcs_view_params(self, request): | |||
r0 | remote = self._remotes[request.matchdict['backend']] | |||
r1069 | payload = msgpack.unpackb(request.body, use_list=True) | |||
r1048 | ||||
r0 | method = payload.get('method') | |||
r768 | params = payload['params'] | |||
r0 | wire = params.get('wire') | |||
args = params.get('args') | ||||
kwargs = params.get('kwargs') | ||||
r483 | context_uid = None | |||
r1078 | request.registry.vcs_call_context = { | |||
'method': method, | ||||
r1123 | 'repo_name': payload.get('_repo_name'), | |||
r1078 | } | |||
r0 | if wire: | |||
try: | ||||
r483 | wire['context'] = context_uid = uuid.UUID(wire['context']) | |||
r0 | except KeyError: | |||
pass | ||||
args.insert(0, wire) | ||||
r768 | repo_state_uid = wire.get('repo_state_uid') if wire else None | |||
r0 | ||||
r742 | # NOTE(marcink): trading complexity for slight performance | |||
if log.isEnabledFor(logging.DEBUG): | ||||
r1073 | # also we SKIP printing out any of those methods args since they maybe excessive | |||
just_args_methods = { | ||||
r1123 | 'commitctx': ('content', 'removed', 'updated'), | |||
'commit': ('content', 'removed', 'updated') | ||||
r1073 | } | |||
if method in just_args_methods: | ||||
skip_args = just_args_methods[method] | ||||
r742 | call_args = '' | |||
r1073 | call_kwargs = {} | |||
for k in kwargs: | ||||
if k in skip_args: | ||||
# replace our skip key with dummy | ||||
call_kwargs[k] = f'RemovedParam({k})' | ||||
else: | ||||
call_kwargs[k] = kwargs[k] | ||||
r742 | else: | |||
call_args = args[1:] | ||||
r1073 | call_kwargs = kwargs | |||
r745 | ||||
r961 | log.debug('Method requested:`%s` with args:%s kwargs:%s context_uid: %s, repo_state_uid:%s', | |||
r1073 | method, call_args, call_kwargs, context_uid, repo_state_uid) | |||
r725 | ||||
r1035 | statsd = request.registry.statsd | |||
if statsd: | ||||
statsd.incr( | ||||
'vcsserver_method_total', tags=[ | ||||
r1114 | f"method:{method}", | |||
r1035 | ]) | |||
r768 | return payload, remote, method, args, kwargs | |||
def vcs_view(self, request): | ||||
payload, remote, method, args, kwargs = self._vcs_view_params(request) | ||||
payload_id = payload.get('id') | ||||
r0 | try: | |||
resp = getattr(remote, method)(*args, **kwargs) | ||||
except Exception as e: | ||||
r491 | exc_info = list(sys.exc_info()) | |||
exc_type, exc_value, exc_traceback = exc_info | ||||
org_exc = getattr(e, '_org_exc', None) | ||||
org_exc_name = None | ||||
r621 | org_exc_tb = '' | |||
r491 | if org_exc: | |||
org_exc_name = org_exc.__class__.__name__ | ||||
r621 | org_exc_tb = getattr(e, '_org_exc_tb', '') | |||
r491 | # replace our "faked" exception with our org | |||
exc_info[0] = org_exc.__class__ | ||||
exc_info[1] = org_exc | ||||
r843 | should_store_exc = True | |||
if org_exc: | ||||
def get_exc_fqn(_exc_obj): | ||||
module_name = getattr(org_exc.__class__, '__module__', 'UNKNOWN') | ||||
return module_name + '.' + org_exc_name | ||||
exc_fqn = get_exc_fqn(org_exc) | ||||
if exc_fqn in ['mercurial.error.RepoLookupError', | ||||
'vcsserver.exceptions.RefNotFoundException']: | ||||
should_store_exc = False | ||||
if should_store_exc: | ||||
r939 | store_exception(id(exc_info), exc_info, request_path=request.path) | |||
r491 | ||||
r1144 | tb_info = format_exc(exc_info) | |||
r146 | ||||
r0 | type_ = e.__class__.__name__ | |||
if type_ not in self.ALLOWED_EXCEPTIONS: | ||||
type_ = None | ||||
resp = { | ||||
r768 | 'id': payload_id, | |||
r0 | 'error': { | |||
r1048 | 'message': str(e), | |||
r146 | 'traceback': tb_info, | |||
r491 | 'org_exc': org_exc_name, | |||
r621 | 'org_exc_tb': org_exc_tb, | |||
r0 | 'type': type_ | |||
} | ||||
} | ||||
r939 | ||||
r0 | try: | |||
r483 | resp['error']['_vcs_kind'] = getattr(e, '_vcs_kind', None) | |||
r0 | except AttributeError: | |||
pass | ||||
else: | ||||
resp = { | ||||
r768 | 'id': payload_id, | |||
r0 | 'result': resp | |||
} | ||||
r1073 | log.debug('Serving data for method %s', method) | |||
r0 | return resp | |||
r768 | def vcs_stream_view(self, request): | |||
payload, remote, method, args, kwargs = self._vcs_view_params(request) | ||||
# this method has a stream: marker we remove it here | ||||
method = method.split('stream:')[-1] | ||||
chunk_size = safe_int(payload.get('chunk_size')) or 4096 | ||||
r1145 | resp = getattr(remote, method)(*args, **kwargs) | |||
r768 | ||||
def get_chunked_data(method_resp): | ||||
r1048 | stream = io.BytesIO(method_resp) | |||
r768 | while 1: | |||
chunk = stream.read(chunk_size) | ||||
if not chunk: | ||||
break | ||||
yield chunk | ||||
response = Response(app_iter=get_chunked_data(resp)) | ||||
response.content_type = 'application/octet-stream' | ||||
return response | ||||
r0 | def status_view(self, request): | |||
r250 | import vcsserver | |||
r1148 | _platform_id = platform.uname()[1] or 'instance' | |||
return { | ||||
"status": "OK", | ||||
r1188 | "vcsserver_version": vcsserver.get_version(), | |||
r1148 | "platform": _platform_id, | |||
"pid": os.getpid(), | ||||
} | ||||
r0 | ||||
r102 | def service_view(self, request): | |||
import vcsserver | ||||
r173 | ||||
r102 | payload = msgpack.unpackb(request.body, use_list=True) | |||
r784 | server_config, app_config = {}, {} | |||
r173 | ||||
try: | ||||
path = self.global_config['__file__'] | ||||
r784 | config = configparser.RawConfigParser() | |||
r173 | config.read(path) | |||
r784 | ||||
if config.has_section('server:main'): | ||||
server_config = dict(config.items('server:main')) | ||||
if config.has_section('app:main'): | ||||
app_config = dict(config.items('app:main')) | ||||
r173 | except Exception: | |||
log.exception('Failed to read .ini file for display') | ||||
r784 | ||||
r983 | environ = list(os.environ.items()) | |||
r173 | ||||
r102 | resp = { | |||
'id': payload.get('id'), | ||||
'result': dict( | ||||
r1188 | version=vcsserver.get_version(), | |||
r784 | config=server_config, | |||
app_config=app_config, | ||||
environ=environ, | ||||
r102 | payload=payload, | |||
) | ||||
} | ||||
return resp | ||||
r0 | def _msgpack_renderer_factory(self, info): | |||
r1089 | ||||
r0 | def _render(value, system): | |||
r1089 | bin_type = False | |||
res = value.get('result') | ||||
r1123 | if isinstance(res, BytesEnvelope): | |||
log.debug('Result is wrapped in BytesEnvelope type') | ||||
bin_type = True | ||||
elif isinstance(res, BinaryEnvelope): | ||||
r1107 | log.debug('Result is wrapped in BinaryEnvelope type') | |||
r1123 | value['result'] = res.val | |||
bin_type = True | ||||
r1089 | ||||
r0 | request = system.get('request') | |||
if request is not None: | ||||
response = request.response | ||||
ct = response.content_type | ||||
if ct == response.default_content_type: | ||||
response.content_type = 'application/x-msgpack' | ||||
r1107 | if bin_type: | |||
response.content_type = 'application/x-msgpack-bin' | ||||
r1069 | ||||
r1089 | return msgpack.packb(value, use_bin_type=bin_type) | |||
r0 | return _render | |||
r269 | def set_env_from_config(self, environ, config): | |||
dict_conf = {} | ||||
try: | ||||
for elem in config: | ||||
if elem[0] == 'rhodecode': | ||||
dict_conf = json.loads(elem[2]) | ||||
break | ||||
except Exception: | ||||
log.exception('Failed to fetch SCM CONFIG') | ||||
return | ||||
username = dict_conf.get('username') | ||||
if username: | ||||
environ['REMOTE_USER'] = username | ||||
r353 | # mercurial specific, some extension api rely on this | |||
environ['HGUSER'] = username | ||||
r269 | ||||
ip = dict_conf.get('ip') | ||||
if ip: | ||||
environ['REMOTE_HOST'] = ip | ||||
r332 | if _is_request_chunked(environ): | |||
# set the compatibility flag for webob | ||||
environ['wsgi.input_terminated'] = True | ||||
r0 | def hg_proxy(self): | |||
@wsgiapp | ||||
def _hg_proxy(environ, start_response): | ||||
app = WsgiProxy(self.remote_wsgi.HgRemoteWsgi()) | ||||
return app(environ, start_response) | ||||
return _hg_proxy | ||||
def git_proxy(self): | ||||
@wsgiapp | ||||
def _git_proxy(environ, start_response): | ||||
app = WsgiProxy(self.remote_wsgi.GitRemoteWsgi()) | ||||
return app(environ, start_response) | ||||
return _git_proxy | ||||
def hg_stream(self): | ||||
if self._use_echo_app: | ||||
@wsgiapp | ||||
def _hg_stream(environ, start_response): | ||||
app = EchoApp('fake_path', 'fake_name', None) | ||||
return app(environ, start_response) | ||||
return _hg_stream | ||||
else: | ||||
@wsgiapp | ||||
def _hg_stream(environ, start_response): | ||||
r247 | log.debug('http-app: handling hg stream') | |||
r1123 | call_context = get_headers_call_context(environ) | |||
r1078 | ||||
repo_path = call_context['repo_path'] | ||||
repo_name = call_context['repo_name'] | ||||
config = call_context['repo_config'] | ||||
r0 | app = scm_app.create_hg_wsgi_app( | |||
repo_path, repo_name, config) | ||||
r269 | # Consistent path information for hgweb | |||
r1078 | environ['PATH_INFO'] = call_context['path_info'] | |||
r0 | environ['REPO_NAME'] = repo_name | |||
r269 | self.set_env_from_config(environ, config) | |||
r247 | log.debug('http-app: starting app handler ' | |||
'with %s and process request', app) | ||||
r0 | return app(environ, ResponseFilter(start_response)) | |||
return _hg_stream | ||||
def git_stream(self): | ||||
if self._use_echo_app: | ||||
@wsgiapp | ||||
def _git_stream(environ, start_response): | ||||
app = EchoApp('fake_path', 'fake_name', None) | ||||
return app(environ, start_response) | ||||
return _git_stream | ||||
else: | ||||
@wsgiapp | ||||
def _git_stream(environ, start_response): | ||||
r247 | log.debug('http-app: handling git stream') | |||
r1078 | ||||
r1123 | call_context = get_headers_call_context(environ) | |||
r0 | ||||
r1078 | repo_path = call_context['repo_path'] | |||
repo_name = call_context['repo_name'] | ||||
config = call_context['repo_config'] | ||||
environ['PATH_INFO'] = call_context['path_info'] | ||||
r269 | self.set_env_from_config(environ, config) | |||
r180 | content_type = environ.get('CONTENT_TYPE', '') | |||
path = environ['PATH_INFO'] | ||||
is_lfs_request = GIT_LFS_CONTENT_TYPE in content_type | ||||
log.debug( | ||||
'LFS: Detecting if request `%s` is LFS server path based ' | ||||
'on content type:`%s`, is_lfs:%s', | ||||
path, content_type, is_lfs_request) | ||||
if not is_lfs_request: | ||||
# fallback detection by path | ||||
if GIT_LFS_PROTO_PAT.match(path): | ||||
is_lfs_request = True | ||||
log.debug( | ||||
'LFS: fallback detection by path of: `%s`, is_lfs:%s', | ||||
path, is_lfs_request) | ||||
if is_lfs_request: | ||||
app = scm_app.create_git_lfs_wsgi_app( | ||||
repo_path, repo_name, config) | ||||
else: | ||||
app = scm_app.create_git_wsgi_app( | ||||
repo_path, repo_name, config) | ||||
r247 | ||||
log.debug('http-app: starting app handler ' | ||||
'with %s and process request', app) | ||||
r332 | ||||
r0 | return app(environ, start_response) | |||
r180 | ||||
r0 | return _git_stream | |||
|
r85 | def handle_vcs_exception(self, exception, request): | ||
r178 | _vcs_kind = getattr(exception, '_vcs_kind', '') | |||
r1123 | ||||
r178 | if _vcs_kind == 'repo_locked': | |||
r1123 | headers_call_context = get_headers_call_context(request.environ) | |||
status_code = safe_int(headers_call_context['locked_status_code']) | ||||
|
r85 | return HTTPRepoLocked( | ||
r1123 | title=str(exception), status_code=status_code, headers=[('X-Rc-Locked', '1')]) | |||
r491 | ||||
r509 | elif _vcs_kind == 'repo_branch_protected': | |||
# Get custom repo-branch-protected status code if present. | ||||
r1123 | return HTTPRepoBranchProtected( | |||
title=str(exception), headers=[('X-Rc-Branch-Protection', '1')]) | ||||
r509 | ||||
r491 | exc_info = request.exc_info | |||
store_exception(id(exc_info), exc_info) | ||||
r478 | traceback_info = 'unavailable' | |||
if request.exc_info: | ||||
r1144 | traceback_info = format_exc(request.exc_info) | |||
|
r85 | |||
r478 | log.error( | |||
r1144 | 'error occurred handling this request for path: %s, \n%s', | |||
r478 | request.path, traceback_info) | |||
r1005 | ||||
statsd = request.registry.statsd | ||||
if statsd: | ||||
r1114 | exc_type = f"{exception.__class__.__module__}.{exception.__class__.__name__}" | |||
r1014 | statsd.incr('vcsserver_exception_total', | |||
r1114 | tags=[f"type:{exc_type}"]) | |||
r150 | raise exception | |||
r0 | ||||
r1152 | class ResponseFilter: | |||
r0 | ||||
def __init__(self, start_response): | ||||
self._start_response = start_response | ||||
def __call__(self, status, response_headers, exc_info=None): | ||||
headers = tuple( | ||||
(h, v) for h, v in response_headers | ||||
if not wsgiref.util.is_hop_by_hop(h)) | ||||
return self._start_response(status, headers, exc_info) | ||||
r1021 | def sanitize_settings_and_apply_defaults(global_config, settings): | |||
r1145 | _global_settings_maker = SettingsMaker(global_config) | |||
r1021 | settings_maker = SettingsMaker(settings) | |||
r1029 | settings_maker.make_setting('logging.autoconfigure', False, parser='bool') | |||
r1021 | ||||
logging_conf = os.path.join(os.path.dirname(global_config.get('__file__')), 'logging.ini') | ||||
settings_maker.enable_logging(logging_conf) | ||||
# Default includes, possible to change as a user | ||||
pyramid_includes = settings_maker.make_setting('pyramid.includes', [], parser='list:newline') | ||||
r1023 | log.debug("Using the following pyramid.includes: %s", pyramid_includes) | |||
r1021 | ||||
settings_maker.make_setting('__file__', global_config.get('__file__')) | ||||
r1023 | settings_maker.make_setting('pyramid.default_locale_name', 'en') | |||
settings_maker.make_setting('locale', 'en_US.UTF-8') | ||||
r1021 | ||||
r1217 | settings_maker.make_setting( | |||
'core.binary_dir', '/usr/local/bin/rhodecode_bin/vcs_bin', | ||||
default_when_empty=True, parser='string:noquote') | ||||
r1021 | ||||
temp_store = tempfile.gettempdir() | ||||
default_cache_dir = os.path.join(temp_store, 'rc_cache') | ||||
# save default, cache dir, and use it for all backends later. | ||||
default_cache_dir = settings_maker.make_setting( | ||||
'cache_dir', | ||||
default=default_cache_dir, default_when_empty=True, | ||||
parser='dir:ensured') | ||||
# exception store cache | ||||
settings_maker.make_setting( | ||||
'exception_tracker.store_path', | ||||
default=os.path.join(default_cache_dir, 'exc_store'), default_when_empty=True, | ||||
parser='dir:ensured' | ||||
) | ||||
# repo_object cache defaults | ||||
settings_maker.make_setting( | ||||
'rc_cache.repo_object.backend', | ||||
default='dogpile.cache.rc.file_namespace', | ||||
parser='string') | ||||
settings_maker.make_setting( | ||||
'rc_cache.repo_object.expiration_time', | ||||
default=30 * 24 * 60 * 60, # 30days | ||||
parser='int') | ||||
r1023 | settings_maker.make_setting( | |||
r1021 | 'rc_cache.repo_object.arguments.filename', | |||
default=os.path.join(default_cache_dir, 'vcsserver_cache_repo_object.db'), | ||||
parser='string') | ||||
# statsd | ||||
r1023 | settings_maker.make_setting('statsd.enabled', False, parser='bool') | |||
settings_maker.make_setting('statsd.statsd_host', 'statsd-exporter', parser='string') | ||||
settings_maker.make_setting('statsd.statsd_port', 9125, parser='int') | ||||
settings_maker.make_setting('statsd.statsd_prefix', '') | ||||
settings_maker.make_setting('statsd.statsd_ipv6', False, parser='bool') | ||||
settings_maker.env_expand() | ||||
r1021 | ||||
r0 | def main(global_config, **settings): | |||
r1021 | start_time = time.time() | |||
log.info('Pyramid app config starting') | ||||
|
r35 | if MercurialFactory: | ||
hgpatches.patch_largefiles_capabilities() | ||||
|
r100 | hgpatches.patch_subrepo_type_mapping() | ||
r483 | ||||
r1021 | # Fill in and sanitize the defaults & do ENV expansion | |||
sanitize_settings_and_apply_defaults(global_config, settings) | ||||
r1005 | # init and bootstrap StatsdClient | |||
StatsdClient.setup(settings) | ||||
r1021 | pyramid_app = HTTPApplication(settings=settings, global_config=global_config).wsgi_app() | |||
total_time = time.time() - start_time | ||||
r1125 | log.info('Pyramid app created and configured in %.2fs', total_time) | |||
r1021 | return pyramid_app | |||