##// END OF EJS Templates
core: synced vendor/ext_json with ce for better compatability
core: synced vendor/ext_json with ce for better compatability

File last commit:

r1249:5745b11f default
r1250:2c57bb5b default
Show More
http_main.py
774 lines | 26.2 KiB | text/x-python | PythonLexer
initial commit
r0 # RhodeCode VCSServer provides access to different vcs backends via network.
source-code: updated copyrights to 2023
r1126 # Copyright (C) 2014-2023 RhodeCode GmbH
initial commit
r0 #
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software Foundation,
# Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
python3: code change for py3 support...
r1048 import io
http: status will return PID to help check stuck workers.
r372 import os
status: expose platform info usefull to debug docker LB issues
r1148 import platform
exceptions: allow error tracking and exception storage on vcsserver.
r491 import sys
initial commit
r0 import locale
import logging
import uuid
core: re-implemented the way how configuration can be made...
r1021 import time
initial commit
r0 import wsgiref.util
exc_store: allow to specify a custom path for exception store.
r519 import tempfile
vcsserver: use safer maxfd reporting, some linux systems get a problem with this
r913 import psutil
core: re-implemented the way how configuration can be made...
r1021
initial commit
r0 from itertools import chain
import msgpack
py3: remove compat module usage
r1038 import configparser
merge: with lataest default branch
r1041
initial commit
r0 from pyramid.config import Configurator
from pyramid.wsgi import wsgiapp
dan
vcsserver: added streaming interface for streaming remote attributes
r768 from pyramid.response import Response
initial commit
r0
http-app: refactor new usage of main http api endpoints
r1123 from vcsserver.base import BytesEnvelope, BinaryEnvelope
chore(refactor): renamed rc_json to ext_json for ce compat
r1243 from vcsserver.lib.ext_json import json
core: re-implemented the way how configuration can be made...
r1021 from vcsserver.config.settings_maker import SettingsMaker
core: moved str_utils and type_utils to lib so it's consistent with ce, and easier to sync up codebases
r1249 from vcsserver.lib.str_utils import safe_int
metrics: use new statsd client logic, and start gathering new metrics
r1005 from vcsserver.lib.statsd_client import StatsdClient
core: renamed remote packages to prevent conflicts with builtin libraries like svn core library called same as svn remote
r1145 from vcsserver.tweens.request_wrapper import get_headers_call_context
core: re-implemented the way how configuration can be made...
r1021
exc_store: allow to specify a custom path for exception store.
r519 import vcsserver
Martin Bornhold
hg: Include mercurial patching when using the http app.
r35 from vcsserver import remote_wsgi, scm_app, settings, hgpatches
git-lfs: added vcsserver handling of git-lfs objects....
r180 from vcsserver.git_lfs.app import GIT_LFS_CONTENT_TYPE, GIT_LFS_PROTO_PAT
initial commit
r0 from vcsserver.echo_stub import remote_wsgi as remote_wsgi_stub
from vcsserver.echo_stub.echo_app import EchoApp
hooks: adjust to handle protected branch cases, and force push.
r509 from vcsserver.exceptions import HTTPRepoLocked, HTTPRepoBranchProtected
exc-tracking: use more rich style tracebacks.
r1144 from vcsserver.lib.exc_tracking import store_exception, format_exc
initial commit
r0 from vcsserver.server import VcsServer
py3: update mercurial changes for 6.3 release
r1039 strict_vcs = True
init: better error reporting on failed components
r988
git_import_err = None
initial commit
r0 try:
core: renamed remote packages to prevent conflicts with builtin libraries like svn core library called same as svn remote
r1145 from vcsserver.remote.git_remote import GitFactory, GitRemote
init: better error reporting on failed components
r988 except ImportError as e:
initial commit
r0 GitFactory = None
GitRemote = None
init: better error reporting on failed components
r988 git_import_err = e
py3: update mercurial changes for 6.3 release
r1039 if strict_vcs:
raise
git-lfs: added vcsserver handling of git-lfs objects....
r180
init: better error reporting on failed components
r988 hg_import_err = None
initial commit
r0 try:
core: renamed remote packages to prevent conflicts with builtin libraries like svn core library called same as svn remote
r1145 from vcsserver.remote.hg_remote import MercurialFactory, HgRemote
init: better error reporting on failed components
r988 except ImportError as e:
initial commit
r0 MercurialFactory = None
HgRemote = None
init: better error reporting on failed components
r988 hg_import_err = e
py3: update mercurial changes for 6.3 release
r1039 if strict_vcs:
raise
git-lfs: added vcsserver handling of git-lfs objects....
r180
init: better error reporting on failed components
r988 svn_import_err = None
initial commit
r0 try:
core: renamed remote packages to prevent conflicts with builtin libraries like svn core library called same as svn remote
r1145 from vcsserver.remote.svn_remote import SubversionFactory, SvnRemote
init: better error reporting on failed components
r988 except ImportError as e:
initial commit
r0 SubversionFactory = None
SvnRemote = None
init: better error reporting on failed components
r988 svn_import_err = e
py3: update mercurial changes for 6.3 release
r1039 if strict_vcs:
raise
initial commit
r0
core: renamed remote packages to prevent conflicts with builtin libraries like svn core library called same as svn remote
r1145 log = logging.getLogger(__name__)
# due to Mercurial/glibc2.27 problems we need to detect if locale settings are
# causing problems and "fix" it in case they do and fallback to LC_ALL = C
try:
locale.setlocale(locale.LC_ALL, '')
except locale.Error as e:
log.error(
'LOCALE ERROR: failed to set LC_ALL, fallback to LC_ALL=C, org error: %s', e)
os.environ['LC_ALL'] = 'C'
locale: use a hacky way to catch the locale set error. We rather "cleanup" the env ourselfs...
r507
stream: use wsgi.input_terminated because of webob changes of handling chunked encoding
r332 def _is_request_chunked(environ):
stream = environ.get('HTTP_TRANSFER_ENCODING', '') == 'chunked'
return stream
vcsserver: use safer maxfd reporting, some linux systems get a problem with this
r913 def log_max_fd():
try:
maxfd = psutil.Process().rlimit(psutil.RLIMIT_NOFILE)[1]
log.info('Max file descriptors value: %s', maxfd)
except Exception:
pass
lint: auto-fixes
r1152 class VCS:
dan
vcsserver: added streaming interface for streaming remote attributes
r768 def __init__(self, locale_conf=None, cache_config=None):
self.locale = locale_conf
initial commit
r0 self.cache_config = cache_config
self._configure_locale()
vcsserver: use safer maxfd reporting, some linux systems get a problem with this
r913 log_max_fd()
vcsserver: report set File Descriptors.
r893
initial commit
r0 if GitFactory and GitRemote:
caches: replaced beaker with dogpile cache.
r483 git_factory = GitFactory()
initial commit
r0 self._git_remote = GitRemote(git_factory)
else:
init: better error reporting on failed components
r988 log.error("Git client import failed: %s", git_import_err)
initial commit
r0
if MercurialFactory and HgRemote:
caches: replaced beaker with dogpile cache.
r483 hg_factory = MercurialFactory()
initial commit
r0 self._hg_remote = HgRemote(hg_factory)
else:
init: better error reporting on failed components
r988 log.error("Mercurial client import failed: %s", hg_import_err)
initial commit
r0
if SubversionFactory and SvnRemote:
caches: replaced beaker with dogpile cache.
r483 svn_factory = SubversionFactory()
svn: added support for hooks management of git and subversion....
r407 # hg factory is used for svn url validation
caches: replaced beaker with dogpile cache.
r483 hg_factory = MercurialFactory()
initial commit
r0 self._svn_remote = SvnRemote(svn_factory, hg_factory=hg_factory)
else:
init: better error reporting on failed components
r988 log.error("Subversion client import failed: %s", svn_import_err)
initial commit
r0
self._vcsserver = VcsServer()
def _configure_locale(self):
if self.locale:
logging: use lazy formatting of log entries
r541 log.info('Settings locale: `LC_ALL` to %s', self.locale)
initial commit
r0 else:
init: better error reporting on failed components
r988 log.info('Configuring locale subsystem based on environment variables')
initial commit
r0 try:
# If self.locale is the empty string, then the locale
# module will use the environment variables. See the
# documentation of the package `locale`.
locale.setlocale(locale.LC_ALL, self.locale)
language_code, encoding = locale.getlocale()
log.info(
'Locale set to language code "%s" with encoding "%s".',
language_code, encoding)
except locale.Error:
init: better error reporting on failed components
r988 log.exception('Cannot set locale, not configuring the locale system')
initial commit
r0
lint: auto-fixes
r1152 class WsgiProxy:
initial commit
r0 def __init__(self, wsgi):
self.wsgi = wsgi
def __call__(self, environ, start_response):
input_data = environ['wsgi.input'].read()
input_data = msgpack.unpackb(input_data)
error = None
try:
data, status, headers = self.wsgi.handle(
input_data['environment'], input_data['input_data'],
*input_data['args'], **input_data['kwargs'])
except Exception as e:
data, status, headers = [], None, None
error = {
'message': str(e),
'_vcs_kind': getattr(e, '_vcs_kind', None)
}
start_response(200, {})
return self._iterator(error, status, headers, data)
def _iterator(self, error, status, headers, data):
initial_data = [
error,
status,
headers,
]
for d in chain(initial_data, data):
yield msgpack.packb(d)
pyramid: don't use deprecated custom_predicates in view config
r583 def not_found(request):
return {'status': '404 NOT FOUND'}
lint: auto-fixes
r1152 class VCSViewPredicate:
pyramid: don't use deprecated custom_predicates in view config
r583 def __init__(self, val, config):
self.remotes = val
def text(self):
python3: fixes and code optimization for python3.11
r1114 return f'vcs view method = {list(self.remotes.keys())}'
pyramid: don't use deprecated custom_predicates in view config
r583
phash = text
def __call__(self, context, request):
"""
View predicate that returns true if given backend is supported by
defined remotes.
"""
backend = request.matchdict.get('backend')
return backend in self.remotes
lint: auto-fixes
r1152 class HTTPApplication:
initial commit
r0 ALLOWED_EXCEPTIONS = ('KeyError', 'URLError')
remote_wsgi = remote_wsgi
_use_echo_app = False
service-endpoint: expose additional data of vcsserver via service-data endpoint....
r173 def __init__(self, settings=None, global_config=None):
caches: replaced beaker with dogpile cache.
r483
initial commit
r0 self.config = Configurator(settings=settings)
metrics: use new statsd client logic, and start gathering new metrics
r1005 # Init our statsd at very start
self.config.registry.statsd = StatsdClient.statsd
http-traffic: mostly use payload from encoded msgpack single header instead of multiple ones....
r1078 self.config.registry.vcs_call_context = {}
metrics: use new statsd client logic, and start gathering new metrics
r1005
service-endpoint: expose additional data of vcsserver via service-data endpoint....
r173 self.global_config = global_config
caches: replaced beaker with dogpile cache.
r483 self.config.include('vcsserver.lib.rc_cache')
http-app: refactor new usage of main http api endpoints
r1123 self.config.include('vcsserver.lib.rc_cache.archive_cache')
service-endpoint: expose additional data of vcsserver via service-data endpoint....
r173
app: small code cleanups
r743 settings_locale = settings.get('locale', '') or 'en_US.UTF-8'
dan
vcsserver: added streaming interface for streaming remote attributes
r768 vcs = VCS(locale_conf=settings_locale, cache_config=settings)
initial commit
r0 self._remotes = {
'hg': vcs._hg_remote,
'git': vcs._git_remote,
'svn': vcs._svn_remote,
'server': vcs._vcsserver,
}
if settings.get('dev.use_echo_app', 'false').lower() == 'true':
self._use_echo_app = True
log.warning("Using EchoApp for VCS operations.")
self.remote_wsgi = remote_wsgi_stub
exc_store: allow to specify a custom path for exception store.
r519
self._configure_settings(global_config, settings)
application: added statsd client for sending usage statistics.
r920
initial commit
r0 self._configure()
exc_store: allow to specify a custom path for exception store.
r519 def _configure_settings(self, global_config, app_settings):
initial commit
r0 """
Configure the settings module.
"""
exc_store: allow to specify a custom path for exception store.
r519 settings_merged = global_config.copy()
settings_merged.update(app_settings)
fix(svn-hooks): fixed problem with svn subprocess execution fixes RCCE-62
r1214 binary_dir = app_settings['core.binary_dir']
settings.BINARY_DIR = binary_dir
exc_store: allow to specify a custom path for exception store.
r519 # Store the settings to make them available to other modules.
vcsserver.PYRAMID_SETTINGS = settings_merged
vcsserver.CONFIG = settings_merged
initial commit
r0 def _configure(self):
pyramid: don't use deprecated custom_predicates in view config
r583 self.config.add_renderer(name='msgpack', factory=self._msgpack_renderer_factory)
initial commit
r0
http: added service backend for testing communication, and extracting vcsserver version...
r102 self.config.add_route('service', '/_service')
initial commit
r0 self.config.add_route('status', '/status')
self.config.add_route('hg_proxy', '/proxy/hg')
self.config.add_route('git_proxy', '/proxy/git')
dan
vcsserver: added streaming interface for streaming remote attributes
r768
# rpc methods
initial commit
r0 self.config.add_route('vcs', '/{backend}')
dan
vcsserver: added streaming interface for streaming remote attributes
r768
# streaming rpc remote methods
self.config.add_route('vcs_stream', '/{backend}/stream')
# vcs operations clone/push as streaming
initial commit
r0 self.config.add_route('stream_git', '/stream/git/*repo_name')
self.config.add_route('stream_hg', '/stream/hg/*repo_name')
pyramid: don't use deprecated custom_predicates in view config
r583 self.config.add_view(self.status_view, route_name='status', renderer='json')
self.config.add_view(self.service_view, route_name='service', renderer='msgpack')
http: added service backend for testing communication, and extracting vcsserver version...
r102
initial commit
r0 self.config.add_view(self.hg_proxy(), route_name='hg_proxy')
self.config.add_view(self.git_proxy(), route_name='git_proxy')
pyramid: don't use deprecated custom_predicates in view config
r583 self.config.add_view(self.vcs_view, route_name='vcs', renderer='msgpack',
vcs_view=self._remotes)
dan
vcsserver: added streaming interface for streaming remote attributes
r768 self.config.add_view(self.vcs_stream_view, route_name='vcs_stream',
vcs_view=self._remotes)
initial commit
r0
self.config.add_view(self.hg_stream(), route_name='stream_hg')
self.config.add_view(self.git_stream(), route_name='stream_git')
http-app: added default not_found view.
r151
pyramid: don't use deprecated custom_predicates in view config
r583 self.config.add_view_predicate('vcs_view', VCSViewPredicate)
self.config.add_notfound_view(not_found, renderer='json')
http-app: added default not_found view.
r151
exceptions: use only one exception handler to track errors with nicer...
r178 self.config.add_view(self.handle_vcs_exception, context=Exception)
http: added catch-all exception handler to show errors in logs.
r150
vcsserver: http main added time measuring tween for debugging.
r154 self.config.add_tween(
request-wrapper: set custom module for better logging.
r744 'vcsserver.tweens.request_wrapper.RequestWrapperTween',
vcsserver: http main added time measuring tween for debugging.
r154 )
request-wrapper: add request counter.
r756 self.config.add_request_method(
'vcsserver.lib.request_counter.get_request_counter',
'request_count')
vcsserver: http main added time measuring tween for debugging.
r154
initial commit
r0 def wsgi_app(self):
return self.config.make_wsgi_app()
dan
vcsserver: added streaming interface for streaming remote attributes
r768 def _vcs_view_params(self, request):
initial commit
r0 remote = self._remotes[request.matchdict['backend']]
deps: bumped msgpack to latest release and by default use byte only protocol
r1069 payload = msgpack.unpackb(request.body, use_list=True)
python3: code change for py3 support...
r1048
initial commit
r0 method = payload.get('method')
dan
vcsserver: added streaming interface for streaming remote attributes
r768 params = payload['params']
initial commit
r0 wire = params.get('wire')
args = params.get('args')
kwargs = params.get('kwargs')
caches: replaced beaker with dogpile cache.
r483 context_uid = None
http-traffic: mostly use payload from encoded msgpack single header instead of multiple ones....
r1078 request.registry.vcs_call_context = {
'method': method,
http-app: refactor new usage of main http api endpoints
r1123 'repo_name': payload.get('_repo_name'),
http-traffic: mostly use payload from encoded msgpack single header instead of multiple ones....
r1078 }
initial commit
r0 if wire:
try:
caches: replaced beaker with dogpile cache.
r483 wire['context'] = context_uid = uuid.UUID(wire['context'])
initial commit
r0 except KeyError:
pass
args.insert(0, wire)
dan
vcsserver: added streaming interface for streaming remote attributes
r768 repo_state_uid = wire.get('repo_state_uid') if wire else None
initial commit
r0
logging: skip large attribute expansion on archive_repo. We don't need arguments there
r742 # NOTE(marcink): trading complexity for slight performance
if log.isEnabledFor(logging.DEBUG):
logging: skip certain very large args/kwargs reporting in logs of called methods
r1073 # also we SKIP printing out any of those methods args since they maybe excessive
just_args_methods = {
http-app: refactor new usage of main http api endpoints
r1123 'commitctx': ('content', 'removed', 'updated'),
'commit': ('content', 'removed', 'updated')
logging: skip certain very large args/kwargs reporting in logs of called methods
r1073 }
if method in just_args_methods:
skip_args = just_args_methods[method]
logging: skip large attribute expansion on archive_repo. We don't need arguments there
r742 call_args = ''
logging: skip certain very large args/kwargs reporting in logs of called methods
r1073 call_kwargs = {}
for k in kwargs:
if k in skip_args:
# replace our skip key with dummy
call_kwargs[k] = f'RemovedParam({k})'
else:
call_kwargs[k] = kwargs[k]
logging: skip large attribute expansion on archive_repo. We don't need arguments there
r742 else:
call_args = args[1:]
logging: skip certain very large args/kwargs reporting in logs of called methods
r1073 call_kwargs = kwargs
http-proto: added some more logging.
r745
caches: allow regional per repo caches, and invalidate caches via a remote call.
r961 log.debug('Method requested:`%s` with args:%s kwargs:%s context_uid: %s, repo_state_uid:%s',
logging: skip certain very large args/kwargs reporting in logs of called methods
r1073 method, call_args, call_kwargs, context_uid, repo_state_uid)
git: switched most git operations to libgit2
r725
metrics: statsd report vcs methods use even on debug logs disabled
r1035 statsd = request.registry.statsd
if statsd:
statsd.incr(
'vcsserver_method_total', tags=[
python3: fixes and code optimization for python3.11
r1114 f"method:{method}",
metrics: statsd report vcs methods use even on debug logs disabled
r1035 ])
dan
vcsserver: added streaming interface for streaming remote attributes
r768 return payload, remote, method, args, kwargs
def vcs_view(self, request):
payload, remote, method, args, kwargs = self._vcs_view_params(request)
payload_id = payload.get('id')
initial commit
r0 try:
resp = getattr(remote, method)(*args, **kwargs)
except Exception as e:
exceptions: allow error tracking and exception storage on vcsserver.
r491 exc_info = list(sys.exc_info())
exc_type, exc_value, exc_traceback = exc_info
org_exc = getattr(e, '_org_exc', None)
org_exc_name = None
exception: store orginal tb and exc inside the new exception passed to rhodecode from vcsserver.
r621 org_exc_tb = ''
exceptions: allow error tracking and exception storage on vcsserver.
r491 if org_exc:
org_exc_name = org_exc.__class__.__name__
exception: store orginal tb and exc inside the new exception passed to rhodecode from vcsserver.
r621 org_exc_tb = getattr(e, '_org_exc_tb', '')
exceptions: allow error tracking and exception storage on vcsserver.
r491 # replace our "faked" exception with our org
exc_info[0] = org_exc.__class__
exc_info[1] = org_exc
exceptions: don't report lookup errors as exceptions stored in the exception store....
r843 should_store_exc = True
if org_exc:
def get_exc_fqn(_exc_obj):
module_name = getattr(org_exc.__class__, '__module__', 'UNKNOWN')
return module_name + '.' + org_exc_name
exc_fqn = get_exc_fqn(org_exc)
if exc_fqn in ['mercurial.error.RepoLookupError',
'vcsserver.exceptions.RefNotFoundException']:
should_store_exc = False
if should_store_exc:
vcsserver: log exceptions into the logs
r939 store_exception(id(exc_info), exc_info, request_path=request.path)
exceptions: allow error tracking and exception storage on vcsserver.
r491
exc-tracking: use more rich style tracebacks.
r1144 tb_info = format_exc(exc_info)
http-mode: expose tracebacks back to the client.
r146
initial commit
r0 type_ = e.__class__.__name__
if type_ not in self.ALLOWED_EXCEPTIONS:
type_ = None
resp = {
dan
vcsserver: added streaming interface for streaming remote attributes
r768 'id': payload_id,
initial commit
r0 'error': {
python3: code change for py3 support...
r1048 'message': str(e),
http-mode: expose tracebacks back to the client.
r146 'traceback': tb_info,
exceptions: allow error tracking and exception storage on vcsserver.
r491 'org_exc': org_exc_name,
exception: store orginal tb and exc inside the new exception passed to rhodecode from vcsserver.
r621 'org_exc_tb': org_exc_tb,
initial commit
r0 'type': type_
}
}
vcsserver: log exceptions into the logs
r939
initial commit
r0 try:
caches: replaced beaker with dogpile cache.
r483 resp['error']['_vcs_kind'] = getattr(e, '_vcs_kind', None)
initial commit
r0 except AttributeError:
pass
else:
resp = {
dan
vcsserver: added streaming interface for streaming remote attributes
r768 'id': payload_id,
initial commit
r0 'result': resp
}
logging: skip certain very large args/kwargs reporting in logs of called methods
r1073 log.debug('Serving data for method %s', method)
initial commit
r0 return resp
dan
vcsserver: added streaming interface for streaming remote attributes
r768 def vcs_stream_view(self, request):
payload, remote, method, args, kwargs = self._vcs_view_params(request)
# this method has a stream: marker we remove it here
method = method.split('stream:')[-1]
chunk_size = safe_int(payload.get('chunk_size')) or 4096
core: renamed remote packages to prevent conflicts with builtin libraries like svn core library called same as svn remote
r1145 resp = getattr(remote, method)(*args, **kwargs)
dan
vcsserver: added streaming interface for streaming remote attributes
r768
def get_chunked_data(method_resp):
python3: code change for py3 support...
r1048 stream = io.BytesIO(method_resp)
dan
vcsserver: added streaming interface for streaming remote attributes
r768 while 1:
chunk = stream.read(chunk_size)
if not chunk:
break
yield chunk
response = Response(app_iter=get_chunked_data(resp))
response.content_type = 'application/octet-stream'
return response
initial commit
r0 def status_view(self, request):
vcsserver-status: report version for easier debugging.
r250 import vcsserver
status: expose platform info usefull to debug docker LB issues
r1148 _platform_id = platform.uname()[1] or 'instance'
return {
"status": "OK",
fix(core): don't rely on pkgutil to extract the version info of project
r1188 "vcsserver_version": vcsserver.get_version(),
status: expose platform info usefull to debug docker LB issues
r1148 "platform": _platform_id,
"pid": os.getpid(),
}
initial commit
r0
http: added service backend for testing communication, and extracting vcsserver version...
r102 def service_view(self, request):
import vcsserver
service-endpoint: expose additional data of vcsserver via service-data endpoint....
r173
http: added service backend for testing communication, and extracting vcsserver version...
r102 payload = msgpack.unpackb(request.body, use_list=True)
service: expose config so enterprise can read it.
r784 server_config, app_config = {}, {}
service-endpoint: expose additional data of vcsserver via service-data endpoint....
r173
try:
path = self.global_config['__file__']
service: expose config so enterprise can read it.
r784 config = configparser.RawConfigParser()
service-endpoint: expose additional data of vcsserver via service-data endpoint....
r173 config.read(path)
service: expose config so enterprise can read it.
r784
if config.has_section('server:main'):
server_config = dict(config.items('server:main'))
if config.has_section('app:main'):
app_config = dict(config.items('app:main'))
service-endpoint: expose additional data of vcsserver via service-data endpoint....
r173 except Exception:
log.exception('Failed to read .ini file for display')
service: expose config so enterprise can read it.
r784
py3: fix http_main changes for py3
r983 environ = list(os.environ.items())
service-endpoint: expose additional data of vcsserver via service-data endpoint....
r173
http: added service backend for testing communication, and extracting vcsserver version...
r102 resp = {
'id': payload.get('id'),
'result': dict(
fix(core): don't rely on pkgutil to extract the version info of project
r1188 version=vcsserver.get_version(),
service: expose config so enterprise can read it.
r784 config=server_config,
app_config=app_config,
environ=environ,
http: added service backend for testing communication, and extracting vcsserver version...
r102 payload=payload,
)
}
return resp
initial commit
r0 def _msgpack_renderer_factory(self, info):
protocol: introduced binaryEnvelope to return raw bytes via msgpack
r1089
initial commit
r0 def _render(value, system):
protocol: introduced binaryEnvelope to return raw bytes via msgpack
r1089 bin_type = False
res = value.get('result')
http-app: refactor new usage of main http api endpoints
r1123 if isinstance(res, BytesEnvelope):
log.debug('Result is wrapped in BytesEnvelope type')
bin_type = True
elif isinstance(res, BinaryEnvelope):
binary envelope: add support for custom response type when using wrapped binary data in envelope
r1107 log.debug('Result is wrapped in BinaryEnvelope type')
http-app: refactor new usage of main http api endpoints
r1123 value['result'] = res.val
bin_type = True
protocol: introduced binaryEnvelope to return raw bytes via msgpack
r1089
initial commit
r0 request = system.get('request')
if request is not None:
response = request.response
ct = response.content_type
if ct == response.default_content_type:
response.content_type = 'application/x-msgpack'
binary envelope: add support for custom response type when using wrapped binary data in envelope
r1107 if bin_type:
response.content_type = 'application/x-msgpack-bin'
deps: bumped msgpack to latest release and by default use byte only protocol
r1069
protocol: introduced binaryEnvelope to return raw bytes via msgpack
r1089 return msgpack.packb(value, use_bin_type=bin_type)
initial commit
r0 return _render
http: set REMOTE_USER and REMOTE_HOST http variables....
r269 def set_env_from_config(self, environ, config):
dict_conf = {}
try:
for elem in config:
if elem[0] == 'rhodecode':
dict_conf = json.loads(elem[2])
break
except Exception:
log.exception('Failed to fetch SCM CONFIG')
return
username = dict_conf.get('username')
if username:
environ['REMOTE_USER'] = username
mercurial: expose HGUSER into environ becuase some extensions explicitly rely on this....
r353 # mercurial specific, some extension api rely on this
environ['HGUSER'] = username
http: set REMOTE_USER and REMOTE_HOST http variables....
r269
ip = dict_conf.get('ip')
if ip:
environ['REMOTE_HOST'] = ip
stream: use wsgi.input_terminated because of webob changes of handling chunked encoding
r332 if _is_request_chunked(environ):
# set the compatibility flag for webob
environ['wsgi.input_terminated'] = True
initial commit
r0 def hg_proxy(self):
@wsgiapp
def _hg_proxy(environ, start_response):
app = WsgiProxy(self.remote_wsgi.HgRemoteWsgi())
return app(environ, start_response)
return _hg_proxy
def git_proxy(self):
@wsgiapp
def _git_proxy(environ, start_response):
app = WsgiProxy(self.remote_wsgi.GitRemoteWsgi())
return app(environ, start_response)
return _git_proxy
def hg_stream(self):
if self._use_echo_app:
@wsgiapp
def _hg_stream(environ, start_response):
app = EchoApp('fake_path', 'fake_name', None)
return app(environ, start_response)
return _hg_stream
else:
@wsgiapp
def _hg_stream(environ, start_response):
logs: added some added logging for stream hg/git.
r247 log.debug('http-app: handling hg stream')
http-app: refactor new usage of main http api endpoints
r1123 call_context = get_headers_call_context(environ)
http-traffic: mostly use payload from encoded msgpack single header instead of multiple ones....
r1078
repo_path = call_context['repo_path']
repo_name = call_context['repo_name']
config = call_context['repo_config']
initial commit
r0 app = scm_app.create_hg_wsgi_app(
repo_path, repo_name, config)
http: set REMOTE_USER and REMOTE_HOST http variables....
r269 # Consistent path information for hgweb
http-traffic: mostly use payload from encoded msgpack single header instead of multiple ones....
r1078 environ['PATH_INFO'] = call_context['path_info']
initial commit
r0 environ['REPO_NAME'] = repo_name
http: set REMOTE_USER and REMOTE_HOST http variables....
r269 self.set_env_from_config(environ, config)
logs: added some added logging for stream hg/git.
r247 log.debug('http-app: starting app handler '
'with %s and process request', app)
initial commit
r0 return app(environ, ResponseFilter(start_response))
return _hg_stream
def git_stream(self):
if self._use_echo_app:
@wsgiapp
def _git_stream(environ, start_response):
app = EchoApp('fake_path', 'fake_name', None)
return app(environ, start_response)
return _git_stream
else:
@wsgiapp
def _git_stream(environ, start_response):
logs: added some added logging for stream hg/git.
r247 log.debug('http-app: handling git stream')
http-traffic: mostly use payload from encoded msgpack single header instead of multiple ones....
r1078
http-app: refactor new usage of main http api endpoints
r1123 call_context = get_headers_call_context(environ)
initial commit
r0
http-traffic: mostly use payload from encoded msgpack single header instead of multiple ones....
r1078 repo_path = call_context['repo_path']
repo_name = call_context['repo_name']
config = call_context['repo_config']
environ['PATH_INFO'] = call_context['path_info']
http: set REMOTE_USER and REMOTE_HOST http variables....
r269 self.set_env_from_config(environ, config)
git-lfs: added vcsserver handling of git-lfs objects....
r180 content_type = environ.get('CONTENT_TYPE', '')
path = environ['PATH_INFO']
is_lfs_request = GIT_LFS_CONTENT_TYPE in content_type
log.debug(
'LFS: Detecting if request `%s` is LFS server path based '
'on content type:`%s`, is_lfs:%s',
path, content_type, is_lfs_request)
if not is_lfs_request:
# fallback detection by path
if GIT_LFS_PROTO_PAT.match(path):
is_lfs_request = True
log.debug(
'LFS: fallback detection by path of: `%s`, is_lfs:%s',
path, is_lfs_request)
if is_lfs_request:
app = scm_app.create_git_lfs_wsgi_app(
repo_path, repo_name, config)
else:
app = scm_app.create_git_wsgi_app(
repo_path, repo_name, config)
logs: added some added logging for stream hg/git.
r247
log.debug('http-app: starting app handler '
'with %s and process request', app)
stream: use wsgi.input_terminated because of webob changes of handling chunked encoding
r332
initial commit
r0 return app(environ, start_response)
git-lfs: added vcsserver handling of git-lfs objects....
r180
initial commit
r0 return _git_stream
Martin Bornhold
http: Add error handling for the repo-locked exception. Part of #4237...
r85 def handle_vcs_exception(self, exception, request):
exceptions: use only one exception handler to track errors with nicer...
r178 _vcs_kind = getattr(exception, '_vcs_kind', '')
http-app: refactor new usage of main http api endpoints
r1123
exceptions: use only one exception handler to track errors with nicer...
r178 if _vcs_kind == 'repo_locked':
http-app: refactor new usage of main http api endpoints
r1123 headers_call_context = get_headers_call_context(request.environ)
status_code = safe_int(headers_call_context['locked_status_code'])
Martin Bornhold
http: Add error handling for the repo-locked exception. Part of #4237...
r85 return HTTPRepoLocked(
http-app: refactor new usage of main http api endpoints
r1123 title=str(exception), status_code=status_code, headers=[('X-Rc-Locked', '1')])
exceptions: allow error tracking and exception storage on vcsserver.
r491
hooks: adjust to handle protected branch cases, and force push.
r509 elif _vcs_kind == 'repo_branch_protected':
# Get custom repo-branch-protected status code if present.
http-app: refactor new usage of main http api endpoints
r1123 return HTTPRepoBranchProtected(
title=str(exception), headers=[('X-Rc-Branch-Protection', '1')])
hooks: adjust to handle protected branch cases, and force push.
r509
exceptions: allow error tracking and exception storage on vcsserver.
r491 exc_info = request.exc_info
store_exception(id(exc_info), exc_info)
errors: use a better interface to track exceptions and tracebacks.
r478 traceback_info = 'unavailable'
if request.exc_info:
exc-tracking: use more rich style tracebacks.
r1144 traceback_info = format_exc(request.exc_info)
Martin Bornhold
http: Add error handling for the repo-locked exception. Part of #4237...
r85
errors: use a better interface to track exceptions and tracebacks.
r478 log.error(
exc-tracking: use more rich style tracebacks.
r1144 'error occurred handling this request for path: %s, \n%s',
errors: use a better interface to track exceptions and tracebacks.
r478 request.path, traceback_info)
metrics: use new statsd client logic, and start gathering new metrics
r1005
statsd = request.registry.statsd
if statsd:
python3: fixes and code optimization for python3.11
r1114 exc_type = f"{exception.__class__.__module__}.{exception.__class__.__name__}"
metrics: expose exc type for vcsserver
r1014 statsd.incr('vcsserver_exception_total',
python3: fixes and code optimization for python3.11
r1114 tags=[f"type:{exc_type}"])
http: added catch-all exception handler to show errors in logs.
r150 raise exception
initial commit
r0
lint: auto-fixes
r1152 class ResponseFilter:
initial commit
r0
def __init__(self, start_response):
self._start_response = start_response
def __call__(self, status, response_headers, exc_info=None):
headers = tuple(
(h, v) for h, v in response_headers
if not wsgiref.util.is_hop_by_hop(h))
return self._start_response(status, headers, exc_info)
core: re-implemented the way how configuration can be made...
r1021 def sanitize_settings_and_apply_defaults(global_config, settings):
core: renamed remote packages to prevent conflicts with builtin libraries like svn core library called same as svn remote
r1145 _global_settings_maker = SettingsMaker(global_config)
core: re-implemented the way how configuration can be made...
r1021 settings_maker = SettingsMaker(settings)
logging: set autoconfigure to False default to not introduce breaking changes
r1029 settings_maker.make_setting('logging.autoconfigure', False, parser='bool')
core: re-implemented the way how configuration can be made...
r1021
logging_conf = os.path.join(os.path.dirname(global_config.get('__file__')), 'logging.ini')
settings_maker.enable_logging(logging_conf)
# Default includes, possible to change as a user
pyramid_includes = settings_maker.make_setting('pyramid.includes', [], parser='list:newline')
env: made configuration completly overridable by env variables
r1023 log.debug("Using the following pyramid.includes: %s", pyramid_includes)
core: re-implemented the way how configuration can be made...
r1021
settings_maker.make_setting('__file__', global_config.get('__file__'))
env: made configuration completly overridable by env variables
r1023 settings_maker.make_setting('pyramid.default_locale_name', 'en')
settings_maker.make_setting('locale', 'en_US.UTF-8')
core: re-implemented the way how configuration can be made...
r1021
fix(settings): fixed string:noquote parser, and make core.binary dir default better
r1217 settings_maker.make_setting(
'core.binary_dir', '/usr/local/bin/rhodecode_bin/vcs_bin',
default_when_empty=True, parser='string:noquote')
core: re-implemented the way how configuration can be made...
r1021
temp_store = tempfile.gettempdir()
default_cache_dir = os.path.join(temp_store, 'rc_cache')
# save default, cache dir, and use it for all backends later.
default_cache_dir = settings_maker.make_setting(
'cache_dir',
default=default_cache_dir, default_when_empty=True,
parser='dir:ensured')
# exception store cache
settings_maker.make_setting(
'exception_tracker.store_path',
default=os.path.join(default_cache_dir, 'exc_store'), default_when_empty=True,
parser='dir:ensured'
)
# repo_object cache defaults
settings_maker.make_setting(
'rc_cache.repo_object.backend',
default='dogpile.cache.rc.file_namespace',
parser='string')
settings_maker.make_setting(
'rc_cache.repo_object.expiration_time',
default=30 * 24 * 60 * 60, # 30days
parser='int')
env: made configuration completly overridable by env variables
r1023 settings_maker.make_setting(
core: re-implemented the way how configuration can be made...
r1021 'rc_cache.repo_object.arguments.filename',
default=os.path.join(default_cache_dir, 'vcsserver_cache_repo_object.db'),
parser='string')
# statsd
env: made configuration completly overridable by env variables
r1023 settings_maker.make_setting('statsd.enabled', False, parser='bool')
settings_maker.make_setting('statsd.statsd_host', 'statsd-exporter', parser='string')
settings_maker.make_setting('statsd.statsd_port', 9125, parser='int')
settings_maker.make_setting('statsd.statsd_prefix', '')
settings_maker.make_setting('statsd.statsd_ipv6', False, parser='bool')
settings_maker.env_expand()
core: re-implemented the way how configuration can be made...
r1021
initial commit
r0 def main(global_config, **settings):
core: re-implemented the way how configuration can be made...
r1021 start_time = time.time()
log.info('Pyramid app config starting')
Martin Bornhold
hg: Include mercurial patching when using the http app.
r35 if MercurialFactory:
hgpatches.patch_largefiles_capabilities()
Martin Bornhold
subrepo: Apply mercurial sub repository patch.
r100 hgpatches.patch_subrepo_type_mapping()
caches: replaced beaker with dogpile cache.
r483
core: re-implemented the way how configuration can be made...
r1021 # Fill in and sanitize the defaults & do ENV expansion
sanitize_settings_and_apply_defaults(global_config, settings)
metrics: use new statsd client logic, and start gathering new metrics
r1005 # init and bootstrap StatsdClient
StatsdClient.setup(settings)
core: re-implemented the way how configuration can be made...
r1021 pyramid_app = HTTPApplication(settings=settings, global_config=global_config).wsgi_app()
total_time = time.time() - start_time
core: unify startup log for pyramid
r1125 log.info('Pyramid app created and configured in %.2fs', total_time)
core: re-implemented the way how configuration can be made...
r1021 return pyramid_app