diff --git a/configs/development.ini b/configs/development.ini --- a/configs/development.ini +++ b/configs/development.ini @@ -26,6 +26,14 @@ locale = en_US.UTF-8 ## at installation time, e.g /home/user/vcsserver-1/profile/bin core.binary_dir = "" +## custom exception store path, defaults to TMPDIR +exception_store_path = + +## Default cache dir for caches. Putting this into a ramdisk +## can boost performance, eg. /tmpfs/data_ramdisk, however this directory might require +## large ammount of space +cache_dir = %(here)s/rcdev/data + ## cache region for storing repo_objects cache rc_cache.repo_object.backend = dogpile.cache.rc.memory_lru ## cache auto-expires after N seconds diff --git a/configs/production.ini b/configs/production.ini --- a/configs/production.ini +++ b/configs/production.ini @@ -47,6 +47,14 @@ locale = en_US.UTF-8 ## at installation time, e.g /home/user/vcsserver-1/profile/bin core.binary_dir = "" +## custom exception store path, defaults to TMPDIR +exception_store_path = + +## Default cache dir for caches. Putting this into a ramdisk +## can boost performance, eg. /tmpfs/data_ramdisk, however this directory might require +## large ammount of space +cache_dir = %(here)s/rcdev/data + ## cache region for storing repo_objects cache rc_cache.repo_object.backend = dogpile.cache.rc.memory_lru ## cache auto-expires after N seconds diff --git a/vcsserver/__init__.py b/vcsserver/__init__.py --- a/vcsserver/__init__.py +++ b/vcsserver/__init__.py @@ -19,3 +19,10 @@ import pkgutil __version__ = pkgutil.get_data('vcsserver', 'VERSION').strip() + +# link to config for pyramid +CONFIG = {} + +# Populated with the settings dictionary from application init in +# +PYRAMID_SETTINGS = {} diff --git a/vcsserver/http_main.py b/vcsserver/http_main.py --- a/vcsserver/http_main.py +++ b/vcsserver/http_main.py @@ -23,6 +23,7 @@ import logging import uuid import wsgiref.util import traceback +import tempfile from itertools import chain import simplejson as json @@ -45,7 +46,7 @@ except locale.Error as e: 'LOCALE ERROR: failed to set LC_ALL, fallback to LC_ALL=C, org error: %s', e) os.environ['LC_ALL'] = 'C' - +import vcsserver from vcsserver import remote_wsgi, scm_app, settings, hgpatches from vcsserver.git_lfs.app import GIT_LFS_CONTENT_TYPE, GIT_LFS_PROTO_PAT from vcsserver.echo_stub import remote_wsgi as remote_wsgi_stub @@ -73,8 +74,6 @@ except ImportError: SvnRemote = None - - def _is_request_chunked(environ): stream = environ.get('HTTP_TRANSFER_ENCODING', '') == 'chunked' return stream @@ -82,6 +81,7 @@ def _is_request_chunked(environ): def _int_setting(settings, name, default): settings[name] = int(settings.get(name, default)) + return settings[name] def _bool_setting(settings, name, default): @@ -89,6 +89,7 @@ def _bool_setting(settings, name, defaul if isinstance(input_val, unicode): input_val = input_val.encode('utf8') settings[name] = asbool(input_val) + return settings[name] def _list_setting(settings, name, default): @@ -96,13 +97,20 @@ def _list_setting(settings, name, defaul # Otherwise we assume it uses pyramids space/newline separation. settings[name] = aslist(raw_value) + return settings[name] -def _string_setting(settings, name, default, lower=True): +def _string_setting(settings, name, default, lower=True, default_when_empty=False): value = settings.get(name, default) + + if default_when_empty and not value: + # use default value when value is empty + value = default + if lower: value = value.lower() settings[name] = value + return settings[name] class VCS(object): @@ -214,13 +222,17 @@ class HTTPApplication(object): self._use_echo_app = True log.warning("Using EchoApp for VCS operations.") self.remote_wsgi = remote_wsgi_stub - self._configure_settings(settings) + + self._configure_settings(global_config, settings) self._configure() - def _configure_settings(self, app_settings): + def _configure_settings(self, global_config, app_settings): """ Configure the settings module. """ + settings_merged = global_config.copy() + settings_merged.update(app_settings) + git_path = app_settings.get('git_path', None) if git_path: settings.GIT_EXECUTABLE = git_path @@ -228,7 +240,30 @@ class HTTPApplication(object): if binary_dir: settings.BINARY_DIR = binary_dir + # Store the settings to make them available to other modules. + vcsserver.PYRAMID_SETTINGS = settings_merged + vcsserver.CONFIG = settings_merged + def _sanitize_settings_and_apply_defaults(self, settings): + + default_cache_dir = os.path.join(tempfile.gettempdir(), 'rc_cache') + + # save default, cache dir, and use it for all backends later. + default_cache_dir = _string_setting( + settings, + 'cache_dir', + default_cache_dir, lower=False, default_when_empty=True) + + # ensure we have our dir created + if not os.path.isdir(default_cache_dir): + os.makedirs(default_cache_dir, mode=0755) + + # exception store cache + _string_setting( + settings, + 'exception_store_path', + default_cache_dir, lower=False) + # repo_object cache _string_setting( settings, diff --git a/vcsserver/lib/exc_tracking.py b/vcsserver/lib/exc_tracking.py --- a/vcsserver/lib/exc_tracking.py +++ b/vcsserver/lib/exc_tracking.py @@ -31,6 +31,7 @@ log = logging.getLogger(__name__) # NOTE: Any changes should be synced with exc_tracking at rhodecode.lib.exc_tracking global_prefix = 'vcsserver' +exc_store_dir_name = 'rc_exception_store_v1' def exc_serialize(exc_id, tb, exc_type): @@ -54,13 +55,10 @@ def get_exc_store(): """ Get and create exception store if it's not existing """ - exc_store_dir = 'rc_exception_store_v1' - # fallback - _exc_store_path = os.path.join(tempfile.gettempdir(), exc_store_dir) + import vcsserver as app - exc_store_dir = '' # TODO: need a persistent cross instance store here - if exc_store_dir: - _exc_store_path = os.path.join(exc_store_dir, exc_store_dir) + exc_store_dir = app.CONFIG.get('exception_store_path', '') or tempfile.gettempdir() + _exc_store_path = os.path.join(exc_store_dir, exc_store_dir_name) _exc_store_path = os.path.abspath(_exc_store_path) if not os.path.isdir(_exc_store_path): @@ -87,6 +85,13 @@ def _store_exception(exc_id, exc_info, p def store_exception(exc_id, exc_info, prefix=global_prefix): + """ + Example usage:: + + exc_info = sys.exc_info() + store_exception(id(exc_info), exc_info) + """ + try: _store_exception(exc_id=exc_id, exc_info=exc_info, prefix=prefix) except Exception: diff --git a/vcsserver/tests/test_main_http.py b/vcsserver/tests/test_main_http.py --- a/vcsserver/tests/test_main_http.py +++ b/vcsserver/tests/test_main_http.py @@ -25,7 +25,7 @@ from vcsserver.base import obfuscate_qs @mock.patch('vcsserver.http_main.VCS', mock.Mock()) @mock.patch('vcsserver.hgpatches.patch_largefiles_capabilities') def test_applies_largefiles_patch(patch_largefiles_capabilities): - http_main.main([]) + http_main.main({}) patch_largefiles_capabilities.assert_called_once_with() @@ -35,7 +35,7 @@ def test_applies_largefiles_patch(patch_ 'vcsserver.hgpatches.patch_largefiles_capabilities', mock.Mock(side_effect=Exception("Must not be called"))) def test_applies_largefiles_patch_only_if_mercurial_is_available(): - http_main.main([]) + http_main.main({}) @pytest.mark.parametrize('given, expected', [