# HG changeset patch # User RhodeCode Admin # Date 2022-12-12 13:05:25 # Node ID 59ec40ea40748fada30f1bdc53705ef64d24617d # Parent c733abbfafe2920d9af9c3a701e6295dc61fdb21 config: major update for the code to make it be almost fully controllable via env for new docker based installer. diff --git a/configs/gunicorn_config.py b/configs/gunicorn_config.py --- a/configs/gunicorn_config.py +++ b/configs/gunicorn_config.py @@ -30,12 +30,12 @@ worker_tmp_dir = None tmp_upload_dir = None # Custom log format -access_log_format = ( - '%(t)s %(p)s INFO [GNCRN] %(h)-15s rqt:%(L)s %(s)s %(b)-6s "%(m)s:%(U)s %(q)s" usr:%(u)s "%(f)s" "%(a)s"') +#access_log_format = ( +# '%(t)s %(p)s INFO [GNCRN] %(h)-15s rqt:%(L)s %(s)s %(b)-6s "%(m)s:%(U)s %(q)s" usr:%(u)s "%(f)s" "%(a)s"') # loki format for easier parsing in grafana -#access_log_format = ( -# 'time="%(t)s" pid=%(p)s level="INFO" type="[GNCRN]" ip="%(h)-15s" rqt="%(L)s" response_code="%(s)s" response_bytes="%(b)-6s" uri="%(m)s:%(U)s %(q)s" user=":%(u)s" user_agent="%(a)s"') +access_log_format = ( + 'time="%(t)s" pid=%(p)s level="INFO" type="[GNCRN]" ip="%(h)-15s" rqt="%(L)s" response_code="%(s)s" response_bytes="%(b)-6s" uri="%(m)s:%(U)s %(q)s" user=":%(u)s" user_agent="%(a)s"') # self adjust workers based on CPU count # workers = get_workers() @@ -97,9 +97,12 @@ def post_fork(server, worker): if conf.has_option(section, 'memory_usage_recovery_threshold'): _memory_usage_recovery_threshold = conf.getfloat(section, 'memory_usage_recovery_threshold') - worker._memory_max_usage = _memory_max_usage - worker._memory_usage_check_interval = _memory_usage_check_interval - worker._memory_usage_recovery_threshold = _memory_usage_recovery_threshold + worker._memory_max_usage = int(os.environ.get('RC_GUNICORN_MEMORY_MAX_USAGE', '') + or _memory_max_usage) + worker._memory_usage_check_interval = int(os.environ.get('RC_GUNICORN_MEMORY_USAGE_CHECK_INTERVAL', '') + or _memory_usage_check_interval) + worker._memory_usage_recovery_threshold = float(os.environ.get('RC_GUNICORN_MEMORY_USAGE_RECOVERY_THRESHOLD', '') + or _memory_usage_recovery_threshold) # register memory last check time, with some random offset so we don't recycle all # at once diff --git a/configs/logging.ini b/configs/logging.ini new file mode 100644 --- /dev/null +++ b/configs/logging.ini @@ -0,0 +1,98 @@ +; ##################### +; LOGGING CONFIGURATION +; ##################### +[loggers] +keys = root, sqlalchemy, beaker, celery, rhodecode, ssh_wrapper + +[handlers] +keys = console, console_sql + +[formatters] +keys = generic, json, color_formatter, color_formatter_sql + +; ####### +; LOGGERS +; ####### +[logger_root] +level = NOTSET +handlers = console + +[logger_sqlalchemy] +level = $RC_LOGGING_LEVEL +handlers = console_sql +qualname = sqlalchemy.engine +propagate = 0 + +[logger_beaker] +level = $RC_LOGGING_LEVEL +handlers = +qualname = beaker.container +propagate = 1 + +[logger_rhodecode] +level = $RC_LOGGING_LEVEL +handlers = +qualname = rhodecode +propagate = 1 + +[logger_ssh_wrapper] +level = $RC_LOGGING_LEVEL +handlers = +qualname = ssh_wrapper +propagate = 1 + +[logger_celery] +level = $RC_LOGGING_LEVEL +handlers = +qualname = celery + + +; ######## +; HANDLERS +; ######## + +[handler_console] +class = StreamHandler +args = (sys.stderr, ) +level = $RC_LOGGING_LEVEL +; To enable JSON formatted logs replace generic with json +; This allows sending properly formatted logs to grafana loki or elasticsearch +#formatter = json +#formatter = generic +formatter = $RC_LOGGING_FORMATTER + +[handler_console_sql] +; "level = DEBUG" logs SQL queries and results. +; "level = INFO" logs SQL queries. +; "level = WARN" logs neither. (Recommended for production systems.) +class = StreamHandler +args = (sys.stderr, ) +level = WARN +; To enable JSON formatted logs replace generic with json +; This allows sending properly formatted logs to grafana loki or elasticsearch +#formatter = json +#formatter = generic +formatter = $RC_LOGGING_FORMATTER + +; ########## +; FORMATTERS +; ########## + +[formatter_generic] +class = rhodecode.lib.logging_formatter.ExceptionAwareFormatter +format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %Y-%m-%d %H:%M:%S + +[formatter_color_formatter] +class = rhodecode.lib.logging_formatter.ColorFormatter +format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %Y-%m-%d %H:%M:%S + +[formatter_color_formatter_sql] +class = rhodecode.lib.logging_formatter.ColorFormatterSql +format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %Y-%m-%d %H:%M:%S + +[formatter_json] +format = %(message)s +class = rhodecode.lib._vendor.jsonlogger.JsonFormatter \ No newline at end of file diff --git a/configs/production.ini b/configs/production.ini --- a/configs/production.ini +++ b/configs/production.ini @@ -124,10 +124,6 @@ use = egg:PasteDeploy#prefix prefix = / [app:main] -; The %(here)s variable will be replaced with the absolute path of parent directory -; of this file -; In addition ENVIRONMENT variables usage is possible, e.g -; sqlalchemy.db1.url = {ENV_RC_DB_URL} use = egg:rhodecode-enterprise-ce @@ -330,6 +326,9 @@ file_store.storage_path = %(here)s/data/ use_celery = false +; path to store schedule database +#celerybeat-schedule.path = + ; connection url to the message broker (default redis) celery.broker_url = redis://localhost:6379/8 @@ -575,6 +574,9 @@ vcs.connection_timeout = 3600 ; Legacy available options are: pre-1.4-compatible, pre-1.5-compatible, pre-1.6-compatible, pre-1.8-compatible, pre-1.9-compatible #vcs.svn.compatible_version = 1.8 +; Cache flag to cache vcsserver remote calls locally +; It uses cache_region `cache_repo` +vcs.methods.cache = true ; #################################################### ; Subversion proxy support (mod_dav_svn) @@ -657,55 +659,55 @@ ssh.enable_ui_key_generator = true ; http://appenlight.rhodecode.com for details how to obtain an account ; Appenlight integration enabled -appenlight = false +#appenlight = false -appenlight.server_url = https://api.appenlight.com -appenlight.api_key = YOUR_API_KEY +#appenlight.server_url = https://api.appenlight.com +#appenlight.api_key = YOUR_API_KEY #appenlight.transport_config = https://api.appenlight.com?threaded=1&timeout=5 ; used for JS client -appenlight.api_public_key = YOUR_API_PUBLIC_KEY +#appenlight.api_public_key = YOUR_API_PUBLIC_KEY ; TWEAK AMOUNT OF INFO SENT HERE ; enables 404 error logging (default False) -appenlight.report_404 = false +#appenlight.report_404 = false ; time in seconds after request is considered being slow (default 1) -appenlight.slow_request_time = 1 +#appenlight.slow_request_time = 1 ; record slow requests in application ; (needs to be enabled for slow datastore recording and time tracking) -appenlight.slow_requests = true +#appenlight.slow_requests = true ; enable hooking to application loggers -appenlight.logging = true +#appenlight.logging = true ; minimum log level for log capture -appenlight.logging.level = WARNING +#ppenlight.logging.level = WARNING ; send logs only from erroneous/slow requests ; (saves API quota for intensive logging) -appenlight.logging_on_error = false +#appenlight.logging_on_error = false ; list of additional keywords that should be grabbed from environ object ; can be string with comma separated list of words in lowercase ; (by default client will always send following info: ; 'REMOTE_USER', 'REMOTE_ADDR', 'SERVER_NAME', 'CONTENT_TYPE' + all keys that ; start with HTTP* this list be extended with additional keywords here -appenlight.environ_keys_whitelist = +#appenlight.environ_keys_whitelist = ; list of keywords that should be blanked from request object ; can be string with comma separated list of words in lowercase ; (by default client will always blank keys that contain following words ; 'password', 'passwd', 'pwd', 'auth_tkt', 'secret', 'csrf' ; this list be extended with additional keywords set here -appenlight.request_keys_blacklist = +#appenlight.request_keys_blacklist = ; list of namespaces that should be ignores when gathering log entries ; can be string with comma separated list of namespaces ; (by default the client ignores own entries: appenlight_client.client) -appenlight.log_namespace_blacklist = +#appenlight.log_namespace_blacklist = ; Statsd client config, this is used to send metrics to statsd ; We recommend setting statsd_exported and scrape them using Promethues @@ -716,6 +718,13 @@ appenlight.log_namespace_blacklist = #statsd.statsd_ipv6 = false +; configure logging automatically at server startup set to false +; to use the below custom logging config. +#logging.autoconfigure = true + +; specify your own custom logging config file to configure logging +#logging.logging_conf_file = /path/to/custom_logging.ini + ; Dummy marker to add new entries after. ; Add any custom entries below. Please don't remove this marker. custom.conf = 1 diff --git a/rhodecode/apps/file_store/__init__.py b/rhodecode/apps/file_store/__init__.py --- a/rhodecode/apps/file_store/__init__.py +++ b/rhodecode/apps/file_store/__init__.py @@ -19,19 +19,20 @@ # and proprietary license terms, please see https://rhodecode.com/licenses/ import os from rhodecode.apps.file_store import config_keys -from rhodecode.config.middleware import _bool_setting, _string_setting +from rhodecode.config.settings_maker import SettingsMaker def _sanitize_settings_and_apply_defaults(settings): """ Set defaults, convert to python types and validate settings. """ - _bool_setting(settings, config_keys.enabled, 'true') + settings_maker = SettingsMaker(settings) - _string_setting(settings, config_keys.backend, 'local') + settings_maker.make_setting(config_keys.enabled, True, parser='bool') + settings_maker.make_setting(config_keys.backend, 'local') default_store = os.path.join(os.path.dirname(settings['__file__']), 'upload_store') - _string_setting(settings, config_keys.store_path, default_store) + settings_maker.make_setting(config_keys.store_path, default_store) def includeme(config): diff --git a/rhodecode/apps/ssh_support/__init__.py b/rhodecode/apps/ssh_support/__init__.py --- a/rhodecode/apps/ssh_support/__init__.py +++ b/rhodecode/apps/ssh_support/__init__.py @@ -24,7 +24,7 @@ from . import config_keys from .events import SshKeyFileChangeEvent from .subscribers import generate_ssh_authorized_keys_file_subscriber -from rhodecode.config.middleware import _bool_setting, _string_setting +from rhodecode.config.settings_maker import SettingsMaker log = logging.getLogger(__name__) @@ -33,28 +33,20 @@ def _sanitize_settings_and_apply_default """ Set defaults, convert to python types and validate settings. """ - _bool_setting(settings, config_keys.generate_authorized_keyfile, 'false') - _bool_setting(settings, config_keys.wrapper_allow_shell, 'false') - _bool_setting(settings, config_keys.enable_debug_logging, 'false') - _bool_setting(settings, config_keys.ssh_key_generator_enabled, 'true') + settings_maker = SettingsMaker(settings) + + settings_maker.make_setting(config_keys.generate_authorized_keyfile, False, parser='bool') + settings_maker.make_setting(config_keys.wrapper_allow_shell, False, parser='bool') + settings_maker.make_setting(config_keys.enable_debug_logging, False, parser='bool') + settings_maker.make_setting(config_keys.ssh_key_generator_enabled, True, parser='bool') - _string_setting(settings, config_keys.authorized_keys_file_path, - '~/.ssh/authorized_keys_rhodecode', - lower=False) - _string_setting(settings, config_keys.wrapper_cmd, '', - lower=False) - _string_setting(settings, config_keys.authorized_keys_line_ssh_opts, '', - lower=False) + settings_maker.make_setting(config_keys.authorized_keys_file_path, '~/.ssh/authorized_keys_rhodecode') + settings_maker.make_setting(config_keys.wrapper_cmd, '') + settings_maker.make_setting(config_keys.authorized_keys_line_ssh_opts, '') - _string_setting(settings, config_keys.ssh_hg_bin, - '~/.rccontrol/vcsserver-1/profile/bin/hg', - lower=False) - _string_setting(settings, config_keys.ssh_git_bin, - '~/.rccontrol/vcsserver-1/profile/bin/git', - lower=False) - _string_setting(settings, config_keys.ssh_svn_bin, - '~/.rccontrol/vcsserver-1/profile/bin/svnserve', - lower=False) + settings_maker.make_setting(config_keys.ssh_hg_bin, '~/.rccontrol/vcsserver-1/profile/bin/hg') + settings_maker.make_setting(config_keys.ssh_git_bin, '~/.rccontrol/vcsserver-1/profile/bin/git') + settings_maker.make_setting(config_keys.ssh_svn_bin, '~/.rccontrol/vcsserver-1/profile/bin/svnserve') def includeme(config): diff --git a/rhodecode/apps/svn_support/__init__.py b/rhodecode/apps/svn_support/__init__.py --- a/rhodecode/apps/svn_support/__init__.py +++ b/rhodecode/apps/svn_support/__init__.py @@ -19,15 +19,13 @@ # and proprietary license terms, please see https://rhodecode.com/licenses/ import os import logging -import shlex from pyramid import compat # Do not use `from rhodecode import events` here, it will be overridden by the # events module in this package due to pythons import mechanism. from rhodecode.events import RepoGroupEvent from rhodecode.subscribers import AsyncSubprocessSubscriber -from rhodecode.config.middleware import ( - _bool_setting, _string_setting, _int_setting) +from rhodecode.config.settings_maker import SettingsMaker from .events import ModDavSvnConfigChange from .subscribers import generate_config_subscriber @@ -41,13 +39,14 @@ def _sanitize_settings_and_apply_default """ Set defaults, convert to python types and validate settings. """ - _bool_setting(settings, config_keys.generate_config, 'false') - _bool_setting(settings, config_keys.list_parent_path, 'true') - _int_setting(settings, config_keys.reload_timeout, 10) - _string_setting(settings, config_keys.config_file_path, '', lower=False) - _string_setting(settings, config_keys.location_root, '/', lower=False) - _string_setting(settings, config_keys.reload_command, '', lower=False) - _string_setting(settings, config_keys.template, '', lower=False) + settings_maker = SettingsMaker(settings) + settings_maker.make_setting(config_keys.generate_config, False, parser='bool') + settings_maker.make_setting(config_keys.list_parent_path, True, parser='bool') + settings_maker.make_setting(config_keys.reload_timeout, 10, parser='bool') + settings_maker.make_setting(config_keys.config_file_path, '') + settings_maker.make_setting(config_keys.location_root, '/') + settings_maker.make_setting(config_keys.reload_command, '') + settings_maker.make_setting(config_keys.template, '') # Convert negative timeout values to zero. if settings[config_keys.reload_timeout] < 0: diff --git a/rhodecode/config/environment.py b/rhodecode/config/environment.py --- a/rhodecode/config/environment.py +++ b/rhodecode/config/environment.py @@ -85,38 +85,3 @@ def load_pyramid_environment(global_conf if vcs_server_enabled: connect_vcs(vcs_server_uri, utils.get_vcs_server_protocol(settings)) - - -def get_rc_env_settings(prefix='ENV_{}'): - return {'ENV_{}'.format(key): value for key, value in os.environ.items()} - - -def substitute_values(mapping, substitutions): - result = {} - - try: - for key, value in mapping.items(): - # initialize without substitution first - result[key] = value - - # Note: Cannot use regular replacements, since they would clash - # with the implementation of ConfigParser. Using "format" instead. - try: - result[key] = value.format(**substitutions) - except KeyError as e: - env_var = '{}'.format(e.args[0]) - - msg = 'Failed to substitute: `{key}={{{var}}}` with environment entry. ' \ - 'Make sure your environment has {var} set, or remove this ' \ - 'variable from config file'.format(key=key, var=env_var) - - if env_var.startswith('ENV_'): - raise ValueError(msg) - else: - log.warning(msg) - - except ValueError as e: - log.warning('Failed to substitute ENV variable: %s', e) - result = mapping - - return result \ No newline at end of file diff --git a/rhodecode/config/middleware.py b/rhodecode/config/middleware.py --- a/rhodecode/config/middleware.py +++ b/rhodecode/config/middleware.py @@ -20,10 +20,10 @@ import os import sys -import logging import collections import tempfile import time +import logging.config from paste.gzipper import make_gzip_middleware import pyramid.events @@ -38,7 +38,8 @@ from pyramid.renderers import render_to_ from rhodecode.model import meta from rhodecode.config import patches from rhodecode.config import utils as config_utils -from rhodecode.config.environment import load_pyramid_environment, substitute_values, get_rc_env_settings +from rhodecode.config.settings_maker import SettingsMaker +from rhodecode.config.environment import load_pyramid_environment import rhodecode.events from rhodecode.lib.middleware.vcs import VCSMiddleware @@ -48,7 +49,7 @@ from rhodecode.lib.exceptions import VCS from rhodecode.lib.middleware.appenlight import wrap_in_appenlight_if_enabled from rhodecode.lib.middleware.https_fixup import HttpsFixup from rhodecode.lib.plugins.utils import register_rhodecode_plugin -from rhodecode.lib.utils2 import aslist as rhodecode_aslist, AttributeDict +from rhodecode.lib.utils2 import AttributeDict from rhodecode.lib.exc_tracking import store_exception from rhodecode.subscribers import ( scan_repositories_if_enabled, write_js_routes_if_enabled, @@ -87,24 +88,14 @@ def make_pyramid_app(global_config, **se cases when these fragments are assembled from another place. """ - - # Allows to use format style "{ENV_NAME}" placeholders in the configuration. It - # will be replaced by the value of the environment variable "NAME" in this case. start_time = time.time() log.info('Pyramid app config starting') - global_config = substitute_values(global_config, get_rc_env_settings()) - settings = substitute_values(settings, get_rc_env_settings()) + sanitize_settings_and_apply_defaults(global_config, settings) # init and bootstrap StatsdClient StatsdClient.setup(settings) - debug = asbool(global_config.get('debug')) - if debug: - enable_debug() - - sanitize_settings_and_apply_defaults(global_config, settings) - config = Configurator(settings=settings) # Init our statsd at very start config.registry.statsd = StatsdClient.statsd @@ -123,17 +114,62 @@ def make_pyramid_app(global_config, **se pyramid_app = wrap_app_in_wsgi_middlewares(pyramid_app, config) pyramid_app.config = config - config.configure_celery(global_config['__file__']) + celery_settings = get_celery_config(settings) + config.configure_celery(celery_settings) # creating the app uses a connection - return it after we are done meta.Session.remove() total_time = time.time() - start_time log.info('Pyramid app `%s` created and configured in %.2fs', - pyramid_app.func_name, total_time) + getattr(pyramid_app, 'func_name', 'pyramid_app'), total_time) return pyramid_app +def get_celery_config(settings): + """ + Converts basic ini configuration into celery 4.X options + """ + + def key_converter(key_name): + pref = 'celery.' + if key_name.startswith(pref): + return key_name[len(pref):].replace('.', '_').lower() + + def type_converter(parsed_key, value): + # cast to int + if value.isdigit(): + return int(value) + + # cast to bool + if value.lower() in ['true', 'false', 'True', 'False']: + return value.lower() == 'true' + return value + + celery_config = {} + for k, v in settings.items(): + pref = 'celery.' + if k.startswith(pref): + celery_config[key_converter(k)] = type_converter(key_converter(k), v) + + # TODO:rethink if we want to support celerybeat based file config, probably NOT + # beat_config = {} + # for section in parser.sections(): + # if section.startswith('celerybeat:'): + # name = section.split(':', 1)[1] + # beat_config[name] = get_beat_config(parser, section) + + # final compose of settings + celery_settings = {} + + if celery_config: + celery_settings.update(celery_config) + # if beat_config: + # celery_settings.update({'beat_schedule': beat_config}) + + return celery_settings + + def not_found_view(request): """ This creates the view which should be registered as not-found-view to @@ -263,7 +299,7 @@ def includeme(config, auth_resources=Non config.add_directive('configure_celery', configure_celery) - if asbool(settings.get('appenlight', 'false')): + if settings.get('appenlight', False): config.include('appenlight_client.ext.pyramid_tween') load_all = should_load_all() @@ -435,8 +471,28 @@ def sanitize_settings_and_apply_defaults function. """ - settings.setdefault('rhodecode.edition', 'Community Edition') - settings.setdefault('rhodecode.edition_id', 'CE') + global_settings_maker = SettingsMaker(global_config) + global_settings_maker.make_setting('debug', default=False, parser='bool') + debug_enabled = asbool(global_config.get('debug')) + + settings_maker = SettingsMaker(settings) + + settings_maker.make_setting( + 'logging.autoconfigure', + default=True, + parser='bool') + + logging_conf = os.path.join(os.path.dirname(global_config.get('__file__')), 'logging.ini') + settings_maker.enable_logging(logging_conf, level='INFO' if debug_enabled else 'DEBUG') + + # Default includes, possible to change as a user + pyramid_includes = settings_maker.make_setting('pyramid.includes', [], parser='list:newline') + log.debug( + "Using the following pyramid.includes: %s", + pyramid_includes) + + settings_maker.make_setting('rhodecode.edition', 'Community Edition') + settings_maker.make_setting('rhodecode.edition_id', 'CE') if 'mako.default_filters' not in settings: # set custom default filters if we don't have it defined @@ -458,25 +514,41 @@ def sanitize_settings_and_apply_defaults if not raw_url.startswith(('redis://', 'rediss://', 'unix://')): settings['beaker.session.url'] = 'redis://' + raw_url - # Default includes, possible to change as a user - pyramid_includes = settings.setdefault('pyramid.includes', []) - log.debug( - "Using the following pyramid.includes: %s", - pyramid_includes) + settings_maker.make_setting('__file__', global_config.get('__file__')) # TODO: johbo: Re-think this, usually the call to config.include # should allow to pass in a prefix. - settings.setdefault('rhodecode.api.url', '/_admin/api') - settings.setdefault('__file__', global_config.get('__file__')) + settings_maker.make_setting('rhodecode.api.url', '/_admin/api') # Sanitize generic settings. - _list_setting(settings, 'default_encoding', 'UTF-8') - _bool_setting(settings, 'is_test', 'false') - _bool_setting(settings, 'gzip_responses', 'false') + settings_maker.make_setting('default_encoding', 'UTF-8', parser='list') + settings_maker.make_setting('is_test', False, parser='bool') + settings_maker.make_setting('gzip_responses', False, parser='bool') - # Call split out functions that sanitize settings for each topic. - _sanitize_appenlight_settings(settings) - _sanitize_vcs_settings(settings) + settings_maker.make_setting('vcs.svn.compatible_version', '') + settings_maker.make_setting('vcs.hooks.protocol', 'http') + settings_maker.make_setting('vcs.hooks.host', '127.0.0.1') + settings_maker.make_setting('vcs.scm_app_implementation', 'http') + settings_maker.make_setting('vcs.server', '') + settings_maker.make_setting('vcs.server.protocol', 'http') + settings_maker.make_setting('startup.import_repos', 'false', parser='bool') + settings_maker.make_setting('vcs.hooks.direct_calls', 'false', parser='bool') + settings_maker.make_setting('vcs.server.enable', 'true', parser='bool') + settings_maker.make_setting('vcs.start_server', 'false', parser='bool') + settings_maker.make_setting('vcs.backends', 'hg, git, svn', parser='list') + settings_maker.make_setting('vcs.connection_timeout', 3600, parser='int') + + settings_maker.make_setting('vcs.methods.cache', True, parser='bool') + + # Support legacy values of vcs.scm_app_implementation. Legacy + # configurations may use 'rhodecode.lib.middleware.utils.scm_app_http', or + # disabled since 4.13 'vcsserver.scm_app' which is now mapped to 'http'. + scm_app_impl = settings['vcs.scm_app_implementation'] + if scm_app_impl in ['rhodecode.lib.middleware.utils.scm_app_http', 'vcsserver.scm_app']: + settings['vcs.scm_app_implementation'] = 'http' + + settings_maker.make_setting('appenlight', False, parser='bool') + _sanitize_cache_settings(settings) # configure instance id @@ -485,277 +557,55 @@ def sanitize_settings_and_apply_defaults return settings -def enable_debug(): - """ - Helper to enable debug on running instance - :return: - """ - import tempfile - import textwrap - import logging.config - - ini_template = textwrap.dedent(""" - ##################################### - ### DEBUG LOGGING CONFIGURATION #### - ##################################### - [loggers] - keys = root, sqlalchemy, beaker, celery, rhodecode, ssh_wrapper - - [handlers] - keys = console, console_sql - - [formatters] - keys = generic, color_formatter, color_formatter_sql - - ############# - ## LOGGERS ## - ############# - [logger_root] - level = NOTSET - handlers = console - - [logger_sqlalchemy] - level = INFO - handlers = console_sql - qualname = sqlalchemy.engine - propagate = 0 - - [logger_beaker] - level = DEBUG - handlers = - qualname = beaker.container - propagate = 1 - - [logger_rhodecode] - level = DEBUG - handlers = - qualname = rhodecode - propagate = 1 - - [logger_ssh_wrapper] - level = DEBUG - handlers = - qualname = ssh_wrapper - propagate = 1 - - [logger_celery] - level = DEBUG - handlers = - qualname = celery - - - ############## - ## HANDLERS ## - ############## - - [handler_console] - class = StreamHandler - args = (sys.stderr, ) - level = DEBUG - formatter = color_formatter +def _sanitize_cache_settings(settings): + settings_maker = SettingsMaker(settings) - [handler_console_sql] - # "level = DEBUG" logs SQL queries and results. - # "level = INFO" logs SQL queries. - # "level = WARN" logs neither. (Recommended for production systems.) - class = StreamHandler - args = (sys.stderr, ) - level = WARN - formatter = color_formatter_sql - - ################ - ## FORMATTERS ## - ################ - - [formatter_generic] - class = rhodecode.lib.logging_formatter.ExceptionAwareFormatter - format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s | %(req_id)s - datefmt = %Y-%m-%d %H:%M:%S - - [formatter_color_formatter] - class = rhodecode.lib.logging_formatter.ColorRequestTrackingFormatter - format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s | %(req_id)s - datefmt = %Y-%m-%d %H:%M:%S - - [formatter_color_formatter_sql] - class = rhodecode.lib.logging_formatter.ColorFormatterSql - format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s - datefmt = %Y-%m-%d %H:%M:%S - """) - - with tempfile.NamedTemporaryFile(prefix='rc_debug_logging_', suffix='.ini', - delete=False) as f: - log.info('Saved Temporary DEBUG config at %s', f.name) - f.write(ini_template) - - logging.config.fileConfig(f.name) - log.debug('DEBUG MODE ON') - os.remove(f.name) - - -def _sanitize_appenlight_settings(settings): - _bool_setting(settings, 'appenlight', 'false') - - -def _sanitize_vcs_settings(settings): - """ - Applies settings defaults and does type conversion for all VCS related - settings. - """ - _string_setting(settings, 'vcs.svn.compatible_version', '') - _string_setting(settings, 'vcs.hooks.protocol', 'http') - _string_setting(settings, 'vcs.hooks.host', '127.0.0.1') - _string_setting(settings, 'vcs.scm_app_implementation', 'http') - _string_setting(settings, 'vcs.server', '') - _string_setting(settings, 'vcs.server.protocol', 'http') - _bool_setting(settings, 'startup.import_repos', 'false') - _bool_setting(settings, 'vcs.hooks.direct_calls', 'false') - _bool_setting(settings, 'vcs.server.enable', 'true') - _bool_setting(settings, 'vcs.start_server', 'false') - _list_setting(settings, 'vcs.backends', 'hg, git, svn') - _int_setting(settings, 'vcs.connection_timeout', 3600) - - # Support legacy values of vcs.scm_app_implementation. Legacy - # configurations may use 'rhodecode.lib.middleware.utils.scm_app_http', or - # disabled since 4.13 'vcsserver.scm_app' which is now mapped to 'http'. - scm_app_impl = settings['vcs.scm_app_implementation'] - if scm_app_impl in ['rhodecode.lib.middleware.utils.scm_app_http', 'vcsserver.scm_app']: - settings['vcs.scm_app_implementation'] = 'http' - - -def _sanitize_cache_settings(settings): temp_store = tempfile.gettempdir() default_cache_dir = os.path.join(temp_store, 'rc_cache') # save default, cache dir, and use it for all backends later. - default_cache_dir = _string_setting( - settings, + default_cache_dir = settings_maker.make_setting( 'cache_dir', - default_cache_dir, lower=False, default_when_empty=True) - - # ensure we have our dir created - if not os.path.isdir(default_cache_dir): - os.makedirs(default_cache_dir, mode=0o755) + default=default_cache_dir, default_when_empty=True, + parser='dir:ensured') # exception store cache - _string_setting( - settings, + settings_maker.make_setting( 'exception_tracker.store_path', - temp_store, lower=False, default_when_empty=True) - _bool_setting( - settings, - 'exception_tracker.send_email', - 'false') - _string_setting( - settings, - 'exception_tracker.email_prefix', - '[RHODECODE ERROR]', lower=False, default_when_empty=True) + default=os.path.join(default_cache_dir, 'exc_store'), default_when_empty=True, + parser='dir:ensured' + ) + + settings_maker.make_setting( + 'celerybeat-schedule.path', + default=os.path.join(default_cache_dir, 'celerybeat_schedule', 'celerybeat-schedule.db'), default_when_empty=True, + parser='file:ensured' + ) + + settings_maker.make_setting('exception_tracker.send_email', False, parser='bool') + settings_maker.make_setting('exception_tracker.email_prefix', '[RHODECODE ERROR]', default_when_empty=True) # cache_perms - _string_setting( - settings, - 'rc_cache.cache_perms.backend', - 'dogpile.cache.rc.file_namespace', lower=False) - _int_setting( - settings, - 'rc_cache.cache_perms.expiration_time', - 60) - _string_setting( - settings, - 'rc_cache.cache_perms.arguments.filename', - os.path.join(default_cache_dir, 'rc_cache_1'), lower=False) + settings_maker.make_setting('rc_cache.cache_perms.backend', 'dogpile.cache.rc.file_namespace') + settings_maker.make_setting('rc_cache.cache_perms.expiration_time', 60, parser='int') + settings_maker.make_setting('rc_cache.cache_perms.arguments.filename', os.path.join(default_cache_dir, 'rhodecode_cache_perms.db')) # cache_repo - _string_setting( - settings, - 'rc_cache.cache_repo.backend', - 'dogpile.cache.rc.file_namespace', lower=False) - _int_setting( - settings, - 'rc_cache.cache_repo.expiration_time', - 60) - _string_setting( - settings, - 'rc_cache.cache_repo.arguments.filename', - os.path.join(default_cache_dir, 'rc_cache_2'), lower=False) + settings_maker.make_setting('rc_cache.cache_repo.backend', 'dogpile.cache.rc.file_namespace') + settings_maker.make_setting('rc_cache.cache_repo.expiration_time', 60, parser='int') + settings_maker.make_setting('rc_cache.cache_repo.arguments.filename', os.path.join(default_cache_dir, 'rhodecode_cache_repo.db')) # cache_license - _string_setting( - settings, - 'rc_cache.cache_license.backend', - 'dogpile.cache.rc.file_namespace', lower=False) - _int_setting( - settings, - 'rc_cache.cache_license.expiration_time', - 5*60) - _string_setting( - settings, - 'rc_cache.cache_license.arguments.filename', - os.path.join(default_cache_dir, 'rc_cache_3'), lower=False) + settings_maker.make_setting('rc_cache.cache_license.backend', 'dogpile.cache.rc.file_namespace') + settings_maker.make_setting('rc_cache.cache_license.expiration_time', 5*60, parser='int') + settings_maker.make_setting('rc_cache.cache_license.arguments.filename', os.path.join(default_cache_dir, 'rhodecode_cache_license.db')) # cache_repo_longterm memory, 96H - _string_setting( - settings, - 'rc_cache.cache_repo_longterm.backend', - 'dogpile.cache.rc.memory_lru', lower=False) - _int_setting( - settings, - 'rc_cache.cache_repo_longterm.expiration_time', - 345600) - _int_setting( - settings, - 'rc_cache.cache_repo_longterm.max_size', - 10000) + settings_maker.make_setting('rc_cache.cache_repo_longterm.backend', 'dogpile.cache.rc.memory_lru') + settings_maker.make_setting('rc_cache.cache_repo_longterm.expiration_time', 345600, parser='int') + settings_maker.make_setting('rc_cache.cache_repo_longterm.max_size', 10000, parser='int') # sql_cache_short - _string_setting( - settings, - 'rc_cache.sql_cache_short.backend', - 'dogpile.cache.rc.memory_lru', lower=False) - _int_setting( - settings, - 'rc_cache.sql_cache_short.expiration_time', - 30) - _int_setting( - settings, - 'rc_cache.sql_cache_short.max_size', - 10000) - - -def _int_setting(settings, name, default): - settings[name] = int(settings.get(name, default)) - return settings[name] - - -def _bool_setting(settings, name, default): - input_val = settings.get(name, default) - if isinstance(input_val, unicode): - input_val = input_val.encode('utf8') - settings[name] = asbool(input_val) - return settings[name] - - -def _list_setting(settings, name, default): - raw_value = settings.get(name, default) - - old_separator = ',' - if old_separator in raw_value: - # If we get a comma separated list, pass it to our own function. - settings[name] = rhodecode_aslist(raw_value, sep=old_separator) - else: - # Otherwise we assume it uses pyramids space/newline separation. - settings[name] = aslist(raw_value) - return settings[name] - - -def _string_setting(settings, name, default, lower=True, default_when_empty=False): - value = settings.get(name, default) - - if default_when_empty and not value: - # use default value when value is empty - value = default - - if lower: - value = value.lower() - settings[name] = value - return settings[name] + settings_maker.make_setting('rc_cache.sql_cache_short.backend', 'dogpile.cache.rc.memory_lru') + settings_maker.make_setting('rc_cache.sql_cache_short.expiration_time', 30, parser='int') + settings_maker.make_setting('rc_cache.sql_cache_short.max_size', 10000, parser='int') diff --git a/rhodecode/config/settings_maker.py b/rhodecode/config/settings_maker.py new file mode 100644 --- /dev/null +++ b/rhodecode/config/settings_maker.py @@ -0,0 +1,182 @@ +# -*- coding: utf-8 -*- + +# Copyright (C) 2010-2020 RhodeCode GmbH +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License, version 3 +# (only), as published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +# This program is dual-licensed. If you wish to learn more about the +# RhodeCode Enterprise Edition, including its added features, Support services, +# and proprietary license terms, please see https://rhodecode.com/licenses/ + +import os +import textwrap +import string +import functools +import logging +import tempfile +import logging.config +log = logging.getLogger(__name__) + + +def str2bool(_str): + """ + returns True/False value from given string, it tries to translate the + string into boolean + + :param _str: string value to translate into boolean + :rtype: boolean + :returns: boolean from given string + """ + if _str is None: + return False + if _str in (True, False): + return _str + _str = str(_str).strip().lower() + return _str in ('t', 'true', 'y', 'yes', 'on', '1') + + +def aslist(obj, sep=None, strip=True): + """ + Returns given string separated by sep as list + + :param obj: + :param sep: + :param strip: + """ + if isinstance(obj, (basestring,)): + if obj in ['', ""]: + return [] + + lst = obj.split(sep) + if strip: + lst = [v.strip() for v in lst] + return lst + elif isinstance(obj, (list, tuple)): + return obj + elif obj is None: + return [] + else: + return [obj] + + +class SettingsMaker(object): + + def __init__(self, app_settings): + self.settings = app_settings + + @classmethod + def _bool_func(cls, input_val): + if isinstance(input_val, unicode): + input_val = input_val.encode('utf8') + return str2bool(input_val) + + @classmethod + def _int_func(cls, input_val): + return int(input_val) + + @classmethod + def _list_func(cls, input_val, sep=','): + return aslist(input_val, sep=sep) + + @classmethod + def _string_func(cls, input_val, lower=True): + if lower: + input_val = input_val.lower() + return input_val + + @classmethod + def _float_func(cls, input_val): + return float(input_val) + + @classmethod + def _dir_func(cls, input_val, ensure_dir=False, mode=0o755): + + # ensure we have our dir created + if not os.path.isdir(input_val) and ensure_dir: + os.makedirs(input_val, mode=mode) + + if not os.path.isdir(input_val): + raise Exception('Dir at {} does not exist'.format(input_val)) + return input_val + + @classmethod + def _file_path_func(cls, input_val, ensure_dir=False, mode=0o755): + dirname = os.path.dirname(input_val) + cls._dir_func(dirname, ensure_dir=ensure_dir) + return input_val + + @classmethod + def _key_transformator(cls, key): + return "{}_{}".format('RC'.upper(), key.upper().replace('.', '_').replace('-', '_')) + + def enable_logging(self, logging_conf=None, level='INFO', formatter='generic'): + """ + Helper to enable debug on running instance + :return: + """ + + if not str2bool(self.settings.get('logging.autoconfigure')): + log.info('logging configuration based on main .ini file') + return + + if logging_conf is None: + logging_conf = self.settings.get('logging.logging_conf_file') or '' + + if not os.path.isfile(logging_conf): + log.error('Unable to setup logging based on %s, file does not exist...', logging_conf) + return + + with open(logging_conf, 'rb') as f: + ini_template = textwrap.dedent(f.read()) + ini_template = string.Template(ini_template).safe_substitute( + RC_LOGGING_LEVEL=os.environ.get('RC_LOGGING_LEVEL', '') or level, + RC_LOGGING_FORMATTER=os.environ.get('RC_LOGGING_FORMATTER', '') or formatter + ) + + with tempfile.NamedTemporaryFile(prefix='rc_logging_', suffix='.ini', delete=False) as f: + log.info('Saved Temporary LOGGING config at %s', f.name) + f.write(ini_template) + + logging.config.fileConfig(f.name) + os.remove(f.name) + + def make_setting(self, key, default, lower=False, default_when_empty=False, parser=None): + + input_val = self.settings.get(key, default) + + if default_when_empty and not input_val: + # use default value when value is set in the config but it is empty + input_val = default + + parser_func = { + 'bool': self._bool_func, + 'int': self._int_func, + 'list': self._list_func, + 'list:newline': functools.partial(self._list_func, sep='/n'), + 'list:spacesep': functools.partial(self._list_func, sep=' '), + 'string': functools.partial(self._string_func, lower=lower), + 'dir': self._dir_func, + 'dir:ensured': functools.partial(self._dir_func, ensure_dir=True), + 'file': self._file_path_func, + 'file:ensured': functools.partial(self._file_path_func, ensure_dir=True), + None: lambda i: i + }[parser] + + # now maybe we have this KEY in env, search and use the value with higher priority. + transformed_key = self._key_transformator(key) + envvar_value = os.environ.get(transformed_key) + if envvar_value: + log.debug('using `%s` key instead of `%s` key for config', transformed_key, key) + input_val = envvar_value + self.settings[key] = parser_func(input_val) + return self.settings[key] diff --git a/rhodecode/lib/base.py b/rhodecode/lib/base.py --- a/rhodecode/lib/base.py +++ b/rhodecode/lib/base.py @@ -371,7 +371,7 @@ def attach_context_attributes(context, r config.get('license.hide_license_info', False)) # AppEnlight - context.appenlight_enabled = str2bool(config.get('appenlight', 'false')) + context.appenlight_enabled = config.get('appenlight', False) context.appenlight_api_public_key = config.get( 'appenlight.api_public_key', '') context.appenlight_server_url = config.get('appenlight.server_url', '') diff --git a/rhodecode/lib/celerylib/loader.py b/rhodecode/lib/celerylib/loader.py --- a/rhodecode/lib/celerylib/loader.py +++ b/rhodecode/lib/celerylib/loader.py @@ -40,7 +40,7 @@ from pyramid.threadlocal import get_curr import rhodecode from rhodecode.lib.auth import AuthUser -from rhodecode.lib.celerylib.utils import get_ini_config, parse_ini_vars, ping_db +from rhodecode.lib.celerylib.utils import parse_ini_vars, ping_db from rhodecode.lib.ext_json import json from rhodecode.lib.pyramid_utils import bootstrap, setup_logging, prepare_request from rhodecode.lib.utils2 import str2bool @@ -116,6 +116,8 @@ def setup_logging_callback(**kwargs): @signals.user_preload_options.connect def on_preload_parsed(options, **kwargs): + from rhodecode.config.middleware import get_celery_config + ini_location = options['ini'] ini_vars = options['ini_var'] celery_app.conf['INI_PYRAMID'] = options['ini'] @@ -134,10 +136,11 @@ def on_preload_parsed(options, **kwargs) log.debug('Bootstrapping RhodeCode application...') env = bootstrap(ini_location, options=options) + celery_settings = get_celery_config(env['registry'].settings) setup_celery_app( app=env['app'], root=env['root'], request=env['request'], registry=env['registry'], closer=env['closer'], - ini_location=ini_location) + celery_settings=celery_settings) # fix the global flag even if it's disabled via .ini file because this # is a worker code that doesn't need this to be disabled. @@ -196,17 +199,15 @@ def task_revoked_signal( closer() -def setup_celery_app(app, root, request, registry, closer, ini_location): - ini_dir = os.path.dirname(os.path.abspath(ini_location)) +def setup_celery_app(app, root, request, registry, closer, celery_settings): + log.debug('Got custom celery conf: %s', celery_settings) celery_config = base_celery_config celery_config.update({ # store celerybeat scheduler db where the .ini file is - 'beat_schedule_filename': os.path.join(ini_dir, 'celerybeat-schedule'), + 'beat_schedule_filename': registry.settings['celerybeat-schedule.path'], }) - ini_settings = get_ini_config(ini_location) - log.debug('Got custom celery conf: %s', ini_settings) - celery_config.update(ini_settings) + celery_config.update(celery_settings) celery_app.config_from_object(celery_config) celery_app.conf.update({'PYRAMID_APP': app}) @@ -216,7 +217,7 @@ def setup_celery_app(app, root, request, celery_app.conf.update({'PYRAMID_CLOSER': closer}) -def configure_celery(config, ini_location): +def configure_celery(config, celery_settings): """ Helper that is called from our application creation logic. It gives connection info into running webapp and allows execution of tasks from @@ -226,10 +227,10 @@ def configure_celery(config, ini_locatio rhodecode.CELERY_ENABLED = str2bool( config.registry.settings.get('use_celery')) if rhodecode.CELERY_ENABLED: - log.info('Configuring celery based on `%s` file', ini_location) + log.info('Configuring celery based on `%s` settings', celery_settings) setup_celery_app( app=None, root=None, request=None, registry=config.registry, - closer=None, ini_location=ini_location) + closer=None, celery_settings=celery_settings) def maybe_prepare_env(req): @@ -238,7 +239,7 @@ def maybe_prepare_env(req): environ.update({ 'PATH_INFO': req.environ['PATH_INFO'], 'SCRIPT_NAME': req.environ['SCRIPT_NAME'], - 'HTTP_HOST':req.environ.get('HTTP_HOST', req.environ['SERVER_NAME']), + 'HTTP_HOST': req.environ.get('HTTP_HOST', req.environ['SERVER_NAME']), 'SERVER_NAME': req.environ['SERVER_NAME'], 'SERVER_PORT': req.environ['SERVER_PORT'], 'wsgi.url_scheme': req.environ['wsgi.url_scheme'], diff --git a/rhodecode/lib/celerylib/utils.py b/rhodecode/lib/celerylib/utils.py --- a/rhodecode/lib/celerylib/utils.py +++ b/rhodecode/lib/celerylib/utils.py @@ -115,52 +115,6 @@ def get_beat_config(parser, section): return config -def get_ini_config(ini_location): - """ - Converts basic ini configuration into celery 4.X options - """ - def key_converter(key_name): - pref = 'celery.' - if key_name.startswith(pref): - return key_name[len(pref):].replace('.', '_').lower() - - def type_converter(parsed_key, value): - # cast to int - if value.isdigit(): - return int(value) - - # cast to bool - if value.lower() in ['true', 'false', 'True', 'False']: - return value.lower() == 'true' - return value - - parser = configparser.SafeConfigParser( - defaults={'here': os.path.abspath(ini_location)}) - parser.read(ini_location) - - ini_config = {} - for k, v in parser.items('app:main'): - pref = 'celery.' - if k.startswith(pref): - ini_config[key_converter(k)] = type_converter(key_converter(k), v) - - beat_config = {} - for section in parser.sections(): - if section.startswith('celerybeat:'): - name = section.split(':', 1)[1] - beat_config[name] = get_beat_config(parser, section) - - # final compose of settings - celery_settings = {} - - if ini_config: - celery_settings.update(ini_config) - if beat_config: - celery_settings.update({'beat_schedule': beat_config}) - - return celery_settings - - def parse_ini_vars(ini_vars): options = {} for pairs in ini_vars.split(','): diff --git a/rhodecode/tests/config/test_sanitize_settings.py b/rhodecode/tests/config/test_sanitize_settings.py --- a/rhodecode/tests/config/test_sanitize_settings.py +++ b/rhodecode/tests/config/test_sanitize_settings.py @@ -22,9 +22,8 @@ import pytest from rhodecode.tests import no_newline_id_generator -from rhodecode.config.middleware import ( - _sanitize_vcs_settings, _bool_setting, _string_setting, _list_setting, - _int_setting) +from rhodecode.config.middleware import sanitize_settings_and_apply_defaults +from rhodecode.config.settings_maker import SettingsMaker class TestHelperFunctions(object): @@ -39,11 +38,9 @@ class TestHelperFunctions(object): ('invalid-∫øø@-√å@¨€', False), (u'invalid-∫øø@-√å@¨€', False), ]) - def test_bool_setting_helper(self, raw, expected): - key = 'dummy-key' - settings = {key: raw} - _bool_setting(settings, key, None) - assert settings[key] is expected + def test_bool_func_helper(self, raw, expected): + val = SettingsMaker._bool_func(raw) + assert val == expected @pytest.mark.parametrize('raw, expected', [ ('', ''), @@ -52,11 +49,9 @@ class TestHelperFunctions(object): ('test-string-烩€', 'test-string-烩€'), (u'test-string-烩€', u'test-string-烩€'), ]) - def test_string_setting_helper(self, raw, expected): - key = 'dummy-key' - settings = {key: raw} - _string_setting(settings, key, None) - assert settings[key] == expected + def test_string_func_helper(self, raw, expected): + val = SettingsMaker._string_func(raw) + assert val == expected @pytest.mark.parametrize('raw, expected', [ ('', []), @@ -64,19 +59,30 @@ class TestHelperFunctions(object): ('CaSe-TeSt', ['CaSe-TeSt']), ('test-string-烩€', ['test-string-烩€']), (u'test-string-烩€', [u'test-string-烩€']), - ('hg git svn', ['hg', 'git', 'svn']), ('hg,git,svn', ['hg', 'git', 'svn']), ('hg, git, svn', ['hg', 'git', 'svn']), - ('hg\ngit\nsvn', ['hg', 'git', 'svn']), - (' hg\n git\n svn ', ['hg', 'git', 'svn']), + (', hg , git , svn , ', ['', 'hg', 'git', 'svn', '']), ('cheese,free node,other', ['cheese', 'free node', 'other']), ], ids=no_newline_id_generator) def test_list_setting_helper(self, raw, expected): - key = 'dummy-key' - settings = {key: raw} - _list_setting(settings, key, None) - assert settings[key] == expected + val = SettingsMaker._list_func(raw) + assert val == expected + + @pytest.mark.parametrize('raw, expected', [ + ('hg git svn', ['hg', 'git', 'svn']), + ], ids=no_newline_id_generator) + def test_list_setting_spaces_helper(self, raw, expected): + val = SettingsMaker._list_func(raw, sep=' ') + assert val == expected + + @pytest.mark.parametrize('raw, expected', [ + ('hg\ngit\nsvn', ['hg', 'git', 'svn']), + (' hg\n git\n svn ', ['hg', 'git', 'svn']), + ], ids=no_newline_id_generator) + def test_list_setting_newlines_helper(self, raw, expected): + val = SettingsMaker._list_func(raw, sep='\n') + assert val == expected @pytest.mark.parametrize('raw, expected', [ ('0', 0), @@ -86,10 +92,8 @@ class TestHelperFunctions(object): (u'-12345', -12345), ]) def test_int_setting_helper(self, raw, expected): - key = 'dummy-key' - settings = {key: raw} - _int_setting(settings, key, None) - assert settings[key] == expected + val = SettingsMaker._int_func(raw) + assert val == expected @pytest.mark.parametrize('raw', [ ('0xff'), @@ -99,21 +103,19 @@ class TestHelperFunctions(object): (u'invalid-⁄~†'), ]) def test_int_setting_helper_invalid_input(self, raw): - key = 'dummy-key' - settings = {key: raw} with pytest.raises(Exception): - _int_setting(settings, key, None) + SettingsMaker._int_func(raw) class TestSanitizeVcsSettings(object): - _bool_settings = [ + _bool_funcs = [ ('vcs.hooks.direct_calls', False), ('vcs.server.enable', True), ('vcs.start_server', False), ('startup.import_repos', False), ] - _string_settings = [ + _string_funcs = [ ('vcs.svn.compatible_version', ''), ('vcs.hooks.protocol', 'http'), ('vcs.hooks.host', '127.0.0.1'), @@ -126,28 +128,28 @@ class TestSanitizeVcsSettings(object): ('vcs.backends', 'hg git'), ] - @pytest.mark.parametrize('key, default', _list_settings) - def test_list_setting_spacesep_list(self, key, default): - test_list = ['test', 'list', 'values', 'for', key] - input_value = ' '.join(test_list) - settings = {key: input_value} - _sanitize_vcs_settings(settings) - assert settings[key] == test_list - - @pytest.mark.parametrize('key, default', _list_settings) - def test_list_setting_newlinesep_list(self, key, default): - test_list = ['test', 'list', 'values', 'for', key] - input_value = '\n'.join(test_list) - settings = {key: input_value} - _sanitize_vcs_settings(settings) - assert settings[key] == test_list + # @pytest.mark.parametrize('key, default', _list_settings) + # def test_list_setting_spacesep_list(self, key, default): + # test_list = ['test', 'list', 'values', 'for', key] + # input_value = ' '.join(test_list) + # settings = {key: input_value} + # sanitize_settings_and_apply_defaults({'__file__': ''}, settings) + # assert settings[key] == test_list + # + # @pytest.mark.parametrize('key, default', _list_settings) + # def test_list_setting_newlinesep_list(self, key, default): + # test_list = ['test', 'list', 'values', 'for', key] + # input_value = '\n'.join(test_list) + # settings = {key: input_value} + # sanitize_settings_and_apply_defaults({'__file__': ''}, settings) + # assert settings[key] == test_list @pytest.mark.parametrize('key, default', _list_settings) def test_list_setting_commasep_list(self, key, default): test_list = ['test', 'list', 'values', 'for', key] input_value = ','.join(test_list) settings = {key: input_value} - _sanitize_vcs_settings(settings) + sanitize_settings_and_apply_defaults({'__file__': ''}, settings) assert settings[key] == test_list @pytest.mark.parametrize('key, default', _list_settings) @@ -155,49 +157,49 @@ class TestSanitizeVcsSettings(object): test_list = ['test', 'list', 'values', 'for', key] input_value = ', '.join(test_list) settings = {key: input_value} - _sanitize_vcs_settings(settings) + sanitize_settings_and_apply_defaults({'__file__': ''}, settings) assert settings[key] == test_list - @pytest.mark.parametrize('key, default', _string_settings) - def test_string_setting_string(self, key, default): + @pytest.mark.parametrize('key, default', _string_funcs) + def test_string_func_string(self, key, default): test_value = 'test-string-for-{}'.format(key) settings = {key: test_value} - _sanitize_vcs_settings(settings) + sanitize_settings_and_apply_defaults({'__file__': ''}, settings) assert settings[key] == test_value - @pytest.mark.parametrize('key, default', _string_settings) - def test_string_setting_default(self, key, default): + @pytest.mark.parametrize('key, default', _string_funcs) + def test_string_func_default(self, key, default): settings = {} - _sanitize_vcs_settings(settings) + sanitize_settings_and_apply_defaults({'__file__': ''}, settings) assert settings[key] == default - @pytest.mark.parametrize('key, default', _string_settings) - def test_string_setting_lowercase(self, key, default): - test_value = 'Test-String-For-{}'.format(key) - settings = {key: test_value} - _sanitize_vcs_settings(settings) - assert settings[key] == test_value.lower() + # @pytest.mark.parametrize('key, default', _string_funcs) + # def test_string_func_lowercase(self, key, default): + # test_value = 'Test-String-For-{}'.format(key) + # settings = {key: test_value} + # sanitize_settings_and_apply_defaults({'__file__': ''}, settings) + # assert settings[key] == test_value.lower() - @pytest.mark.parametrize('key, default', _bool_settings) - def test_bool_setting_true(self, key, default): + @pytest.mark.parametrize('key, default', _bool_funcs) + def test_bool_func_true(self, key, default): settings = {key: 'true'} - _sanitize_vcs_settings(settings) + sanitize_settings_and_apply_defaults({'__file__': ''}, settings) assert settings[key] is True - @pytest.mark.parametrize('key, default', _bool_settings) - def test_bool_setting_false(self, key, default): + @pytest.mark.parametrize('key, default', _bool_funcs) + def test_bool_func_false(self, key, default): settings = {key: 'false'} - _sanitize_vcs_settings(settings) + sanitize_settings_and_apply_defaults({'__file__': ''}, settings) assert settings[key] is False - @pytest.mark.parametrize('key, default', _bool_settings) - def test_bool_setting_invalid_string(self, key, default): + @pytest.mark.parametrize('key, default', _bool_funcs) + def test_bool_func_invalid_string(self, key, default): settings = {key: 'no-bool-val-string'} - _sanitize_vcs_settings(settings) + sanitize_settings_and_apply_defaults({'__file__': ''}, settings) assert settings[key] is False - @pytest.mark.parametrize('key, default', _bool_settings) - def test_bool_setting_default(self, key, default): + @pytest.mark.parametrize('key, default', _bool_funcs) + def test_bool_func_default(self, key, default): settings = {} - _sanitize_vcs_settings(settings) + sanitize_settings_and_apply_defaults({'__file__': ''}, settings) assert settings[key] is default diff --git a/rhodecode/tests/rhodecode.ini b/rhodecode/tests/rhodecode.ini --- a/rhodecode/tests/rhodecode.ini +++ b/rhodecode/tests/rhodecode.ini @@ -1,30 +1,23 @@ - -################################################################################ -## RHODECODE COMMUNITY EDITION CONFIGURATION ## -# The %(here)s variable will be replaced with the parent directory of this file# -################################################################################ +; ######################################### +; RHODECODE COMMUNITY EDITION CONFIGURATION +; ######################################### [DEFAULT] +; Debug flag sets all loggers to debug, and enables request tracking debug = true -################################################################################ -## EMAIL CONFIGURATION ## -## Uncomment and replace with the email address which should receive ## -## any error reports after an application crash ## -## Additionally these settings will be used by the RhodeCode mailing system ## -################################################################################ +; ######################################################################## +; EMAIL CONFIGURATION +; These settings will be used by the RhodeCode mailing system +; ######################################################################## -## prefix all emails subjects with given prefix, helps filtering out emails +; prefix all emails subjects with given prefix, helps filtering out emails #email_prefix = [RhodeCode] -## email FROM address all mails will be sent +; email FROM address all mails will be sent #app_email_from = rhodecode-noreply@localhost -## Uncomment and replace with the address which should receive any error report -## note: using appenlight for error handling doesn't need this to be uncommented -#email_to = admin@localhost - #smtp_server = mail.server.com #smtp_username = #smtp_password = @@ -33,16 +26,20 @@ debug = true #smtp_use_ssl = true [server:main] -## COMMON ## +; COMMON HOST/IP CONFIG host = 0.0.0.0 port = 5000 -########################## -## GUNICORN WSGI SERVER ## -########################## -## run with gunicorn --log-config rhodecode.ini --paste rhodecode.ini + +; ########################### +; GUNICORN APPLICATION SERVER +; ########################### +; run with gunicorn --log-config rhodecode.ini --paste rhodecode.ini + +; Module to use, this setting shouldn't be changed use = egg:gunicorn#main + ## Sets the number of process workers. You must set `instance_id = *` ## when this option is set to more than one worker, recommended ## value is (2 * NUMBER_OF_CPUS + 1), eg 2CPU = 5 workers @@ -81,7 +78,7 @@ prefix = / is_test = True use = egg:rhodecode-enterprise-ce -## enable proxy prefix middleware, defined above +; enable proxy prefix middleware, defined above #filter-with = proxy-prefix @@ -99,64 +96,69 @@ rhodecode.api.url = /_admin/api ## `beaker.session.secret` #rhodecode.encrypted_values.secret = -## decryption strict mode (enabled by default). It controls if decryption raises -## `SignatureVerificationError` in case of wrong key, or damaged encryption data. +; decryption strict mode (enabled by default). It controls if decryption raises +; `SignatureVerificationError` in case of wrong key, or damaged encryption data. #rhodecode.encrypted_values.strict = false -## return gzipped responses from Rhodecode (static files/application) +; Pick algorithm for encryption. Either fernet (more secure) or aes (default) +; fernet is safer, and we strongly recommend switching to it. +; Due to backward compatibility aes is used as default. +#rhodecode.encrypted_values.algorithm = fernet + +; Return gzipped responses from RhodeCode (static files/application) gzip_responses = false -## autogenerate javascript routes file on startup +; Auto-generate javascript routes file on startup generate_js_files = false -## Optional Languages -## en(default), be, de, es, fr, it, ja, pl, pt, ru, zh +; System global default language. +; All available languages: en (default), be, de, es, fr, it, ja, pl, pt, ru, zh lang = en ## perform a full repository scan on each server start, this should be ## set to false after first startup, to allow faster server restarts. startup.import_repos = true -## Uncomment and set this path to use archive download cache. -## Once enabled, generated archives will be cached at this location -## and served from the cache during subsequent requests for the same archive of -## the repository. +; Uncomment and set this path to use archive download cache. +; Once enabled, generated archives will be cached at this location +; and served from the cache during subsequent requests for the same archive of +; the repository. #archive_cache_dir = /tmp/tarballcache -## URL at which the application is running. This is used for bootstraping -## requests in context when no web request is available. Used in ishell, or -## SSH calls. Set this for events to receive proper url for SSH calls. +; URL at which the application is running. This is used for Bootstrapping +; requests in context when no web request is available. Used in ishell, or +; SSH calls. Set this for events to receive proper url for SSH calls. app.base_url = http://rhodecode.local -## change this to unique ID for security +; Unique application ID. Should be a random unique string for security. app_instance_uuid = rc-production ## cut off limit for large diffs (size in bytes) cut_off_limit_diff = 1024000 cut_off_limit_file = 256000 -## use cache version of scm repo everywhere +; Use cached version of vcs repositories everywhere. Recommended to be `true` vcs_full_cache = false -## force https in RhodeCode, fixes https redirects, assumes it's always https -## Normally this is controlled by proper http flags sent from http server +; Force https in RhodeCode, fixes https redirects, assumes it's always https. +; Normally this is controlled by proper flags sent from http server such as Nginx or Apache force_https = false -## use Strict-Transport-Security headers +; use Strict-Transport-Security headers use_htsts = false -# Set to true if your repos are exposed using the dumb protocol +; Set to true if your repos are exposed using the dumb protocol git_update_server_info = false -## RSS/ATOM feed options +; RSS/ATOM feed options rss_cut_off_limit = 256000 rss_items_per_page = 10 rss_include_diff = false -## gist URL alias, used to create nicer urls for gist. This should be an -## url that does rewrites to _admin/gists/{gistid}. -## example: http://gist.rhodecode.org/{gistid}. Empty means use the internal -## RhodeCode url, ie. http[s]://rhodecode.server/_admin/gists/{gistid} +; gist URL alias, used to create nicer urls for gist. This should be an +; url that does rewrites to _admin/gists/{gistid}. +; example: http://gist.rhodecode.org/{gistid}. Empty means use the internal +; RhodeCode url, ie. http[s]://rhodecode.server/_admin/gists/{gistid} gist_alias_url = ## List of views (using glob pattern syntax) that AUTH TOKENS could be @@ -181,36 +183,39 @@ gist_alias_url = # GistView:* api_access_controllers_whitelist = -## default encoding used to convert from and to unicode -## can be also a comma separated list of encoding in case of mixed encodings +; Default encoding used to convert from and to unicode +; can be also a comma separated list of encoding in case of mixed encodings default_encoding = UTF-8 -## instance-id prefix -## a prefix key for this instance used for cache invalidation when running -## multiple instances of rhodecode, make sure it's globally unique for -## all running rhodecode instances. Leave empty if you don't use it +; instance-id prefix +; a prefix key for this instance used for cache invalidation when running +; multiple instances of RhodeCode, make sure it's globally unique for +; all running RhodeCode instances. Leave empty if you don't use it instance_id = -## Fallback authentication plugin. Set this to a plugin ID to force the usage -## of an authentication plugin also if it is disabled by it's settings. -## This could be useful if you are unable to log in to the system due to broken -## authentication settings. Then you can enable e.g. the internal rhodecode auth -## module to log in again and fix the settings. -## -## Available builtin plugin IDs (hash is part of the ID): -## egg:rhodecode-enterprise-ce#rhodecode -## egg:rhodecode-enterprise-ce#pam -## egg:rhodecode-enterprise-ce#ldap -## egg:rhodecode-enterprise-ce#jasig_cas -## egg:rhodecode-enterprise-ce#headers -## egg:rhodecode-enterprise-ce#crowd +; Fallback authentication plugin. Set this to a plugin ID to force the usage +; of an authentication plugin also if it is disabled by it's settings. +; This could be useful if you are unable to log in to the system due to broken +; authentication settings. Then you can enable e.g. the internal RhodeCode auth +; module to log in again and fix the settings. +; Available builtin plugin IDs (hash is part of the ID): +; egg:rhodecode-enterprise-ce#rhodecode +; egg:rhodecode-enterprise-ce#pam +; egg:rhodecode-enterprise-ce#ldap +; egg:rhodecode-enterprise-ce#jasig_cas +; egg:rhodecode-enterprise-ce#headers +; egg:rhodecode-enterprise-ce#crowd + #rhodecode.auth_plugin_fallback = egg:rhodecode-enterprise-ce#rhodecode -## alternative return HTTP header for failed authentication. Default HTTP -## response is 401 HTTPUnauthorized. Currently HG clients have troubles with -## handling that causing a series of failed authentication calls. -## Set this variable to 403 to return HTTPForbidden, or any other HTTP code -## This will be served instead of default 401 on bad authnetication +; Flag to control loading of legacy plugins in py:/path format +auth_plugin.import_legacy_plugins = true + +; alternative return HTTP header for failed authentication. Default HTTP +; response is 401 HTTPUnauthorized. Currently HG clients have troubles with +; handling that causing a series of failed authentication calls. +; Set this variable to 403 to return HTTPForbidden, or any other HTTP code +; This will be served instead of default 401 on bad authentication auth_ret_code = ## use special detection method when serving auth_ret_code, instead of serving @@ -240,38 +245,35 @@ supervisor.group_id = dev ## Display extended labs settings labs_settings_active = true -#################################### -### CELERY CONFIG #### -#################################### +; ############# +; CELERY CONFIG +; ############# + +; manually run celery: /path/to/celery worker -E --beat --app rhodecode.lib.celerylib.loader --scheduler rhodecode.lib.celerylib.scheduler.RcScheduler --loglevel DEBUG --ini /path/to/rhodecode.ini + use_celery = false -broker.host = localhost -broker.vhost = rabbitmqhost -broker.port = 5672 -broker.user = rabbitmq -broker.password = qweqwe -celery.imports = rhodecode.lib.celerylib.tasks +; path to store schedule database +#celerybeat-schedule.path = -celery.result.backend = amqp -celery.result.dburi = amqp:// -celery.result.serialier = json +; connection url to the message broker (default redis) +celery.broker_url = redis://localhost:6379/8 -#celery.send.task.error.emails = true -#celery.amqp.task.result.expires = 18000 +; rabbitmq example +#celery.broker_url = amqp://rabbitmq:qweqwe@localhost:5672/rabbitmqhost -celeryd.concurrency = 2 -#celeryd.log.file = celeryd.log -celeryd.log.level = debug -celeryd.max.tasks.per.child = 1 +; maximum tasks to execute before worker restart +celery.max_tasks_per_child = 100 -## tasks will never be sent to the queue, but executed locally instead. -celery.always.eager = false +; tasks will never be sent to the queue, but executed locally instead. +celery.task_always_eager = false -#################################### -### BEAKER CACHE #### -#################################### -# default cache dir for templates. Putting this into a ramdisk -## can boost performance, eg. %(here)s/data_ramdisk +; ############# +; DOGPILE CACHE +; ############# + +; Default cache dir for caches. Putting this into a ramdisk can boost performance. +; eg. /tmpfs/data_ramdisk, however this directory might require large amount of space cache_dir = %(here)s/data ## locking and default file storage for Beaker. Putting this into a ramdisk @@ -301,16 +303,21 @@ rc_cache.sql_cache_short.backend = dogpi rc_cache.sql_cache_short.expiration_time = 0 -#################################### -### BEAKER SESSION #### -#################################### +; ############## +; BEAKER SESSION +; ############## -## .session.type is type of storage options for the session, current allowed -## types are file, ext:memcached, ext:database, and memory (default). +; beaker.session.type is type of storage options for the logged users sessions. Current allowed +; types are file, ext:redis, ext:database, ext:memcached, and memory (default if not specified). +; Fastest ones are Redis and ext:database beaker.session.type = file beaker.session.data_dir = %(here)s/rc/data/sessions/data -## db based session, fast, and allows easy management over logged in users +; Redis based sessions +#beaker.session.type = ext:redis +#beaker.session.url = redis://127.0.0.1:6379/2 + +; DB based session, fast, and allows easy management over logged in users #beaker.session.type = ext:database #beaker.session.table_name = db_session #beaker.session.sa.url = postgresql://postgres:secret@localhost/rhodecode @@ -322,19 +329,20 @@ beaker.session.key = rhodecode beaker.session.secret = test-rc-uytcxaz beaker.session.lock_dir = %(here)s/rc/data/sessions/lock -## Secure encrypted cookie. Requires AES and AES python libraries -## you must disable beaker.session.secret to use this +; Secure encrypted cookie. Requires AES and AES python libraries +; you must disable beaker.session.secret to use this #beaker.session.encrypt_key = key_for_encryption #beaker.session.validate_key = validation_key -## sets session as invalid(also logging out user) if it haven not been -## accessed for given amount of time in seconds +; Sets session as invalid (also logging out user) if it haven not been +; accessed for given amount of time in seconds beaker.session.timeout = 2592000 beaker.session.httponly = true -## Path to use for the cookie. Set to prefix if you use prefix middleware + +; Path to use for the cookie. Set to prefix if you use prefix middleware #beaker.session.cookie_path = /custom_prefix -## uncomment for https secure cookie +; Set https secure cookie beaker.session.secure = false ## auto save the session to not to use .save() @@ -344,242 +352,213 @@ beaker.session.auto = false ## at browser close #beaker.session.cookie_expires = 3600 -################################### -## SEARCH INDEXING CONFIGURATION ## -################################### -## Full text search indexer is available in rhodecode-tools under -## `rhodecode-tools index` command +; ############################# +; SEARCH INDEXING CONFIGURATION +; ############################# ## WHOOSH Backend, doesn't require additional services to run ## it works good with few dozen repos search.module = rhodecode.lib.index.whoosh search.location = %(here)s/data/index -######################################## -### CHANNELSTREAM CONFIG #### -######################################## -## channelstream enables persistent connections and live notification -## in the system. It's also used by the chat system +; #################### +; CHANNELSTREAM CONFIG +; #################### + +; channelstream enables persistent connections and live notification +; in the system. It's also used by the chat system channelstream.enabled = false -## server address for channelstream server on the backend +; server address for channelstream server on the backend channelstream.server = 127.0.0.1:9800 -## location of the channelstream server from outside world -## use ws:// for http or wss:// for https. This address needs to be handled -## by external HTTP server such as Nginx or Apache -## see nginx/apache configuration examples in our docs + +; location of the channelstream server from outside world +; use ws:// for http or wss:// for https. This address needs to be handled +; by external HTTP server such as Nginx or Apache +; see Nginx/Apache configuration examples in our docs channelstream.ws_url = ws://rhodecode.yourserver.com/_channelstream channelstream.secret = secret channelstream.history.location = %(here)s/channelstream_history -## Internal application path that Javascript uses to connect into. -## If you use proxy-prefix the prefix should be added before /_channelstream +; Internal application path that Javascript uses to connect into. +; If you use proxy-prefix the prefix should be added before /_channelstream channelstream.proxy_path = /_channelstream -################################### -## APPENLIGHT CONFIG ## -################################### - -## Appenlight is tailored to work with RhodeCode, see -## http://appenlight.com for details how to obtain an account - -## appenlight integration enabled -appenlight = false - -appenlight.server_url = https://api.appenlight.com -appenlight.api_key = YOUR_API_KEY -#appenlight.transport_config = https://api.appenlight.com?threaded=1&timeout=5 - -# used for JS client -appenlight.api_public_key = YOUR_API_PUBLIC_KEY - -## TWEAK AMOUNT OF INFO SENT HERE - -## enables 404 error logging (default False) -appenlight.report_404 = false - -## time in seconds after request is considered being slow (default 1) -appenlight.slow_request_time = 1 - -## record slow requests in application -## (needs to be enabled for slow datastore recording and time tracking) -appenlight.slow_requests = true - -## enable hooking to application loggers -appenlight.logging = true - -## minimum log level for log capture -appenlight.logging.level = WARNING - -## send logs only from erroneous/slow requests -## (saves API quota for intensive logging) -appenlight.logging_on_error = false +; ############################## +; MAIN RHODECODE DATABASE CONFIG +; ############################## -## list of additonal keywords that should be grabbed from environ object -## can be string with comma separated list of words in lowercase -## (by default client will always send following info: -## 'REMOTE_USER', 'REMOTE_ADDR', 'SERVER_NAME', 'CONTENT_TYPE' + all keys that -## start with HTTP* this list be extended with additional keywords here -appenlight.environ_keys_whitelist = - -## list of keywords that should be blanked from request object -## can be string with comma separated list of words in lowercase -## (by default client will always blank keys that contain following words -## 'password', 'passwd', 'pwd', 'auth_tkt', 'secret', 'csrf' -## this list be extended with additional keywords set here -appenlight.request_keys_blacklist = - -## list of namespaces that should be ignores when gathering log entries -## can be string with comma separated list of namespaces -## (by default the client ignores own entries: appenlight_client.client) -appenlight.log_namespace_blacklist = - +#sqlalchemy.db1.url = sqlite:///%(here)s/rhodecode.db?timeout=30 +#sqlalchemy.db1.url = postgresql://postgres:qweqwe@localhost/rhodecode +#sqlalchemy.db1.url = mysql://root:qweqwe@localhost/rhodecode?charset=utf8 +; pymysql is an alternative driver for MySQL, use in case of problems with default one +#sqlalchemy.db1.url = mysql+pymysql://root:qweqwe@localhost/rhodecode -################################################################################ -## WARNING: *THE LINE BELOW MUST BE UNCOMMENTED ON A PRODUCTION ENVIRONMENT* ## -## Debug mode will enable the interactive debugging tool, allowing ANYONE to ## -## execute malicious code after an exception is raised. ## -################################################################################ -set debug = false - - -############## -## STYLING ## -############## -debug_style = false - -########################################### -### MAIN RHODECODE DATABASE CONFIG ### -########################################### -#sqlalchemy.db1.url = sqlite:///%(here)s/rhodecode_test.db?timeout=30 -#sqlalchemy.db1.url = postgresql://postgres:qweqwe@localhost/rhodecode_test -#sqlalchemy.db1.url = mysql://root:qweqwe@localhost/rhodecode_test sqlalchemy.db1.url = sqlite:///%(here)s/rhodecode_test.db?timeout=30 -# see sqlalchemy docs for other advanced settings - -## print the sql statements to output +; see sqlalchemy docs for other advanced settings +; print the sql statements to output sqlalchemy.db1.echo = false -## recycle the connections after this amount of seconds -sqlalchemy.db1.pool_recycle = 3600 -## the number of connections to keep open inside the connection pool. -## 0 indicates no limit +; recycle the connections after this amount of seconds +sqlalchemy.db1.pool_recycle = 3600 +sqlalchemy.db1.convert_unicode = true + +; the number of connections to keep open inside the connection pool. +; 0 indicates no limit #sqlalchemy.db1.pool_size = 5 -## the number of connections to allow in connection pool "overflow", that is -## connections that can be opened above and beyond the pool_size setting, -## which defaults to five. +; The number of connections to allow in connection pool "overflow", that is +; connections that can be opened above and beyond the pool_size setting, +; which defaults to five. #sqlalchemy.db1.max_overflow = 10 +; Connection check ping, used to detect broken database connections +; could be enabled to better handle cases if MySQL has gone away errors +#sqlalchemy.db1.ping_connection = true -################## -### VCS CONFIG ### -################## +; ########## +; VCS CONFIG +; ########## vcs.server.enable = true vcs.server = localhost:9901 -## Web server connectivity protocol, responsible for web based VCS operatations -## Available protocols are: -## `http` - use http-rpc backend (default) +; Web server connectivity protocol, responsible for web based VCS operations +; Available protocols are: +; `http` - use http-rpc backend (default) vcs.server.protocol = http -## Push/Pull operations protocol, available options are: -## `http` - use http-rpc backend (default) -## `vcsserver.scm_app` - internal app (EE only) +; Push/Pull operations protocol, available options are: +; `http` - use http-rpc backend (default) vcs.scm_app_implementation = http -## Push/Pull operations hooks protocol, available options are: -## `http` - use http-rpc backend (default) +; Push/Pull operations hooks protocol, available options are: +; `http` - use http-rpc backend (default) vcs.hooks.protocol = http + +; Host on which this instance is listening for hooks. If vcsserver is in other location +; this should be adjusted. vcs.hooks.host = 127.0.0.1 - -## Start VCSServer with this instance as a subprocess, Useful for development +; Start VCSServer with this instance as a subprocess, useful for development vcs.start_server = false -## List of enabled VCS backends, available options are: -## `hg` - mercurial -## `git` - git -## `svn` - subversion +; List of enabled VCS backends, available options are: +; `hg` - mercurial +; `git` - git +; `svn` - subversion vcs.backends = hg, git, svn +; Wait this number of seconds before killing connection to the vcsserver vcs.connection_timeout = 3600 -## Compatibility version when creating SVN repositories. Defaults to newest version when commented out. -## Available options are: pre-1.4-compatible, pre-1.5-compatible, pre-1.6-compatible, pre-1.8-compatible, pre-1.9-compatible -#vcs.svn.compatible_version = pre-1.8-compatible +; Compatibility version when creating SVN repositories. Defaults to newest version when commented out. +; Set a numeric version for your current SVN e.g 1.8, or 1.12 +; Legacy available options are: pre-1.4-compatible, pre-1.5-compatible, pre-1.6-compatible, pre-1.8-compatible, pre-1.9-compatible +#vcs.svn.compatible_version = 1.8 -############################################################ -### Subversion proxy support (mod_dav_svn) ### -### Maps RhodeCode repo groups into SVN paths for Apache ### -############################################################ -## Enable or disable the config file generation. +; Cache flag to cache vcsserver remote calls locally +; It uses cache_region `cache_repo` +vcs.methods.cache = false + +; #################################################### +; Subversion proxy support (mod_dav_svn) +; Maps RhodeCode repo groups into SVN paths for Apache +; #################################################### + +; Enable or disable the config file generation. svn.proxy.generate_config = false -## Generate config file with `SVNListParentPath` set to `On`. + +; Generate config file with `SVNListParentPath` set to `On`. svn.proxy.list_parent_path = true -## Set location and file name of generated config file. + +; Set location and file name of generated config file. svn.proxy.config_file_path = %(here)s/mod_dav_svn.conf -## Used as a prefix to the `Location` block in the generated config file. -## In most cases it should be set to `/`. + +; alternative mod_dav config template. This needs to be a valid mako template +; Example template can be found in the source code: +; rhodecode/apps/svn_support/templates/mod-dav-svn.conf.mako +#svn.proxy.config_template = ~/.rccontrol/enterprise-1/custom_svn_conf.mako + +; Used as a prefix to the `Location` block in the generated config file. +; In most cases it should be set to `/`. svn.proxy.location_root = / -## Command to reload the mod dav svn configuration on change. -## Example: `/etc/init.d/apache2 reload` + +; Command to reload the mod dav svn configuration on change. +; Example: `/etc/init.d/apache2 reload` or /home/USER/apache_reload.sh +; Make sure user who runs RhodeCode process is allowed to reload Apache #svn.proxy.reload_cmd = /etc/init.d/apache2 reload -## If the timeout expires before the reload command finishes, the command will -## be killed. Setting it to zero means no timeout. Defaults to 10 seconds. + +; If the timeout expires before the reload command finishes, the command will +; be killed. Setting it to zero means no timeout. Defaults to 10 seconds. #svn.proxy.reload_timeout = 10 -############################################################ -### SSH Support Settings ### -############################################################ +; #################### +; SSH Support Settings +; #################### -## Defines if the authorized_keys file should be written on any change of -## user ssh keys, setting this to false also disables posibility of adding -## ssh keys for users from web interface. +; Defines if a custom authorized_keys file should be created and written on +; any change user ssh keys. Setting this to false also disables possibility +; of adding SSH keys by users from web interface. Super admins can still +; manage SSH Keys. ssh.generate_authorized_keyfile = true -## Options for ssh, default is `no-pty,no-port-forwarding,no-X11-forwarding,no-agent-forwarding` +; Options for ssh, default is `no-pty,no-port-forwarding,no-X11-forwarding,no-agent-forwarding` # ssh.authorized_keys_ssh_opts = -## File to generate the authorized keys together with options -## It is possible to have multiple key files specified in `sshd_config` e.g. -## AuthorizedKeysFile %h/.ssh/authorized_keys %h/.ssh/authorized_keys_rhodecode +; Path to the authorized_keys file where the generate entries are placed. +; It is possible to have multiple key files specified in `sshd_config` e.g. +; AuthorizedKeysFile %h/.ssh/authorized_keys %h/.ssh/authorized_keys_rhodecode ssh.authorized_keys_file_path = %(here)s/rc/authorized_keys_rhodecode -## Command to execute the SSH wrapper. The binary is available in the -## rhodecode installation directory. -## e.g ~/.rccontrol/community-1/profile/bin/rc-ssh-wrapper +; Command to execute the SSH wrapper. The binary is available in the +; RhodeCode installation directory. +; e.g ~/.rccontrol/community-1/profile/bin/rc-ssh-wrapper ssh.wrapper_cmd = ~/.rccontrol/community-1/rc-ssh-wrapper -## Allow shell when executing the ssh-wrapper command +; Allow shell when executing the ssh-wrapper command ssh.wrapper_cmd_allow_shell = false -## Enables logging, and detailed output send back to the client. Useful for -## debugging, shouldn't be used in production. +; Enables logging, and detailed output send back to the client during SSH +; operations. Useful for debugging, shouldn't be used in production. ssh.enable_debug_logging = false -## Paths to binary executrables, by default they are the names, but we can -## override them if we want to use a custom one +; Paths to binary executable, by default they are the names, but we can +; override them if we want to use a custom one ssh.executable.hg = ~/.rccontrol/vcsserver-1/profile/bin/hg ssh.executable.git = ~/.rccontrol/vcsserver-1/profile/bin/git ssh.executable.svn = ~/.rccontrol/vcsserver-1/profile/bin/svnserve -## Enables SSH key generator web interface. Disabling this still allows users -## to add their own keys. +; Enables SSH key generator web interface. Disabling this still allows users +; to add their own keys. ssh.enable_ui_key_generator = true +; Statsd client config, this is used to send metrics to statsd +; We recommend setting statsd_exported and scrape them using Promethues +#statsd.enabled = false +#statsd.statsd_host = 0.0.0.0 +#statsd.statsd_port = 8125 +#statsd.statsd_prefix = +#statsd.statsd_ipv6 = false -## Dummy marker to add new entries after. -## Add any custom entries below. Please don't remove. + +; configure logging automatically at server startup set to false +; to use the below custom logging config. +logging.autoconfigure = false + +; specify your own custom logging config file to configure logging +#logging.logging_conf_file = /path/to/custom_logging.ini + +; Dummy marker to add new entries after. +; Add any custom entries below. Please don't remove this marker. custom.conf = 1 -################################ -### LOGGING CONFIGURATION #### -################################ +; ##################### +; LOGGING CONFIGURATION +; ##################### [loggers] keys = root, sqlalchemy, beaker, rhodecode, ssh_wrapper @@ -589,9 +568,9 @@ keys = console, console_sql [formatters] keys = generic, color_formatter, color_formatter_sql -############# -## LOGGERS ## -############# +; ####### +; LOGGERS +; ####### [logger_root] level = NOTSET handlers = console @@ -603,6 +582,12 @@ qualname = routes.middleware ## "level = DEBUG" logs the route matched and routing variables. propagate = 1 +[logger_sqlalchemy] +level = INFO +handlers = console_sql +qualname = sqlalchemy.engine +propagate = 0 + [logger_beaker] level = DEBUG handlers = @@ -615,50 +600,59 @@ handlers = qualname = rhodecode propagate = 1 -[logger_sqlalchemy] -level = ERROR -handlers = console_sql -qualname = sqlalchemy.engine -propagate = 0 - [logger_ssh_wrapper] level = DEBUG handlers = qualname = ssh_wrapper propagate = 1 +[logger_celery] +level = DEBUG +handlers = +qualname = celery -############## -## HANDLERS ## -############## + +; ######## +; HANDLERS +; ######## [handler_console] class = StreamHandler -args = (sys.stderr,) +args = (sys.stderr, ) level = DEBUG formatter = generic +; To enable JSON formatted logs replace generic with json +; This allows sending properly formatted logs to grafana loki or elasticsearch +#formatter = json [handler_console_sql] +; "level = DEBUG" logs SQL queries and results. +; "level = INFO" logs SQL queries. +; "level = WARN" logs neither. (Recommended for production systems.) class = StreamHandler -args = (sys.stderr,) +args = (sys.stderr, ) level = WARN formatter = generic -################ -## FORMATTERS ## -################ +; ########## +; FORMATTERS +; ########## [formatter_generic] class = rhodecode.lib.logging_formatter.ExceptionAwareFormatter -format = %(asctime)s.%(msecs)03d %(levelname)-5.5s [%(name)s] %(message)s +format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s datefmt = %Y-%m-%d %H:%M:%S [formatter_color_formatter] class = rhodecode.lib.logging_formatter.ColorFormatter -format = %(asctime)s.%(msecs)03d %(levelname)-5.5s [%(name)s] %(message)s +format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s datefmt = %Y-%m-%d %H:%M:%S [formatter_color_formatter_sql] class = rhodecode.lib.logging_formatter.ColorFormatterSql -format = %(asctime)s.%(msecs)03d %(levelname)-5.5s [%(name)s] %(message)s +format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s datefmt = %Y-%m-%d %H:%M:%S + +[formatter_json] +format = %(message)s +class = rhodecode.lib._vendor.jsonlogger.JsonFormatter \ No newline at end of file