config: major update for the code to make it be almost fully controllable via env for new docker based installer.

configs/logging.ini

@@ -0,0 +1,98 b''
+; #####################
+; LOGGING CONFIGURATION
+; #####################
+[loggers]
+keys = root, sqlalchemy, beaker, celery, rhodecode, ssh_wrapper
+
+[handlers]
+keys = console, console_sql
+
+[formatters]
+keys = generic, json, color_formatter, color_formatter_sql
+
+; #######
+; LOGGERS
+; #######
+[logger_root]
+level = NOTSET
+handlers = console
+
+[logger_sqlalchemy]
+level = $RC_LOGGING_LEVEL
+handlers = console_sql
+qualname = sqlalchemy.engine
+propagate = 0
+
+[logger_beaker]
+level = $RC_LOGGING_LEVEL
+handlers =
+qualname = beaker.container
+propagate = 1
+
+[logger_rhodecode]
+level = $RC_LOGGING_LEVEL
+handlers =
+qualname = rhodecode
+propagate = 1
+
+[logger_ssh_wrapper]
+level = $RC_LOGGING_LEVEL
+handlers =
+qualname = ssh_wrapper
+propagate = 1
+
+[logger_celery]
+level = $RC_LOGGING_LEVEL
+handlers =
+qualname = celery
+
+
+; ########
+; HANDLERS
+; ########
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr, )
+level = $RC_LOGGING_LEVEL
+; To enable JSON formatted logs replace generic with json
+; This allows sending properly formatted logs to grafana loki or elasticsearch
+#formatter = json
+#formatter = generic
+formatter = $RC_LOGGING_FORMATTER
+
+[handler_console_sql]
+; "level = DEBUG" logs SQL queries and results.
+; "level = INFO" logs SQL queries.
+; "level = WARN" logs neither.  (Recommended for production systems.)
+class = StreamHandler
+args = (sys.stderr, )
+level = WARN
+; To enable JSON formatted logs replace generic with json
+; This allows sending properly formatted logs to grafana loki or elasticsearch
+#formatter = json
+#formatter = generic
+formatter = $RC_LOGGING_FORMATTER
+
+; ##########
+; FORMATTERS
+; ##########
+
+[formatter_generic]
+class = rhodecode.lib.logging_formatter.ExceptionAwareFormatter
+format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s
+datefmt = %Y-%m-%d %H:%M:%S
+
+[formatter_color_formatter]
+class = rhodecode.lib.logging_formatter.ColorFormatter
+format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s
+datefmt = %Y-%m-%d %H:%M:%S
+
+[formatter_color_formatter_sql]
+class = rhodecode.lib.logging_formatter.ColorFormatterSql
+format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s
+datefmt = %Y-%m-%d %H:%M:%S
+
+[formatter_json]
+format = %(message)s
+class = rhodecode.lib._vendor.jsonlogger.JsonFormatter
  No newline at end of file

rhodecode/config/settings_maker.py

@@ -0,0 +1,182 b''
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2010-2020 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+
+import os
+import textwrap
+import string
+import functools
+import logging
+import tempfile
+import logging.config
+log = logging.getLogger(__name__)
+
+
+def str2bool(_str):
+    """
+    returns True/False value from given string, it tries to translate the
+    string into boolean
+
+    :param _str: string value to translate into boolean
+    :rtype: boolean
+    :returns: boolean from given string
+    """
+    if _str is None:
+        return False
+    if _str in (True, False):
+        return _str
+    _str = str(_str).strip().lower()
+    return _str in ('t', 'true', 'y', 'yes', 'on', '1')
+
+
+def aslist(obj, sep=None, strip=True):
+    """
+    Returns given string separated by sep as list
+
+    :param obj:
+    :param sep:
+    :param strip:
+    """
+    if isinstance(obj, (basestring,)):
+        if obj in ['', ""]:
+            return []
+
+        lst = obj.split(sep)
+        if strip:
+            lst = [v.strip() for v in lst]
+        return lst
+    elif isinstance(obj, (list, tuple)):
+        return obj
+    elif obj is None:
+        return []
+    else:
+        return [obj]
+
+
+class SettingsMaker(object):
+
+    def __init__(self, app_settings):
+        self.settings = app_settings
+
+    @classmethod
+    def _bool_func(cls, input_val):
+        if isinstance(input_val, unicode):
+            input_val = input_val.encode('utf8')
+        return str2bool(input_val)
+
+    @classmethod
+    def _int_func(cls, input_val):
+        return int(input_val)
+
+    @classmethod
+    def _list_func(cls, input_val, sep=','):
+        return aslist(input_val, sep=sep)
+
+    @classmethod
+    def _string_func(cls, input_val, lower=True):
+        if lower:
+            input_val = input_val.lower()
+        return input_val
+
+    @classmethod
+    def _float_func(cls, input_val):
+        return float(input_val)
+
+    @classmethod
+    def _dir_func(cls, input_val, ensure_dir=False, mode=0o755):
+
+        # ensure we have our dir created
+        if not os.path.isdir(input_val) and ensure_dir:
+            os.makedirs(input_val, mode=mode)
+
+        if not os.path.isdir(input_val):
+            raise Exception('Dir at {} does not exist'.format(input_val))
+        return input_val
+
+    @classmethod
+    def _file_path_func(cls, input_val, ensure_dir=False, mode=0o755):
+        dirname = os.path.dirname(input_val)
+        cls._dir_func(dirname, ensure_dir=ensure_dir)
+        return input_val
+
+    @classmethod
+    def _key_transformator(cls, key):
+        return "{}_{}".format('RC'.upper(), key.upper().replace('.', '_').replace('-', '_'))
+
+    def enable_logging(self, logging_conf=None, level='INFO', formatter='generic'):
+        """
+        Helper to enable debug on running instance
+        :return:
+        """
+
+        if not str2bool(self.settings.get('logging.autoconfigure')):
+            log.info('logging configuration based on main .ini file')
+            return
+
+        if logging_conf is None:
+            logging_conf = self.settings.get('logging.logging_conf_file') or ''
+
+        if not os.path.isfile(logging_conf):
+            log.error('Unable to setup logging based on %s, file does not exist...', logging_conf)
+            return
+
+        with open(logging_conf, 'rb') as f:
+            ini_template = textwrap.dedent(f.read())
+            ini_template = string.Template(ini_template).safe_substitute(
+                RC_LOGGING_LEVEL=os.environ.get('RC_LOGGING_LEVEL', '') or level,
+                RC_LOGGING_FORMATTER=os.environ.get('RC_LOGGING_FORMATTER', '') or formatter
+            )
+
+        with tempfile.NamedTemporaryFile(prefix='rc_logging_', suffix='.ini', delete=False) as f:
+            log.info('Saved Temporary LOGGING config at %s', f.name)
+            f.write(ini_template)
+
+        logging.config.fileConfig(f.name)
+        os.remove(f.name)
+
+    def make_setting(self, key, default, lower=False, default_when_empty=False, parser=None):
+
+        input_val = self.settings.get(key, default)
+
+        if default_when_empty and not input_val:
+            # use default value when value is set in the config but it is empty
+            input_val = default
+
+        parser_func = {
+            'bool': self._bool_func,
+            'int': self._int_func,
+            'list': self._list_func,
+            'list:newline': functools.partial(self._list_func, sep='/n'),
+            'list:spacesep': functools.partial(self._list_func, sep=' '),
+            'string': functools.partial(self._string_func, lower=lower),
+            'dir': self._dir_func,
+            'dir:ensured': functools.partial(self._dir_func, ensure_dir=True),
+            'file': self._file_path_func,
+            'file:ensured': functools.partial(self._file_path_func, ensure_dir=True),
+            None: lambda i: i
+        }[parser]
+
+        # now maybe we have this KEY in env, search and use the value with higher priority.
+        transformed_key = self._key_transformator(key)
+        envvar_value = os.environ.get(transformed_key)
+        if envvar_value:
+            log.debug('using `%s` key instead of `%s` key for config', transformed_key, key)
+            input_val = envvar_value
+        self.settings[key] = parser_func(input_val)
+        return self.settings[key]

configs/gunicorn_config.py

@@ -30,12 +30,12 b' worker_tmp_dir = None'
 tmp_upload_dir = None
 
 # Custom log format
-access_log_format = (
-    '%(t)s %(p)s INFO  [GNCRN] %(h)-15s rqt:%(L)s %(s)s %(b)-6s "%(m)s:%(U)s %(q)s" usr:%(u)s "%(f)s" "%(a)s"')
+#access_log_format = (
+#    '%(t)s %(p)s INFO  [GNCRN] %(h)-15s rqt:%(L)s %(s)s %(b)-6s "%(m)s:%(U)s %(q)s" usr:%(u)s "%(f)s" "%(a)s"')
 
 # loki format for easier parsing in grafana
-#access_log_format = (
-#        'time="%(t)s" pid=%(p)s level="INFO" type="[GNCRN]" ip="%(h)-15s" rqt="%(L)s" response_code="%(s)s" response_bytes="%(b)-6s" uri="%(m)s:%(U)s %(q)s" user=":%(u)s" user_agent="%(a)s"')
+access_log_format = (
+    'time="%(t)s" pid=%(p)s level="INFO" type="[GNCRN]" ip="%(h)-15s" rqt="%(L)s" response_code="%(s)s" response_bytes="%(b)-6s" uri="%(m)s:%(U)s %(q)s" user=":%(u)s" user_agent="%(a)s"')
 
 # self adjust workers based on CPU count
 # workers = get_workers()
@@ -97,9 +97,12 b' def post_fork(server, worker):'
         if conf.has_option(section, 'memory_usage_recovery_threshold'):
             _memory_usage_recovery_threshold = conf.getfloat(section, 'memory_usage_recovery_threshold')
 
-    worker._memory_max_usage = _memory_max_usage
-    worker._memory_usage_check_interval = _memory_usage_check_interval
-    worker._memory_usage_recovery_threshold = _memory_usage_recovery_threshold
+    worker._memory_max_usage = int(os.environ.get('RC_GUNICORN_MEMORY_MAX_USAGE', '')
+                                   or _memory_max_usage)
+    worker._memory_usage_check_interval = int(os.environ.get('RC_GUNICORN_MEMORY_USAGE_CHECK_INTERVAL', '')
+                                              or _memory_usage_check_interval)
+    worker._memory_usage_recovery_threshold = float(os.environ.get('RC_GUNICORN_MEMORY_USAGE_RECOVERY_THRESHOLD', '')
+                                                    or _memory_usage_recovery_threshold)
 
     # register memory last check time, with some random offset so we don't recycle all
     # at once

configs/production.ini

@@ -124,10 +124,6 b' use = egg:PasteDeploy#prefix'
 prefix = /
 
 [app:main]
-; The %(here)s variable will be replaced with the absolute path of parent directory
-; of this file
-; In addition ENVIRONMENT variables usage is possible, e.g
-; sqlalchemy.db1.url = {ENV_RC_DB_URL}
 
 use = egg:rhodecode-enterprise-ce
 
@@ -330,6 +326,9 b' file_store.storage_path = %(here)s/data/'
 
 use_celery = false
 
+; path to store schedule database
+#celerybeat-schedule.path =
+
 ; connection url to the message broker (default redis)
 celery.broker_url = redis://localhost:6379/8
 
@@ -575,6 +574,9 b' vcs.connection_timeout = 3600'
 ; Legacy available options are: pre-1.4-compatible, pre-1.5-compatible, pre-1.6-compatible, pre-1.8-compatible, pre-1.9-compatible
 #vcs.svn.compatible_version = 1.8
 
+; Cache flag to cache vcsserver remote calls locally
+; It uses cache_region `cache_repo`
+vcs.methods.cache = true
 
 ; ####################################################
 ; Subversion proxy support (mod_dav_svn)
@@ -657,55 +659,55 b' ssh.enable_ui_key_generator = true'
 ; http://appenlight.rhodecode.com for details how to obtain an account
 
 ; Appenlight integration enabled
-appenlight = false
+#appenlight = false
 
-appenlight.server_url = https://api.appenlight.com
-appenlight.api_key = YOUR_API_KEY
+#appenlight.server_url = https://api.appenlight.com
+#appenlight.api_key = YOUR_API_KEY
 #appenlight.transport_config = https://api.appenlight.com?threaded=1&timeout=5
 
 ; used for JS client
-appenlight.api_public_key = YOUR_API_PUBLIC_KEY
+#appenlight.api_public_key = YOUR_API_PUBLIC_KEY
 
 ; TWEAK AMOUNT OF INFO SENT HERE
 
 ; enables 404 error logging (default False)
-appenlight.report_404 = false
+#appenlight.report_404 = false
 
 ; time in seconds after request is considered being slow (default 1)
-appenlight.slow_request_time = 1
+#appenlight.slow_request_time = 1
 
 ; record slow requests in application
 ; (needs to be enabled for slow datastore recording and time tracking)
-appenlight.slow_requests = true
+#appenlight.slow_requests = true
 
 ; enable hooking to application loggers
-appenlight.logging = true
+#appenlight.logging = true
 
 ; minimum log level for log capture
-appenlight.logging.level = WARNING
+#ppenlight.logging.level = WARNING
 
 ; send logs only from erroneous/slow requests
 ; (saves API quota for intensive logging)
-appenlight.logging_on_error = false
+#appenlight.logging_on_error = false
 
 ; list of additional keywords that should be grabbed from environ object
 ; can be string with comma separated list of words in lowercase
 ; (by default client will always send following info:
 ; 'REMOTE_USER', 'REMOTE_ADDR', 'SERVER_NAME', 'CONTENT_TYPE' + all keys that
 ; start with HTTP* this list be extended with additional keywords here
-appenlight.environ_keys_whitelist =
+#appenlight.environ_keys_whitelist =
 
 ; list of keywords that should be blanked from request object
 ; can be string with comma separated list of words in lowercase
 ; (by default client will always blank keys that contain following words
 ; 'password', 'passwd', 'pwd', 'auth_tkt', 'secret', 'csrf'
 ; this list be extended with additional keywords set here
-appenlight.request_keys_blacklist =
+#appenlight.request_keys_blacklist =
 
 ; list of namespaces that should be ignores when gathering log entries
 ; can be string with comma separated list of namespaces
 ; (by default the client ignores own entries: appenlight_client.client)
-appenlight.log_namespace_blacklist =
+#appenlight.log_namespace_blacklist =
 
 ; Statsd client config, this is used to send metrics to statsd
 ; We recommend setting statsd_exported and scrape them using Promethues
@@ -716,6 +718,13 b' appenlight.log_namespace_blacklist ='
 #statsd.statsd_ipv6 = false
 
 
+; configure logging automatically at server startup set to false
+; to use the below custom logging config.
+#logging.autoconfigure = true
+
+; specify your own custom logging config file to configure logging
+#logging.logging_conf_file = /path/to/custom_logging.ini
+
 ; Dummy marker to add new entries after.
 ; Add any custom entries below. Please don't remove this marker.
 custom.conf = 1

rhodecode/apps/file_store/__init__.py

@@ -19,19 +19,20 b''
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 import os
 from rhodecode.apps.file_store import config_keys
-from rhodecode.config.middleware import _bool_setting, _string_setting
+from rhodecode.config.settings_maker import SettingsMaker
 
 
 def _sanitize_settings_and_apply_defaults(settings):
     """
     Set defaults, convert to python types and validate settings.
     """
-    _bool_setting(settings, config_keys.enabled, 'true')
+    settings_maker = SettingsMaker(settings)
 
-    _string_setting(settings, config_keys.backend, 'local')
+    settings_maker.make_setting(config_keys.enabled, True, parser='bool')
+    settings_maker.make_setting(config_keys.backend, 'local')
 
     default_store = os.path.join(os.path.dirname(settings['__file__']), 'upload_store')
-    _string_setting(settings, config_keys.store_path, default_store)
+    settings_maker.make_setting(config_keys.store_path, default_store)
 
 
 def includeme(config):

rhodecode/apps/ssh_support/__init__.py

@@ -24,7 +24,7 b' from . import config_keys'
 from .events import SshKeyFileChangeEvent
 from .subscribers import generate_ssh_authorized_keys_file_subscriber
 
-from rhodecode.config.middleware import _bool_setting, _string_setting
+from rhodecode.config.settings_maker import SettingsMaker
 
 log = logging.getLogger(__name__)
 
@@ -33,28 +33,20 b' def _sanitize_settings_and_apply_default'
     """
     Set defaults, convert to python types and validate settings.
     """
-    _bool_setting(settings, config_keys.generate_authorized_keyfile, 'false')
-    _bool_setting(settings, config_keys.wrapper_allow_shell, 'false')
-    _bool_setting(settings, config_keys.enable_debug_logging, 'false')
-    _bool_setting(settings, config_keys.ssh_key_generator_enabled, 'true')
+    settings_maker = SettingsMaker(settings)
+
+    settings_maker.make_setting(config_keys.generate_authorized_keyfile, False, parser='bool')
+    settings_maker.make_setting(config_keys.wrapper_allow_shell, False, parser='bool')
+    settings_maker.make_setting(config_keys.enable_debug_logging, False, parser='bool')
+    settings_maker.make_setting(config_keys.ssh_key_generator_enabled, True, parser='bool')
 
-    _string_setting(settings, config_keys.authorized_keys_file_path,
-                    '~/.ssh/authorized_keys_rhodecode',
-                    lower=False)
-    _string_setting(settings, config_keys.wrapper_cmd, '',
-                    lower=False)
-    _string_setting(settings, config_keys.authorized_keys_line_ssh_opts, '',
-                    lower=False)
+    settings_maker.make_setting(config_keys.authorized_keys_file_path, '~/.ssh/authorized_keys_rhodecode')
+    settings_maker.make_setting(config_keys.wrapper_cmd, '')
+    settings_maker.make_setting(config_keys.authorized_keys_line_ssh_opts, '')
 
-    _string_setting(settings, config_keys.ssh_hg_bin,
-                    '~/.rccontrol/vcsserver-1/profile/bin/hg',
-                    lower=False)
-    _string_setting(settings, config_keys.ssh_git_bin,
-                    '~/.rccontrol/vcsserver-1/profile/bin/git',
-                    lower=False)
-    _string_setting(settings, config_keys.ssh_svn_bin,
-                    '~/.rccontrol/vcsserver-1/profile/bin/svnserve',
-                    lower=False)
+    settings_maker.make_setting(config_keys.ssh_hg_bin, '~/.rccontrol/vcsserver-1/profile/bin/hg')
+    settings_maker.make_setting(config_keys.ssh_git_bin, '~/.rccontrol/vcsserver-1/profile/bin/git')
+    settings_maker.make_setting(config_keys.ssh_svn_bin, '~/.rccontrol/vcsserver-1/profile/bin/svnserve')
 
 
 def includeme(config):

rhodecode/apps/svn_support/__init__.py

@@ -19,15 +19,13 b''
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 import os
 import logging
-import shlex
 from pyramid import compat
 
 # Do not use `from rhodecode import events` here, it will be overridden by the
 # events module in this package due to pythons import mechanism.
 from rhodecode.events import RepoGroupEvent
 from rhodecode.subscribers import AsyncSubprocessSubscriber
-from rhodecode.config.middleware import (
-    _bool_setting, _string_setting, _int_setting)
+from rhodecode.config.settings_maker import SettingsMaker
 
 from .events import ModDavSvnConfigChange
 from .subscribers import generate_config_subscriber
@@ -41,13 +39,14 b' def _sanitize_settings_and_apply_default'
     """
     Set defaults, convert to python types and validate settings.
     """
-    _bool_setting(settings, config_keys.generate_config, 'false')
-    _bool_setting(settings, config_keys.list_parent_path, 'true')
-    _int_setting(settings, config_keys.reload_timeout, 10)
-    _string_setting(settings, config_keys.config_file_path, '', lower=False)
-    _string_setting(settings, config_keys.location_root, '/', lower=False)
-    _string_setting(settings, config_keys.reload_command, '', lower=False)
-    _string_setting(settings, config_keys.template, '', lower=False)
+    settings_maker = SettingsMaker(settings)
+    settings_maker.make_setting(config_keys.generate_config, False, parser='bool')
+    settings_maker.make_setting(config_keys.list_parent_path, True, parser='bool')
+    settings_maker.make_setting(config_keys.reload_timeout, 10, parser='bool')
+    settings_maker.make_setting(config_keys.config_file_path, '')
+    settings_maker.make_setting(config_keys.location_root, '/')
+    settings_maker.make_setting(config_keys.reload_command, '')
+    settings_maker.make_setting(config_keys.template, '')
 
     # Convert negative timeout values to zero.
     if settings[config_keys.reload_timeout] < 0:

rhodecode/config/environment.py

@@ -85,38 +85,3 b' def load_pyramid_environment(global_conf'
 
     if vcs_server_enabled:
         connect_vcs(vcs_server_uri, utils.get_vcs_server_protocol(settings))
-
-
-def get_rc_env_settings(prefix='ENV_{}'):
-    return {'ENV_{}'.format(key): value for key, value in os.environ.items()}
-
-
-def substitute_values(mapping, substitutions):
-    result = {}
-
-    try:
-        for key, value in mapping.items():
-            # initialize without substitution first
-            result[key] = value
-
-            # Note: Cannot use regular replacements, since they would clash
-            # with the implementation of ConfigParser. Using "format" instead.
-            try:
-                result[key] = value.format(**substitutions)
-            except KeyError as e:
-                env_var = '{}'.format(e.args[0])
-
-                msg = 'Failed to substitute: `{key}={{{var}}}` with environment entry. ' \
-                      'Make sure your environment has {var} set, or remove this ' \
-                      'variable from config file'.format(key=key, var=env_var)
-
-                if env_var.startswith('ENV_'):
-                    raise ValueError(msg)
-                else:
-                    log.warning(msg)
-
-    except ValueError as e:
-        log.warning('Failed to substitute ENV variable: %s', e)
-        result = mapping
-
-    return result
  No newline at end of file

rhodecode/config/middleware.py

@@ -20,10 +20,10 b''
 
 import os
 import sys
-import logging
 import collections
 import tempfile
 import time
+import logging.config
 
 from paste.gzipper import make_gzip_middleware
 import pyramid.events
@@ -38,7 +38,8 b' from pyramid.renderers import render_to_'
 from rhodecode.model import meta
 from rhodecode.config import patches
 from rhodecode.config import utils as config_utils
-from rhodecode.config.environment import load_pyramid_environment, substitute_values, get_rc_env_settings
+from rhodecode.config.settings_maker import SettingsMaker
+from rhodecode.config.environment import load_pyramid_environment
 
 import rhodecode.events
 from rhodecode.lib.middleware.vcs import VCSMiddleware
@@ -48,7 +49,7 b' from rhodecode.lib.exceptions import VCS'
 from rhodecode.lib.middleware.appenlight import wrap_in_appenlight_if_enabled
 from rhodecode.lib.middleware.https_fixup import HttpsFixup
 from rhodecode.lib.plugins.utils import register_rhodecode_plugin
-from rhodecode.lib.utils2 import aslist as rhodecode_aslist, AttributeDict
+from rhodecode.lib.utils2 import AttributeDict
 from rhodecode.lib.exc_tracking import store_exception
 from rhodecode.subscribers import (
     scan_repositories_if_enabled, write_js_routes_if_enabled,
@@ -87,24 +88,14 b' def make_pyramid_app(global_config, **se'
       cases when these fragments are assembled from another place.
 
     """
-
-    # Allows to use format style "{ENV_NAME}" placeholders in the configuration. It
-    # will be replaced by the value of the environment variable "NAME" in this case.
     start_time = time.time()
     log.info('Pyramid app config starting')
 
-    global_config = substitute_values(global_config, get_rc_env_settings())
-    settings = substitute_values(settings, get_rc_env_settings())
+    sanitize_settings_and_apply_defaults(global_config, settings)
 
     # init and bootstrap StatsdClient
     StatsdClient.setup(settings)
 
-    debug = asbool(global_config.get('debug'))
-    if debug:
-        enable_debug()
-
-    sanitize_settings_and_apply_defaults(global_config, settings)
-
     config = Configurator(settings=settings)
     # Init our statsd at very start
     config.registry.statsd = StatsdClient.statsd
@@ -123,17 +114,62 b' def make_pyramid_app(global_config, **se'
     pyramid_app = wrap_app_in_wsgi_middlewares(pyramid_app, config)
     pyramid_app.config = config
 
-    config.configure_celery(global_config['__file__'])
+    celery_settings = get_celery_config(settings)
+    config.configure_celery(celery_settings)
 
     # creating the app uses a connection - return it after we are done
     meta.Session.remove()
 
     total_time = time.time() - start_time
     log.info('Pyramid app `%s` created and configured in %.2fs',
-             pyramid_app.func_name, total_time)
+             getattr(pyramid_app, 'func_name', 'pyramid_app'), total_time)
     return pyramid_app
 
 
+def get_celery_config(settings):
+    """
+    Converts basic ini configuration into celery 4.X options
+    """
+
+    def key_converter(key_name):
+        pref = 'celery.'
+        if key_name.startswith(pref):
+            return key_name[len(pref):].replace('.', '_').lower()
+
+    def type_converter(parsed_key, value):
+        # cast to int
+        if value.isdigit():
+            return int(value)
+
+        # cast to bool
+        if value.lower() in ['true', 'false', 'True', 'False']:
+            return value.lower() == 'true'
+        return value
+
+    celery_config = {}
+    for k, v in settings.items():
+        pref = 'celery.'
+        if k.startswith(pref):
+            celery_config[key_converter(k)] = type_converter(key_converter(k), v)
+
+    # TODO:rethink if we want to support celerybeat based file config, probably NOT
+    # beat_config = {}
+    # for section in parser.sections():
+    #     if section.startswith('celerybeat:'):
+    #         name = section.split(':', 1)[1]
+    #         beat_config[name] = get_beat_config(parser, section)
+
+    # final compose of settings
+    celery_settings = {}
+
+    if celery_config:
+        celery_settings.update(celery_config)
+    # if beat_config:
+    #     celery_settings.update({'beat_schedule': beat_config})
+
+    return celery_settings
+
+
 def not_found_view(request):
     """
     This creates the view which should be registered as not-found-view to
@@ -263,7 +299,7 b' def includeme(config, auth_resources=Non'
 
     config.add_directive('configure_celery', configure_celery)
 
-    if asbool(settings.get('appenlight', 'false')):
+    if settings.get('appenlight', False):
         config.include('appenlight_client.ext.pyramid_tween')
 
     load_all = should_load_all()
@@ -435,8 +471,28 b' def sanitize_settings_and_apply_defaults'
     function.
     """
 
-    settings.setdefault('rhodecode.edition', 'Community Edition')
-    settings.setdefault('rhodecode.edition_id', 'CE')
+    global_settings_maker = SettingsMaker(global_config)
+    global_settings_maker.make_setting('debug', default=False, parser='bool')
+    debug_enabled = asbool(global_config.get('debug'))
+
+    settings_maker = SettingsMaker(settings)
+
+    settings_maker.make_setting(
+        'logging.autoconfigure',
+        default=True,
+        parser='bool')
+
+    logging_conf = os.path.join(os.path.dirname(global_config.get('__file__')), 'logging.ini')
+    settings_maker.enable_logging(logging_conf, level='INFO' if debug_enabled else 'DEBUG')
+
+    # Default includes, possible to change as a user
+    pyramid_includes = settings_maker.make_setting('pyramid.includes', [], parser='list:newline')
+    log.debug(
+        "Using the following pyramid.includes: %s",
+        pyramid_includes)
+
+    settings_maker.make_setting('rhodecode.edition', 'Community Edition')
+    settings_maker.make_setting('rhodecode.edition_id', 'CE')
 
     if 'mako.default_filters' not in settings:
         # set custom default filters if we don't have it defined
@@ -458,25 +514,41 b' def sanitize_settings_and_apply_defaults'
         if not raw_url.startswith(('redis://', 'rediss://', 'unix://')):
             settings['beaker.session.url'] = 'redis://' + raw_url
 
-    # Default includes, possible to change as a user
-    pyramid_includes = settings.setdefault('pyramid.includes', [])
-    log.debug(
-        "Using the following pyramid.includes: %s",
-        pyramid_includes)
+    settings_maker.make_setting('__file__', global_config.get('__file__'))
 
     # TODO: johbo: Re-think this, usually the call to config.include
     # should allow to pass in a prefix.
-    settings.setdefault('rhodecode.api.url', '/_admin/api')
-    settings.setdefault('__file__', global_config.get('__file__'))
+    settings_maker.make_setting('rhodecode.api.url', '/_admin/api')
 
     # Sanitize generic settings.
-    _list_setting(settings, 'default_encoding', 'UTF-8')
-    _bool_setting(settings, 'is_test', 'false')
-    _bool_setting(settings, 'gzip_responses', 'false')
+    settings_maker.make_setting('default_encoding', 'UTF-8', parser='list')
+    settings_maker.make_setting('is_test', False, parser='bool')
+    settings_maker.make_setting('gzip_responses', False, parser='bool')
 
-    # Call split out functions that sanitize settings for each topic.
-    _sanitize_appenlight_settings(settings)
-    _sanitize_vcs_settings(settings)
+    settings_maker.make_setting('vcs.svn.compatible_version', '')
+    settings_maker.make_setting('vcs.hooks.protocol', 'http')
+    settings_maker.make_setting('vcs.hooks.host', '127.0.0.1')
+    settings_maker.make_setting('vcs.scm_app_implementation', 'http')
+    settings_maker.make_setting('vcs.server', '')
+    settings_maker.make_setting('vcs.server.protocol', 'http')
+    settings_maker.make_setting('startup.import_repos', 'false', parser='bool')
+    settings_maker.make_setting('vcs.hooks.direct_calls', 'false', parser='bool')
+    settings_maker.make_setting('vcs.server.enable', 'true', parser='bool')
+    settings_maker.make_setting('vcs.start_server', 'false', parser='bool')
+    settings_maker.make_setting('vcs.backends', 'hg, git, svn', parser='list')
+    settings_maker.make_setting('vcs.connection_timeout', 3600, parser='int')
+
+    settings_maker.make_setting('vcs.methods.cache', True, parser='bool')
+
+    # Support legacy values of vcs.scm_app_implementation. Legacy
+    # configurations may use 'rhodecode.lib.middleware.utils.scm_app_http', or
+    # disabled since 4.13 'vcsserver.scm_app' which is now mapped to 'http'.
+    scm_app_impl = settings['vcs.scm_app_implementation']
+    if scm_app_impl in ['rhodecode.lib.middleware.utils.scm_app_http', 'vcsserver.scm_app']:
+        settings['vcs.scm_app_implementation'] = 'http'
+
+    settings_maker.make_setting('appenlight', False, parser='bool')
+
     _sanitize_cache_settings(settings)
 
     # configure instance id
@@ -485,277 +557,55 b' def sanitize_settings_and_apply_defaults'
     return settings
 
 
-def enable_debug():
-    """
-    Helper to enable debug on running instance
-    :return:
-    """
-    import tempfile
-    import textwrap
-    import logging.config
-
-    ini_template = textwrap.dedent("""
-    #####################################
-    ### DEBUG LOGGING CONFIGURATION  ####
-    #####################################
-    [loggers]
-    keys = root, sqlalchemy, beaker, celery, rhodecode, ssh_wrapper
-
-    [handlers]
-    keys = console, console_sql
-
-    [formatters]
-    keys = generic, color_formatter, color_formatter_sql
-
-    #############
-    ## LOGGERS ##
-    #############
-    [logger_root]
-    level = NOTSET
-    handlers = console
-
-    [logger_sqlalchemy]
-    level = INFO
-    handlers = console_sql
-    qualname = sqlalchemy.engine
-    propagate = 0
-
-    [logger_beaker]
-    level = DEBUG
-    handlers =
-    qualname = beaker.container
-    propagate = 1
-
-    [logger_rhodecode]
-    level = DEBUG
-    handlers =
-    qualname = rhodecode
-    propagate = 1
-
-    [logger_ssh_wrapper]
-    level = DEBUG
-    handlers =
-    qualname = ssh_wrapper
-    propagate = 1
-
-    [logger_celery]
-    level = DEBUG
-    handlers =
-    qualname = celery
-
-
-    ##############
-    ## HANDLERS ##
-    ##############
-
-    [handler_console]
-    class = StreamHandler
-    args = (sys.stderr, )
-    level = DEBUG
-    formatter = color_formatter
+def _sanitize_cache_settings(settings):
+    settings_maker = SettingsMaker(settings)
 
-    [handler_console_sql]
-    # "level = DEBUG" logs SQL queries and results.
-    # "level = INFO" logs SQL queries.
-    # "level = WARN" logs neither.  (Recommended for production systems.)
-    class = StreamHandler
-    args = (sys.stderr, )
-    level = WARN
-    formatter = color_formatter_sql
-
-    ################
-    ## FORMATTERS ##
-    ################
-
-    [formatter_generic]
-    class = rhodecode.lib.logging_formatter.ExceptionAwareFormatter
-    format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s | %(req_id)s
-    datefmt = %Y-%m-%d %H:%M:%S
-
-    [formatter_color_formatter]
-    class = rhodecode.lib.logging_formatter.ColorRequestTrackingFormatter
-    format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s | %(req_id)s
-    datefmt = %Y-%m-%d %H:%M:%S
-
-    [formatter_color_formatter_sql]
-    class = rhodecode.lib.logging_formatter.ColorFormatterSql
-    format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s
-    datefmt = %Y-%m-%d %H:%M:%S    
-    """)
-
-    with tempfile.NamedTemporaryFile(prefix='rc_debug_logging_', suffix='.ini',
-                                     delete=False) as f:
-        log.info('Saved Temporary DEBUG config at %s', f.name)
-        f.write(ini_template)
-
-    logging.config.fileConfig(f.name)
-    log.debug('DEBUG MODE ON')
-    os.remove(f.name)
-
-
-def _sanitize_appenlight_settings(settings):
-    _bool_setting(settings, 'appenlight', 'false')
-
-
-def _sanitize_vcs_settings(settings):
-    """
-    Applies settings defaults and does type conversion for all VCS related
-    settings.
-    """
-    _string_setting(settings, 'vcs.svn.compatible_version', '')
-    _string_setting(settings, 'vcs.hooks.protocol', 'http')
-    _string_setting(settings, 'vcs.hooks.host', '127.0.0.1')
-    _string_setting(settings, 'vcs.scm_app_implementation', 'http')
-    _string_setting(settings, 'vcs.server', '')
-    _string_setting(settings, 'vcs.server.protocol', 'http')
-    _bool_setting(settings, 'startup.import_repos', 'false')
-    _bool_setting(settings, 'vcs.hooks.direct_calls', 'false')
-    _bool_setting(settings, 'vcs.server.enable', 'true')
-    _bool_setting(settings, 'vcs.start_server', 'false')
-    _list_setting(settings, 'vcs.backends', 'hg, git, svn')
-    _int_setting(settings, 'vcs.connection_timeout', 3600)
-
-    # Support legacy values of vcs.scm_app_implementation. Legacy
-    # configurations may use 'rhodecode.lib.middleware.utils.scm_app_http', or
-    # disabled since 4.13 'vcsserver.scm_app' which is now mapped to 'http'.
-    scm_app_impl = settings['vcs.scm_app_implementation']
-    if scm_app_impl in ['rhodecode.lib.middleware.utils.scm_app_http', 'vcsserver.scm_app']:
-        settings['vcs.scm_app_implementation'] = 'http'
-
-
-def _sanitize_cache_settings(settings):
     temp_store = tempfile.gettempdir()
     default_cache_dir = os.path.join(temp_store, 'rc_cache')
 
     # save default, cache dir, and use it for all backends later.
-    default_cache_dir = _string_setting(
-        settings,
+    default_cache_dir = settings_maker.make_setting(
         'cache_dir',
-        default_cache_dir, lower=False, default_when_empty=True)
-
-    # ensure we have our dir created
-    if not os.path.isdir(default_cache_dir):
-        os.makedirs(default_cache_dir, mode=0o755)
+        default=default_cache_dir, default_when_empty=True,
+        parser='dir:ensured')
 
     # exception store cache
-    _string_setting(
-        settings,
+    settings_maker.make_setting(
         'exception_tracker.store_path',
-        temp_store, lower=False, default_when_empty=True)
-    _bool_setting(
-        settings,
-        'exception_tracker.send_email',
-        'false')
-    _string_setting(
-        settings,
-        'exception_tracker.email_prefix',
-        '[RHODECODE ERROR]', lower=False, default_when_empty=True)
+        default=os.path.join(default_cache_dir, 'exc_store'), default_when_empty=True,
+        parser='dir:ensured'
+    )
+
+    settings_maker.make_setting(
+        'celerybeat-schedule.path',
+        default=os.path.join(default_cache_dir, 'celerybeat_schedule', 'celerybeat-schedule.db'), default_when_empty=True,
+        parser='file:ensured'
+    )
+
+    settings_maker.make_setting('exception_tracker.send_email', False, parser='bool')
+    settings_maker.make_setting('exception_tracker.email_prefix', '[RHODECODE ERROR]', default_when_empty=True)
 
     # cache_perms
-    _string_setting(
-        settings,
-        'rc_cache.cache_perms.backend',
-        'dogpile.cache.rc.file_namespace', lower=False)
-    _int_setting(
-        settings,
-        'rc_cache.cache_perms.expiration_time',
-        60)
-    _string_setting(
-        settings,
-        'rc_cache.cache_perms.arguments.filename',
-        os.path.join(default_cache_dir, 'rc_cache_1'), lower=False)
+    settings_maker.make_setting('rc_cache.cache_perms.backend', 'dogpile.cache.rc.file_namespace')
+    settings_maker.make_setting('rc_cache.cache_perms.expiration_time', 60, parser='int')
+    settings_maker.make_setting('rc_cache.cache_perms.arguments.filename', os.path.join(default_cache_dir, 'rhodecode_cache_perms.db'))
 
     # cache_repo
-    _string_setting(
-        settings,
-        'rc_cache.cache_repo.backend',
-        'dogpile.cache.rc.file_namespace', lower=False)
-    _int_setting(
-        settings,
-        'rc_cache.cache_repo.expiration_time',
-        60)
-    _string_setting(
-        settings,
-        'rc_cache.cache_repo.arguments.filename',
-        os.path.join(default_cache_dir, 'rc_cache_2'), lower=False)
+    settings_maker.make_setting('rc_cache.cache_repo.backend', 'dogpile.cache.rc.file_namespace')
+    settings_maker.make_setting('rc_cache.cache_repo.expiration_time', 60, parser='int')
+    settings_maker.make_setting('rc_cache.cache_repo.arguments.filename', os.path.join(default_cache_dir, 'rhodecode_cache_repo.db'))
 
     # cache_license
-    _string_setting(
-        settings,
-        'rc_cache.cache_license.backend',
-        'dogpile.cache.rc.file_namespace', lower=False)
-    _int_setting(
-        settings,
-        'rc_cache.cache_license.expiration_time',
-        5*60)
-    _string_setting(
-        settings,
-        'rc_cache.cache_license.arguments.filename',
-        os.path.join(default_cache_dir, 'rc_cache_3'), lower=False)
+    settings_maker.make_setting('rc_cache.cache_license.backend', 'dogpile.cache.rc.file_namespace')
+    settings_maker.make_setting('rc_cache.cache_license.expiration_time', 5*60, parser='int')
+    settings_maker.make_setting('rc_cache.cache_license.arguments.filename', os.path.join(default_cache_dir, 'rhodecode_cache_license.db'))
 
     # cache_repo_longterm memory, 96H
-    _string_setting(
-        settings,
-        'rc_cache.cache_repo_longterm.backend',
-        'dogpile.cache.rc.memory_lru', lower=False)
-    _int_setting(
-        settings,
-        'rc_cache.cache_repo_longterm.expiration_time',
-        345600)
-    _int_setting(
-        settings,
-        'rc_cache.cache_repo_longterm.max_size',
-        10000)
+    settings_maker.make_setting('rc_cache.cache_repo_longterm.backend', 'dogpile.cache.rc.memory_lru')
+    settings_maker.make_setting('rc_cache.cache_repo_longterm.expiration_time', 345600, parser='int')
+    settings_maker.make_setting('rc_cache.cache_repo_longterm.max_size', 10000, parser='int')
 
     # sql_cache_short
-    _string_setting(
-        settings,
-        'rc_cache.sql_cache_short.backend',
-        'dogpile.cache.rc.memory_lru', lower=False)
-    _int_setting(
-        settings,
-        'rc_cache.sql_cache_short.expiration_time',
-        30)
-    _int_setting(
-        settings,
-        'rc_cache.sql_cache_short.max_size',
-        10000)
-
-
-def _int_setting(settings, name, default):
-    settings[name] = int(settings.get(name, default))
-    return settings[name]
-
-
-def _bool_setting(settings, name, default):
-    input_val = settings.get(name, default)
-    if isinstance(input_val, unicode):
-        input_val = input_val.encode('utf8')
-    settings[name] = asbool(input_val)
-    return settings[name]
-
-
-def _list_setting(settings, name, default):
-    raw_value = settings.get(name, default)
-
-    old_separator = ','
-    if old_separator in raw_value:
-        # If we get a comma separated list, pass it to our own function.
-        settings[name] = rhodecode_aslist(raw_value, sep=old_separator)
-    else:
-        # Otherwise we assume it uses pyramids space/newline separation.
-        settings[name] = aslist(raw_value)
-    return settings[name]
-
-
-def _string_setting(settings, name, default, lower=True, default_when_empty=False):
-    value = settings.get(name, default)
-
-    if default_when_empty and not value:
-        # use default value when value is empty
-        value = default
-
-    if lower:
-        value = value.lower()
-    settings[name] = value
-    return settings[name]
+    settings_maker.make_setting('rc_cache.sql_cache_short.backend', 'dogpile.cache.rc.memory_lru')
+    settings_maker.make_setting('rc_cache.sql_cache_short.expiration_time', 30, parser='int')
+    settings_maker.make_setting('rc_cache.sql_cache_short.max_size', 10000, parser='int')

rhodecode/lib/base.py

@@ -371,7 +371,7 b' def attach_context_attributes(context, r'
         config.get('license.hide_license_info', False))
 
     # AppEnlight
-    context.appenlight_enabled = str2bool(config.get('appenlight', 'false'))
+    context.appenlight_enabled = config.get('appenlight', False)
     context.appenlight_api_public_key = config.get(
         'appenlight.api_public_key', '')
     context.appenlight_server_url = config.get('appenlight.server_url', '')

rhodecode/lib/celerylib/loader.py

@@ -40,7 +40,7 b' from pyramid.threadlocal import get_curr'
 import rhodecode
 
 from rhodecode.lib.auth import AuthUser
-from rhodecode.lib.celerylib.utils import get_ini_config, parse_ini_vars, ping_db
+from rhodecode.lib.celerylib.utils import parse_ini_vars, ping_db
 from rhodecode.lib.ext_json import json
 from rhodecode.lib.pyramid_utils import bootstrap, setup_logging, prepare_request
 from rhodecode.lib.utils2 import str2bool
@@ -116,6 +116,8 b' def setup_logging_callback(**kwargs):'
 
 @signals.user_preload_options.connect
 def on_preload_parsed(options, **kwargs):
+    from rhodecode.config.middleware import get_celery_config
+
     ini_location = options['ini']
     ini_vars = options['ini_var']
     celery_app.conf['INI_PYRAMID'] = options['ini']
@@ -134,10 +136,11 b' def on_preload_parsed(options, **kwargs)'
     log.debug('Bootstrapping RhodeCode application...')
     env = bootstrap(ini_location, options=options)
 
+    celery_settings = get_celery_config(env['registry'].settings)
     setup_celery_app(
         app=env['app'], root=env['root'], request=env['request'],
         registry=env['registry'], closer=env['closer'],
-        ini_location=ini_location)
+        celery_settings=celery_settings)
 
     # fix the global flag even if it's disabled via .ini file because this
     # is a worker code that doesn't need this to be disabled.
@@ -196,17 +199,15 b' def task_revoked_signal('
         closer()
 
 
-def setup_celery_app(app, root, request, registry, closer, ini_location):
-    ini_dir = os.path.dirname(os.path.abspath(ini_location))
+def setup_celery_app(app, root, request, registry, closer, celery_settings):
+    log.debug('Got custom celery conf: %s', celery_settings)
     celery_config = base_celery_config
     celery_config.update({
         # store celerybeat scheduler db where the .ini file is
-        'beat_schedule_filename': os.path.join(ini_dir, 'celerybeat-schedule'),
+        'beat_schedule_filename': registry.settings['celerybeat-schedule.path'],
     })
-    ini_settings = get_ini_config(ini_location)
-    log.debug('Got custom celery conf: %s', ini_settings)
 
-    celery_config.update(ini_settings)
+    celery_config.update(celery_settings)
     celery_app.config_from_object(celery_config)
 
     celery_app.conf.update({'PYRAMID_APP': app})
@@ -216,7 +217,7 b' def setup_celery_app(app, root, request,'
     celery_app.conf.update({'PYRAMID_CLOSER': closer})
 
 
-def configure_celery(config, ini_location):
+def configure_celery(config, celery_settings):
     """
     Helper that is called from our application creation logic. It gives
     connection info into running webapp and allows execution of tasks from
@@ -226,10 +227,10 b' def configure_celery(config, ini_locatio'
     rhodecode.CELERY_ENABLED = str2bool(
         config.registry.settings.get('use_celery'))
     if rhodecode.CELERY_ENABLED:
-        log.info('Configuring celery based on `%s` file', ini_location)
+        log.info('Configuring celery based on `%s` settings', celery_settings)
         setup_celery_app(
             app=None, root=None, request=None, registry=config.registry,
-            closer=None, ini_location=ini_location)
+            closer=None, celery_settings=celery_settings)
 
 
 def maybe_prepare_env(req):
@@ -238,7 +239,7 b' def maybe_prepare_env(req):'
         environ.update({
             'PATH_INFO': req.environ['PATH_INFO'],
             'SCRIPT_NAME': req.environ['SCRIPT_NAME'],
-            'HTTP_HOST':req.environ.get('HTTP_HOST', req.environ['SERVER_NAME']),
+            'HTTP_HOST': req.environ.get('HTTP_HOST', req.environ['SERVER_NAME']),
             'SERVER_NAME': req.environ['SERVER_NAME'],
             'SERVER_PORT': req.environ['SERVER_PORT'],
             'wsgi.url_scheme': req.environ['wsgi.url_scheme'],

rhodecode/lib/celerylib/utils.py

@@ -115,52 +115,6 b' def get_beat_config(parser, section):'
     return config
 
 
-def get_ini_config(ini_location):
-    """
-    Converts basic ini configuration into celery 4.X options
-    """
-    def key_converter(key_name):
-        pref = 'celery.'
-        if key_name.startswith(pref):
-            return key_name[len(pref):].replace('.', '_').lower()
-
-    def type_converter(parsed_key, value):
-        # cast to int
-        if value.isdigit():
-            return int(value)
-
-        # cast to bool
-        if value.lower() in ['true', 'false', 'True', 'False']:
-            return value.lower() == 'true'
-        return value
-
-    parser = configparser.SafeConfigParser(
-        defaults={'here': os.path.abspath(ini_location)})
-    parser.read(ini_location)
-
-    ini_config = {}
-    for k, v in parser.items('app:main'):
-        pref = 'celery.'
-        if k.startswith(pref):
-            ini_config[key_converter(k)] = type_converter(key_converter(k), v)
-
-    beat_config = {}
-    for section in parser.sections():
-        if section.startswith('celerybeat:'):
-            name = section.split(':', 1)[1]
-            beat_config[name] = get_beat_config(parser, section)
-
-    # final compose of settings
-    celery_settings = {}
-
-    if ini_config:
-        celery_settings.update(ini_config)
-    if beat_config:
-        celery_settings.update({'beat_schedule': beat_config})
-
-    return celery_settings
-
-
 def parse_ini_vars(ini_vars):
     options = {}
     for pairs in ini_vars.split(','):

rhodecode/tests/config/test_sanitize_settings.py

@@ -22,9 +22,8 b''
 import pytest
 
 from rhodecode.tests import no_newline_id_generator
-from rhodecode.config.middleware import (
-    _sanitize_vcs_settings, _bool_setting, _string_setting, _list_setting,
-    _int_setting)
+from rhodecode.config.middleware import sanitize_settings_and_apply_defaults
+from rhodecode.config.settings_maker import SettingsMaker
 
 
 class TestHelperFunctions(object):
@@ -39,11 +38,9 b' class TestHelperFunctions(object):'
         ('invalid-∫øø@-√å@¨€', False),
         (u'invalid-∫øø@-√å@¨€', False),
     ])
-    def test_bool_setting_helper(self, raw, expected):
-        key = 'dummy-key'
-        settings = {key: raw}
-        _bool_setting(settings, key, None)
-        assert settings[key] is expected
+    def test_bool_func_helper(self, raw, expected):
+        val = SettingsMaker._bool_func(raw)
+        assert val == expected
 
     @pytest.mark.parametrize('raw, expected', [
         ('', ''),
@@ -52,11 +49,9 b' class TestHelperFunctions(object):'
         ('test-string-烩€', 'test-string-烩€'),
         (u'test-string-烩€', u'test-string-烩€'),
     ])
-    def test_string_setting_helper(self, raw, expected):
-        key = 'dummy-key'
-        settings = {key: raw}
-        _string_setting(settings, key, None)
-        assert settings[key] == expected
+    def test_string_func_helper(self, raw, expected):
+        val = SettingsMaker._string_func(raw)
+        assert val == expected
 
     @pytest.mark.parametrize('raw, expected', [
         ('', []),
@@ -64,19 +59,30 b' class TestHelperFunctions(object):'
         ('CaSe-TeSt', ['CaSe-TeSt']),
         ('test-string-烩€', ['test-string-烩€']),
         (u'test-string-烩€', [u'test-string-烩€']),
-        ('hg git svn', ['hg', 'git', 'svn']),
         ('hg,git,svn', ['hg', 'git', 'svn']),
         ('hg, git, svn', ['hg', 'git', 'svn']),
-        ('hg\ngit\nsvn', ['hg', 'git', 'svn']),
-        (' hg\n git\n svn ', ['hg', 'git', 'svn']),
+
         (', hg , git , svn , ', ['', 'hg', 'git', 'svn', '']),
         ('cheese,free node,other', ['cheese', 'free node', 'other']),
     ], ids=no_newline_id_generator)
     def test_list_setting_helper(self, raw, expected):
-        key = 'dummy-key'
-        settings = {key: raw}
-        _list_setting(settings, key, None)
-        assert settings[key] == expected
+        val = SettingsMaker._list_func(raw)
+        assert val == expected
+
+    @pytest.mark.parametrize('raw, expected', [
+        ('hg git svn', ['hg', 'git', 'svn']),
+    ], ids=no_newline_id_generator)
+    def test_list_setting_spaces_helper(self, raw, expected):
+        val = SettingsMaker._list_func(raw, sep=' ')
+        assert val == expected
+
+    @pytest.mark.parametrize('raw, expected', [
+        ('hg\ngit\nsvn', ['hg', 'git', 'svn']),
+        (' hg\n git\n svn ', ['hg', 'git', 'svn']),
+    ], ids=no_newline_id_generator)
+    def test_list_setting_newlines_helper(self, raw, expected):
+        val = SettingsMaker._list_func(raw, sep='\n')
+        assert val == expected
 
     @pytest.mark.parametrize('raw, expected', [
         ('0', 0),
@@ -86,10 +92,8 b' class TestHelperFunctions(object):'
         (u'-12345', -12345),
     ])
     def test_int_setting_helper(self, raw, expected):
-        key = 'dummy-key'
-        settings = {key: raw}
-        _int_setting(settings, key, None)
-        assert settings[key] == expected
+        val = SettingsMaker._int_func(raw)
+        assert val == expected
 
     @pytest.mark.parametrize('raw', [
         ('0xff'),
@@ -99,21 +103,19 b' class TestHelperFunctions(object):'
         (u'invalid-⁄~†'),
     ])
     def test_int_setting_helper_invalid_input(self, raw):
-        key = 'dummy-key'
-        settings = {key: raw}
         with pytest.raises(Exception):
-            _int_setting(settings, key, None)
+            SettingsMaker._int_func(raw)
 
 
 class TestSanitizeVcsSettings(object):
-    _bool_settings = [
+    _bool_funcs = [
         ('vcs.hooks.direct_calls', False),
         ('vcs.server.enable', True),
         ('vcs.start_server', False),
         ('startup.import_repos', False),
     ]
 
-    _string_settings = [
+    _string_funcs = [
         ('vcs.svn.compatible_version', ''),
         ('vcs.hooks.protocol', 'http'),
         ('vcs.hooks.host', '127.0.0.1'),
@@ -126,28 +128,28 b' class TestSanitizeVcsSettings(object):'
         ('vcs.backends', 'hg git'),
     ]
 
-    @pytest.mark.parametrize('key, default', _list_settings)
-    def test_list_setting_spacesep_list(self, key, default):
-        test_list = ['test', 'list', 'values', 'for', key]
-        input_value = ' '.join(test_list)
-        settings = {key: input_value}
-        _sanitize_vcs_settings(settings)
-        assert settings[key] == test_list
-
-    @pytest.mark.parametrize('key, default', _list_settings)
-    def test_list_setting_newlinesep_list(self, key, default):
-        test_list = ['test', 'list', 'values', 'for', key]
-        input_value = '\n'.join(test_list)
-        settings = {key: input_value}
-        _sanitize_vcs_settings(settings)
-        assert settings[key] == test_list
+    # @pytest.mark.parametrize('key, default', _list_settings)
+    # def test_list_setting_spacesep_list(self, key, default):
+    #     test_list = ['test', 'list', 'values', 'for', key]
+    #     input_value = ' '.join(test_list)
+    #     settings = {key: input_value}
+    #     sanitize_settings_and_apply_defaults({'__file__': ''}, settings)
+    #     assert settings[key] == test_list
+    #
+    # @pytest.mark.parametrize('key, default', _list_settings)
+    # def test_list_setting_newlinesep_list(self, key, default):
+    #     test_list = ['test', 'list', 'values', 'for', key]
+    #     input_value = '\n'.join(test_list)
+    #     settings = {key: input_value}
+    #     sanitize_settings_and_apply_defaults({'__file__': ''}, settings)
+    #     assert settings[key] == test_list
 
     @pytest.mark.parametrize('key, default', _list_settings)
     def test_list_setting_commasep_list(self, key, default):
         test_list = ['test', 'list', 'values', 'for', key]
         input_value = ','.join(test_list)
         settings = {key: input_value}
-        _sanitize_vcs_settings(settings)
+        sanitize_settings_and_apply_defaults({'__file__': ''}, settings)
         assert settings[key] == test_list
 
     @pytest.mark.parametrize('key, default', _list_settings)
@@ -155,49 +157,49 b' class TestSanitizeVcsSettings(object):'
         test_list = ['test', 'list', 'values', 'for', key]
         input_value = ', '.join(test_list)
         settings = {key: input_value}
-        _sanitize_vcs_settings(settings)
+        sanitize_settings_and_apply_defaults({'__file__': ''}, settings)
         assert settings[key] == test_list
 
-    @pytest.mark.parametrize('key, default', _string_settings)
-    def test_string_setting_string(self, key, default):
+    @pytest.mark.parametrize('key, default', _string_funcs)
+    def test_string_func_string(self, key, default):
         test_value = 'test-string-for-{}'.format(key)
         settings = {key: test_value}
-        _sanitize_vcs_settings(settings)
+        sanitize_settings_and_apply_defaults({'__file__': ''}, settings)
         assert settings[key] == test_value
 
-    @pytest.mark.parametrize('key, default', _string_settings)
-    def test_string_setting_default(self, key, default):
+    @pytest.mark.parametrize('key, default', _string_funcs)
+    def test_string_func_default(self, key, default):
         settings = {}
-        _sanitize_vcs_settings(settings)
+        sanitize_settings_and_apply_defaults({'__file__': ''}, settings)
         assert settings[key] == default
 
-    @pytest.mark.parametrize('key, default', _string_settings)
-    def test_string_setting_lowercase(self, key, default):
-        test_value = 'Test-String-For-{}'.format(key)
-        settings = {key: test_value}
-        _sanitize_vcs_settings(settings)
-        assert settings[key] == test_value.lower()
+    # @pytest.mark.parametrize('key, default', _string_funcs)
+    # def test_string_func_lowercase(self, key, default):
+    #     test_value = 'Test-String-For-{}'.format(key)
+    #     settings = {key: test_value}
+    #     sanitize_settings_and_apply_defaults({'__file__': ''}, settings)
+    #     assert settings[key] == test_value.lower()
 
-    @pytest.mark.parametrize('key, default', _bool_settings)
-    def test_bool_setting_true(self, key, default):
+    @pytest.mark.parametrize('key, default', _bool_funcs)
+    def test_bool_func_true(self, key, default):
         settings = {key: 'true'}
-        _sanitize_vcs_settings(settings)
+        sanitize_settings_and_apply_defaults({'__file__': ''}, settings)
         assert settings[key] is True
 
-    @pytest.mark.parametrize('key, default', _bool_settings)
-    def test_bool_setting_false(self, key, default):
+    @pytest.mark.parametrize('key, default', _bool_funcs)
+    def test_bool_func_false(self, key, default):
         settings = {key: 'false'}
-        _sanitize_vcs_settings(settings)
+        sanitize_settings_and_apply_defaults({'__file__': ''}, settings)
         assert settings[key] is False
 
-    @pytest.mark.parametrize('key, default', _bool_settings)
-    def test_bool_setting_invalid_string(self, key, default):
+    @pytest.mark.parametrize('key, default', _bool_funcs)
+    def test_bool_func_invalid_string(self, key, default):
         settings = {key: 'no-bool-val-string'}
-        _sanitize_vcs_settings(settings)
+        sanitize_settings_and_apply_defaults({'__file__': ''}, settings)
         assert settings[key] is False
 
-    @pytest.mark.parametrize('key, default', _bool_settings)
-    def test_bool_setting_default(self, key, default):
+    @pytest.mark.parametrize('key, default', _bool_funcs)
+    def test_bool_func_default(self, key, default):
         settings = {}
-        _sanitize_vcs_settings(settings)
+        sanitize_settings_and_apply_defaults({'__file__': ''}, settings)
         assert settings[key] is default

rhodecode/tests/rhodecode.ini

@@ -1,30 +1,23 b''
 
-
-################################################################################
-##                 RHODECODE COMMUNITY EDITION CONFIGURATION                  ##
-# The %(here)s variable will be replaced with the parent directory of this file#
-################################################################################
+; #########################################
+; RHODECODE COMMUNITY EDITION CONFIGURATION
+; #########################################
 
 [DEFAULT]
+; Debug flag sets all loggers to debug, and enables request tracking
 debug = true
 
-################################################################################
-##                            EMAIL CONFIGURATION                             ##
-## Uncomment and replace with the email address which should receive          ##
-## any error reports after an application crash                               ##
-## Additionally these settings will be used by the RhodeCode mailing system   ##
-################################################################################
+; ########################################################################
+; EMAIL CONFIGURATION
+; These settings will be used by the RhodeCode mailing system
+; ########################################################################
 
-## prefix all emails subjects with given prefix, helps filtering out emails
+; prefix all emails subjects with given prefix, helps filtering out emails
 #email_prefix = [RhodeCode]
 
-## email FROM address all mails will be sent
+; email FROM address all mails will be sent
 #app_email_from = rhodecode-noreply@localhost
 
-## Uncomment and replace with the address which should receive any error report
-## note: using appenlight for error handling doesn't need this to be uncommented
-#email_to = admin@localhost
-
 #smtp_server = mail.server.com
 #smtp_username =
 #smtp_password =
@@ -33,16 +26,20 b' debug = true'
 #smtp_use_ssl = true
 
 [server:main]
-## COMMON ##
+; COMMON HOST/IP CONFIG
 host = 0.0.0.0
 port = 5000
 
-##########################
-## GUNICORN WSGI SERVER ##
-##########################
-## run with gunicorn --log-config rhodecode.ini --paste rhodecode.ini
+
+; ###########################
+; GUNICORN APPLICATION SERVER
+; ###########################
 
+; run with gunicorn --log-config rhodecode.ini --paste rhodecode.ini
+
+; Module to use, this setting shouldn't be changed
 use = egg:gunicorn#main
+
 ## Sets the number of process workers. You must set `instance_id = *`
 ## when this option is set to more than one worker, recommended
 ## value is (2 * NUMBER_OF_CPUS + 1), eg 2CPU = 5 workers
@@ -81,7 +78,7 b' prefix = /'
 is_test = True
 use = egg:rhodecode-enterprise-ce
 
-## enable proxy prefix middleware, defined above
+; enable proxy prefix middleware, defined above
 #filter-with = proxy-prefix
 
 
@@ -99,64 +96,69 b' rhodecode.api.url = /_admin/api'
 ## `beaker.session.secret`
 #rhodecode.encrypted_values.secret =
 
-## decryption strict mode (enabled by default). It controls if decryption raises
-## `SignatureVerificationError` in case of wrong key, or damaged encryption data.
+; decryption strict mode (enabled by default). It controls if decryption raises
+; `SignatureVerificationError` in case of wrong key, or damaged encryption data.
 #rhodecode.encrypted_values.strict = false
 
-## return gzipped responses from Rhodecode (static files/application)
+; Pick algorithm for encryption. Either fernet (more secure) or aes (default)
+; fernet is safer, and we strongly recommend switching to it.
+; Due to backward compatibility aes is used as default.
+#rhodecode.encrypted_values.algorithm = fernet
+
+; Return gzipped responses from RhodeCode (static files/application)
 gzip_responses = false
 
-## autogenerate javascript routes file on startup
+; Auto-generate javascript routes file on startup
 generate_js_files = false
 
-## Optional Languages
-## en(default), be, de, es, fr, it, ja, pl, pt, ru, zh
+; System global default language.
+; All available languages: en (default), be, de, es, fr, it, ja, pl, pt, ru, zh
 lang = en
 
 ## perform a full repository scan on each server start, this should be
 ## set to false after first startup, to allow faster server restarts.
 startup.import_repos = true
 
-## Uncomment and set this path to use archive download cache.
-## Once enabled, generated archives will be cached at this location
-## and served from the cache during subsequent requests for the same archive of
-## the repository.
+; Uncomment and set this path to use archive download cache.
+; Once enabled, generated archives will be cached at this location
+; and served from the cache during subsequent requests for the same archive of
+; the repository.
 #archive_cache_dir = /tmp/tarballcache
 
-## URL at which the application is running. This is used for bootstraping
-## requests in context when no web request is available. Used in ishell, or
-## SSH calls. Set this for events to receive proper url for SSH calls.
+; URL at which the application is running. This is used for Bootstrapping
+; requests in context when no web request is available. Used in ishell, or
+; SSH calls. Set this for events to receive proper url for SSH calls.
 app.base_url = http://rhodecode.local
 
-## change this to unique ID for security
+; Unique application ID. Should be a random unique string for security.
 app_instance_uuid = rc-production
 
 ## cut off limit for large diffs (size in bytes)
 cut_off_limit_diff = 1024000
 cut_off_limit_file = 256000
 
-## use cache version of scm repo everywhere
+; Use cached version of vcs repositories everywhere. Recommended to be `true`
 vcs_full_cache = false
 
-## force https in RhodeCode, fixes https redirects, assumes it's always https
-## Normally this is controlled by proper http flags sent from http server
+; Force https in RhodeCode, fixes https redirects, assumes it's always https.
+; Normally this is controlled by proper flags sent from http server such as Nginx or Apache
 force_https = false
 
-## use Strict-Transport-Security headers
+; use Strict-Transport-Security headers
 use_htsts = false
 
-# Set to true if your repos are exposed using the dumb protocol
+; Set to true if your repos are exposed using the dumb protocol
 git_update_server_info = false
 
-## RSS/ATOM feed options
+; RSS/ATOM feed options
 rss_cut_off_limit = 256000
 rss_items_per_page = 10
 rss_include_diff = false
 
-## gist URL alias, used to create nicer urls for gist. This should be an
-## url that does rewrites to _admin/gists/{gistid}.
-## example: http://gist.rhodecode.org/{gistid}. Empty means use the internal
-## RhodeCode url, ie. http[s]://rhodecode.server/_admin/gists/{gistid}
+; gist URL alias, used to create nicer urls for gist. This should be an
+; url that does rewrites to _admin/gists/{gistid}.
+; example: http://gist.rhodecode.org/{gistid}. Empty means use the internal
+; RhodeCode url, ie. http[s]://rhodecode.server/_admin/gists/{gistid}
 gist_alias_url =
 
 ## List of views (using glob pattern syntax) that AUTH TOKENS could be
@@ -181,36 +183,39 b' gist_alias_url ='
 #    GistView:*
 api_access_controllers_whitelist =
 
-## default encoding used to convert from and to unicode
-## can be also a comma separated list of encoding in case of mixed encodings
+; Default encoding used to convert from and to unicode
+; can be also a comma separated list of encoding in case of mixed encodings
 default_encoding = UTF-8
 
-## instance-id prefix
-## a prefix key for this instance used for cache invalidation when running
-## multiple instances of rhodecode, make sure it's globally unique for
-## all running rhodecode instances. Leave empty if you don't use it
+; instance-id prefix
+; a prefix key for this instance used for cache invalidation when running
+; multiple instances of RhodeCode, make sure it's globally unique for
+; all running RhodeCode instances. Leave empty if you don't use it
 instance_id =
 
-## Fallback authentication plugin. Set this to a plugin ID to force the usage
-## of an authentication plugin also if it is disabled by it's settings.
-## This could be useful if you are unable to log in to the system due to broken
-## authentication settings. Then you can enable e.g. the internal rhodecode auth
-## module to log in again and fix the settings.
-##
-## Available builtin plugin IDs (hash is part of the ID):
-## egg:rhodecode-enterprise-ce#rhodecode
-## egg:rhodecode-enterprise-ce#pam
-## egg:rhodecode-enterprise-ce#ldap
-## egg:rhodecode-enterprise-ce#jasig_cas
-## egg:rhodecode-enterprise-ce#headers
-## egg:rhodecode-enterprise-ce#crowd
+; Fallback authentication plugin. Set this to a plugin ID to force the usage
+; of an authentication plugin also if it is disabled by it's settings.
+; This could be useful if you are unable to log in to the system due to broken
+; authentication settings. Then you can enable e.g. the internal RhodeCode auth
+; module to log in again and fix the settings.
+; Available builtin plugin IDs (hash is part of the ID):
+; egg:rhodecode-enterprise-ce#rhodecode
+; egg:rhodecode-enterprise-ce#pam
+; egg:rhodecode-enterprise-ce#ldap
+; egg:rhodecode-enterprise-ce#jasig_cas
+; egg:rhodecode-enterprise-ce#headers
+; egg:rhodecode-enterprise-ce#crowd
+
 #rhodecode.auth_plugin_fallback = egg:rhodecode-enterprise-ce#rhodecode
 
-## alternative return HTTP header for failed authentication. Default HTTP
-## response is 401 HTTPUnauthorized. Currently HG clients have troubles with
-## handling that causing a series of failed authentication calls.
-## Set this variable to 403 to return HTTPForbidden, or any other HTTP code
-## This will be served instead of default 401 on bad authnetication
+; Flag to control loading of legacy plugins in py:/path format
+auth_plugin.import_legacy_plugins = true
+
+; alternative return HTTP header for failed authentication. Default HTTP
+; response is 401 HTTPUnauthorized. Currently HG clients have troubles with
+; handling that causing a series of failed authentication calls.
+; Set this variable to 403 to return HTTPForbidden, or any other HTTP code
+; This will be served instead of default 401 on bad authentication
 auth_ret_code =
 
 ## use special detection method when serving auth_ret_code, instead of serving
@@ -240,38 +245,35 b' supervisor.group_id = dev'
 ## Display extended labs settings
 labs_settings_active = true
 
-####################################
-###        CELERY CONFIG        ####
-####################################
+; #############
+; CELERY CONFIG
+; #############
+
+; manually run celery: /path/to/celery worker -E --beat --app rhodecode.lib.celerylib.loader --scheduler rhodecode.lib.celerylib.scheduler.RcScheduler --loglevel DEBUG --ini /path/to/rhodecode.ini
+
 use_celery = false
-broker.host = localhost
-broker.vhost = rabbitmqhost
-broker.port = 5672
-broker.user = rabbitmq
-broker.password = qweqwe
 
-celery.imports = rhodecode.lib.celerylib.tasks
+; path to store schedule database
+#celerybeat-schedule.path =
 
-celery.result.backend = amqp
-celery.result.dburi = amqp://
-celery.result.serialier = json
+; connection url to the message broker (default redis)
+celery.broker_url = redis://localhost:6379/8
 
-#celery.send.task.error.emails = true
-#celery.amqp.task.result.expires = 18000
+; rabbitmq example
+#celery.broker_url = amqp://rabbitmq:qweqwe@localhost:5672/rabbitmqhost
 
-celeryd.concurrency = 2
-#celeryd.log.file = celeryd.log
-celeryd.log.level = debug
-celeryd.max.tasks.per.child = 1
+; maximum tasks to execute before worker restart
+celery.max_tasks_per_child = 100
 
-## tasks will never be sent to the queue, but executed locally instead.
-celery.always.eager = false
+; tasks will never be sent to the queue, but executed locally instead.
+celery.task_always_eager = false
 
-####################################
-###         BEAKER CACHE        ####
-####################################
-# default cache dir for templates.  Putting this into a ramdisk
-## can boost performance, eg. %(here)s/data_ramdisk
+; #############
+; DOGPILE CACHE
+; #############
+
+; Default cache dir for caches. Putting this into a ramdisk can boost performance.
+; eg. /tmpfs/data_ramdisk, however this directory might require large amount of space
 cache_dir = %(here)s/data
 
 ## locking and default file storage for Beaker. Putting this into a ramdisk
@@ -301,16 +303,21 b' rc_cache.sql_cache_short.backend = dogpi'
 rc_cache.sql_cache_short.expiration_time = 0
 
 
-####################################
-###       BEAKER SESSION        ####
-####################################
+; ##############
+; BEAKER SESSION
+; ##############
 
-## .session.type is type of storage options for the session, current allowed
-## types are file, ext:memcached, ext:database, and memory (default).
+; beaker.session.type is type of storage options for the logged users sessions. Current allowed
+; types are file, ext:redis, ext:database, ext:memcached, and memory (default if not specified).
+; Fastest ones are Redis and ext:database
 beaker.session.type = file
 beaker.session.data_dir = %(here)s/rc/data/sessions/data
 
-## db based session, fast, and allows easy management over logged in users
+; Redis based sessions
+#beaker.session.type = ext:redis
+#beaker.session.url = redis://127.0.0.1:6379/2
+
+; DB based session, fast, and allows easy management over logged in users
 #beaker.session.type = ext:database
 #beaker.session.table_name = db_session
 #beaker.session.sa.url = postgresql://postgres:secret@localhost/rhodecode
@@ -322,19 +329,20 b' beaker.session.key = rhodecode'
 beaker.session.secret = test-rc-uytcxaz
 beaker.session.lock_dir = %(here)s/rc/data/sessions/lock
 
-## Secure encrypted cookie. Requires AES and AES python libraries
-## you must disable beaker.session.secret to use this
+; Secure encrypted cookie. Requires AES and AES python libraries
+; you must disable beaker.session.secret to use this
 #beaker.session.encrypt_key = key_for_encryption
 #beaker.session.validate_key = validation_key
 
-## sets session as invalid(also logging out user) if it haven not been
-## accessed for given amount of time in seconds
+; Sets session as invalid (also logging out user) if it haven not been
+; accessed for given amount of time in seconds
 beaker.session.timeout = 2592000
 beaker.session.httponly = true
-## Path to use for the cookie. Set to prefix if you use prefix middleware
+
+; Path to use for the cookie. Set to prefix if you use prefix middleware
 #beaker.session.cookie_path = /custom_prefix
 
-## uncomment for https secure cookie
+; Set https secure cookie
 beaker.session.secure = false
 
 ## auto save the session to not to use .save()
@@ -344,242 +352,213 b' beaker.session.auto = false'
 ## at browser close
 #beaker.session.cookie_expires = 3600
 
-###################################
-## SEARCH INDEXING CONFIGURATION ##
-###################################
-## Full text search indexer is available in rhodecode-tools under
-## `rhodecode-tools index` command
+; #############################
+; SEARCH INDEXING CONFIGURATION
+; #############################
 
 ## WHOOSH Backend, doesn't require additional services to run
 ## it works good with few dozen repos
 search.module = rhodecode.lib.index.whoosh
 search.location = %(here)s/data/index
 
-########################################
-###    CHANNELSTREAM CONFIG         ####
-########################################
-## channelstream enables persistent connections and live notification
-## in the system. It's also used by the chat system
+; ####################
+; CHANNELSTREAM CONFIG
+; ####################
+
+; channelstream enables persistent connections and live notification
+; in the system. It's also used by the chat system
 
 channelstream.enabled = false
 
-## server address for channelstream server on the backend
+; server address for channelstream server on the backend
 channelstream.server = 127.0.0.1:9800
-## location of the channelstream server from outside world
-## use ws:// for http or wss:// for https. This address needs to be handled
-## by external HTTP server such as Nginx or Apache
-## see nginx/apache configuration examples in our docs
+
+; location of the channelstream server from outside world
+; use ws:// for http or wss:// for https. This address needs to be handled
+; by external HTTP server such as Nginx or Apache
+; see Nginx/Apache configuration examples in our docs
 channelstream.ws_url = ws://rhodecode.yourserver.com/_channelstream
 channelstream.secret = secret
 channelstream.history.location = %(here)s/channelstream_history
 
-## Internal application path that Javascript uses to connect into.
-## If you use proxy-prefix the prefix should be added before /_channelstream
+; Internal application path that Javascript uses to connect into.
+; If you use proxy-prefix the prefix should be added before /_channelstream
 channelstream.proxy_path = /_channelstream
 
 
-###################################
-##       APPENLIGHT CONFIG       ##
-###################################
-
-## Appenlight is tailored to work with RhodeCode, see
-## http://appenlight.com for details how to obtain an account
-
-## appenlight integration enabled
-appenlight = false
-
-appenlight.server_url = https://api.appenlight.com
-appenlight.api_key = YOUR_API_KEY
-#appenlight.transport_config = https://api.appenlight.com?threaded=1&timeout=5
-
-# used for JS client
-appenlight.api_public_key = YOUR_API_PUBLIC_KEY
-
-## TWEAK AMOUNT OF INFO SENT HERE
-
-## enables 404 error logging (default False)
-appenlight.report_404 = false
-
-## time in seconds after request is considered being slow (default 1)
-appenlight.slow_request_time = 1
-
-## record slow requests in application
-## (needs to be enabled for slow datastore recording and time tracking)
-appenlight.slow_requests = true
-
-## enable hooking to application loggers
-appenlight.logging = true
-
-## minimum log level for log capture
-appenlight.logging.level = WARNING
-
-## send logs only from erroneous/slow requests
-## (saves API quota for intensive logging)
-appenlight.logging_on_error = false
+; ##############################
+; MAIN RHODECODE DATABASE CONFIG
+; ##############################
 
-## list of additonal keywords that should be grabbed from environ object
-## can be string with comma separated list of words in lowercase
-## (by default client will always send following info:
-## 'REMOTE_USER', 'REMOTE_ADDR', 'SERVER_NAME', 'CONTENT_TYPE' + all keys that
-## start with HTTP* this list be extended with additional keywords here
-appenlight.environ_keys_whitelist =
-
-## list of keywords that should be blanked from request object
-## can be string with comma separated list of words in lowercase
-## (by default client will always blank keys that contain following words
-## 'password', 'passwd', 'pwd', 'auth_tkt', 'secret', 'csrf'
-## this list be extended with additional keywords set here
-appenlight.request_keys_blacklist =
-
-## list of namespaces that should be ignores when gathering log entries
-## can be string with comma separated list of namespaces
-## (by default the client ignores own entries: appenlight_client.client)
-appenlight.log_namespace_blacklist =
-
+#sqlalchemy.db1.url = sqlite:///%(here)s/rhodecode.db?timeout=30
+#sqlalchemy.db1.url = postgresql://postgres:qweqwe@localhost/rhodecode
+#sqlalchemy.db1.url = mysql://root:qweqwe@localhost/rhodecode?charset=utf8
+; pymysql is an alternative driver for MySQL, use in case of problems with default one
+#sqlalchemy.db1.url = mysql+pymysql://root:qweqwe@localhost/rhodecode
 
-################################################################################
-## WARNING: *THE LINE BELOW MUST BE UNCOMMENTED ON A PRODUCTION ENVIRONMENT*  ##
-## Debug mode will enable the interactive debugging tool, allowing ANYONE to  ##
-## execute malicious code after an exception is raised.                       ##
-################################################################################
-set debug = false
-
-
-##############
-## STYLING  ##
-##############
-debug_style = false
-
-###########################################
-###   MAIN RHODECODE DATABASE CONFIG    ###
-###########################################
-#sqlalchemy.db1.url = sqlite:///%(here)s/rhodecode_test.db?timeout=30
-#sqlalchemy.db1.url = postgresql://postgres:qweqwe@localhost/rhodecode_test
-#sqlalchemy.db1.url = mysql://root:qweqwe@localhost/rhodecode_test
 sqlalchemy.db1.url = sqlite:///%(here)s/rhodecode_test.db?timeout=30
 
-# see sqlalchemy docs for other advanced settings
-
-## print the sql statements to output
+; see sqlalchemy docs for other advanced settings
+; print the sql statements to output
 sqlalchemy.db1.echo = false
-## recycle the connections after this amount of seconds
-sqlalchemy.db1.pool_recycle = 3600
 
-## the number of connections to keep open inside the connection pool.
-## 0 indicates no limit
+; recycle the connections after this amount of seconds
+sqlalchemy.db1.pool_recycle = 3600
+sqlalchemy.db1.convert_unicode = true
+
+; the number of connections to keep open inside the connection pool.
+; 0 indicates no limit
 #sqlalchemy.db1.pool_size = 5
 
-## the number of connections to allow in connection pool "overflow", that is
-## connections that can be opened above and beyond the pool_size setting,
-## which defaults to five.
+; The number of connections to allow in connection pool "overflow", that is
+; connections that can be opened above and beyond the pool_size setting,
+; which defaults to five.
 #sqlalchemy.db1.max_overflow = 10
 
+; Connection check ping, used to detect broken database connections
+; could be enabled to better handle cases if MySQL has gone away errors
+#sqlalchemy.db1.ping_connection = true
 
-##################
-### VCS CONFIG ###
-##################
+; ##########
+; VCS CONFIG
+; ##########
 vcs.server.enable = true
 vcs.server = localhost:9901
 
-## Web server connectivity protocol, responsible for web based VCS operatations
-## Available protocols are:
-## `http` - use http-rpc backend (default)
+; Web server connectivity protocol, responsible for web based VCS operations
+; Available protocols are:
+; `http` - use http-rpc backend (default)
 vcs.server.protocol = http
 
-## Push/Pull operations protocol, available options are:
-## `http` - use http-rpc backend (default)
-## `vcsserver.scm_app` - internal app (EE only)
+; Push/Pull operations protocol, available options are:
+; `http` - use http-rpc backend (default)
 vcs.scm_app_implementation = http
 
-## Push/Pull operations hooks protocol, available options are:
-## `http` - use http-rpc backend (default)
+; Push/Pull operations hooks protocol, available options are:
+; `http` - use http-rpc backend (default)
 vcs.hooks.protocol = http
+
+; Host on which this instance is listening for hooks. If vcsserver is in other location
+; this should be adjusted.
 vcs.hooks.host = 127.0.0.1
 
-
-## Start VCSServer with this instance as a subprocess, Useful for development
+; Start VCSServer with this instance as a subprocess, useful for development
 vcs.start_server = false
 
-## List of enabled VCS backends, available options are:
-## `hg`  - mercurial
-## `git` - git
-## `svn` - subversion
+; List of enabled VCS backends, available options are:
+; `hg`  - mercurial
+; `git` - git
+; `svn` - subversion
 vcs.backends = hg, git, svn
 
+; Wait this number of seconds before killing connection to the vcsserver
 vcs.connection_timeout = 3600
-## Compatibility version when creating SVN repositories. Defaults to newest version when commented out.
-## Available options are: pre-1.4-compatible, pre-1.5-compatible, pre-1.6-compatible, pre-1.8-compatible, pre-1.9-compatible
-#vcs.svn.compatible_version = pre-1.8-compatible
 
+; Compatibility version when creating SVN repositories. Defaults to newest version when commented out.
+; Set a numeric version for your current SVN e.g 1.8, or 1.12
+; Legacy available options are: pre-1.4-compatible, pre-1.5-compatible, pre-1.6-compatible, pre-1.8-compatible, pre-1.9-compatible
+#vcs.svn.compatible_version = 1.8
 
-############################################################
-### Subversion proxy support (mod_dav_svn)               ###
-### Maps RhodeCode repo groups into SVN paths for Apache ###
-############################################################
-## Enable or disable the config file generation.
+; Cache flag to cache vcsserver remote calls locally
+; It uses cache_region `cache_repo`
+vcs.methods.cache = false
+
+; ####################################################
+; Subversion proxy support (mod_dav_svn)
+; Maps RhodeCode repo groups into SVN paths for Apache
+; ####################################################
+
+; Enable or disable the config file generation.
 svn.proxy.generate_config = false
-## Generate config file with `SVNListParentPath` set to `On`.
+
+; Generate config file with `SVNListParentPath` set to `On`.
 svn.proxy.list_parent_path = true
-## Set location and file name of generated config file.
+
+; Set location and file name of generated config file.
 svn.proxy.config_file_path = %(here)s/mod_dav_svn.conf
-## Used as a prefix to the `Location` block in the generated config file.
-## In most cases it should be set to `/`.
+
+; alternative mod_dav config template. This needs to be a valid mako template
+; Example template can be found in the source code:
+; rhodecode/apps/svn_support/templates/mod-dav-svn.conf.mako
+#svn.proxy.config_template = ~/.rccontrol/enterprise-1/custom_svn_conf.mako
+
+; Used as a prefix to the `Location` block in the generated config file.
+; In most cases it should be set to `/`.
 svn.proxy.location_root = /
-## Command to reload the mod dav svn configuration on change.
-## Example: `/etc/init.d/apache2 reload`
+
+; Command to reload the mod dav svn configuration on change.
+; Example: `/etc/init.d/apache2 reload` or /home/USER/apache_reload.sh
+; Make sure user who runs RhodeCode process is allowed to reload Apache
 #svn.proxy.reload_cmd = /etc/init.d/apache2 reload
-## If the timeout expires before the reload command finishes, the command will
-## be killed. Setting it to zero means no timeout. Defaults to 10 seconds.
+
+; If the timeout expires before the reload command finishes, the command will
+; be killed. Setting it to zero means no timeout. Defaults to 10 seconds.
 #svn.proxy.reload_timeout = 10
 
-############################################################
-### SSH Support Settings                                 ###
-############################################################
+; ####################
+; SSH Support Settings
+; ####################
 
-## Defines if the authorized_keys file should be written on any change of
-## user ssh keys, setting this to false also disables posibility of adding
-## ssh keys for users from web interface.
+; Defines if a custom authorized_keys file should be created and written on
+; any change user ssh keys. Setting this to false also disables possibility
+; of adding SSH keys by users from web interface. Super admins can still
+; manage SSH Keys.
 ssh.generate_authorized_keyfile = true
 
-## Options for ssh, default is `no-pty,no-port-forwarding,no-X11-forwarding,no-agent-forwarding`
+; Options for ssh, default is `no-pty,no-port-forwarding,no-X11-forwarding,no-agent-forwarding`
 # ssh.authorized_keys_ssh_opts =
 
-## File to generate the authorized keys together with options
-## It is possible to have multiple key files specified in `sshd_config` e.g.
-## AuthorizedKeysFile %h/.ssh/authorized_keys %h/.ssh/authorized_keys_rhodecode
+; Path to the authorized_keys file where the generate entries are placed.
+; It is possible to have multiple key files specified in `sshd_config` e.g.
+; AuthorizedKeysFile %h/.ssh/authorized_keys %h/.ssh/authorized_keys_rhodecode
 ssh.authorized_keys_file_path = %(here)s/rc/authorized_keys_rhodecode
 
-## Command to execute the SSH wrapper. The binary is available in the
-## rhodecode installation directory.
-## e.g ~/.rccontrol/community-1/profile/bin/rc-ssh-wrapper
+; Command to execute the SSH wrapper. The binary is available in the
+; RhodeCode installation directory.
+; e.g ~/.rccontrol/community-1/profile/bin/rc-ssh-wrapper
 ssh.wrapper_cmd = ~/.rccontrol/community-1/rc-ssh-wrapper
 
-## Allow shell when executing the ssh-wrapper command
+; Allow shell when executing the ssh-wrapper command
 ssh.wrapper_cmd_allow_shell = false
 
-## Enables logging, and detailed output send back to the client. Useful for
-## debugging, shouldn't be used in production.
+; Enables logging, and detailed output send back to the client during SSH
+; operations. Useful for debugging, shouldn't be used in production.
 ssh.enable_debug_logging = false
 
-## Paths to binary executrables, by default they are the names, but we can
-## override them if we want to use a custom one
+; Paths to binary executable, by default they are the names, but we can
+; override them if we want to use a custom one
 ssh.executable.hg = ~/.rccontrol/vcsserver-1/profile/bin/hg
 ssh.executable.git = ~/.rccontrol/vcsserver-1/profile/bin/git
 ssh.executable.svn = ~/.rccontrol/vcsserver-1/profile/bin/svnserve
 
-## Enables SSH key generator web interface. Disabling this still allows users
-## to add their own keys.
+; Enables SSH key generator web interface. Disabling this still allows users
+; to add their own keys.
 ssh.enable_ui_key_generator = true
 
+; Statsd client config, this is used to send metrics to statsd
+; We recommend setting statsd_exported and scrape them using Promethues
+#statsd.enabled = false
+#statsd.statsd_host = 0.0.0.0
+#statsd.statsd_port = 8125
+#statsd.statsd_prefix =
+#statsd.statsd_ipv6 = false
 
-## Dummy marker to add new entries after.
-## Add any custom entries below. Please don't remove.
+
+; configure logging automatically at server startup set to false
+; to use the below custom logging config.
+logging.autoconfigure = false
+
+; specify your own custom logging config file to configure logging
+#logging.logging_conf_file = /path/to/custom_logging.ini
+
+; Dummy marker to add new entries after.
+; Add any custom entries below. Please don't remove this marker.
 custom.conf = 1
 
 
-################################
-### LOGGING CONFIGURATION   ####
-################################
+; #####################
+; LOGGING CONFIGURATION
+; #####################
 [loggers]
 keys = root, sqlalchemy, beaker, rhodecode, ssh_wrapper
 
@@ -589,9 +568,9 b' keys = console, console_sql'
 [formatters]
 keys = generic, color_formatter, color_formatter_sql
 
-#############
-## LOGGERS ##
-#############
+; #######
+; LOGGERS
+; #######
 [logger_root]
 level = NOTSET
 handlers = console
@@ -603,6 +582,12 b' qualname = routes.middleware'
 ## "level = DEBUG" logs the route matched and routing variables.
 propagate = 1
 
+[logger_sqlalchemy]
+level = INFO
+handlers = console_sql
+qualname = sqlalchemy.engine
+propagate = 0
+
 [logger_beaker]
 level = DEBUG
 handlers =
@@ -615,50 +600,59 b' handlers ='
 qualname = rhodecode
 propagate = 1
 
-[logger_sqlalchemy]
-level = ERROR
-handlers = console_sql
-qualname = sqlalchemy.engine
-propagate = 0
-
 [logger_ssh_wrapper]
 level = DEBUG
 handlers =
 qualname = ssh_wrapper
 propagate = 1
 
+[logger_celery]
+level = DEBUG
+handlers =
+qualname = celery
 
-##############
-## HANDLERS ##
-##############
+
+; ########
+; HANDLERS
+; ########
 
 [handler_console]
 class = StreamHandler
-args = (sys.stderr,)
+args = (sys.stderr, )
 level = DEBUG
 formatter = generic
+; To enable JSON formatted logs replace generic with json
+; This allows sending properly formatted logs to grafana loki or elasticsearch
+#formatter = json
 
 [handler_console_sql]
+; "level = DEBUG" logs SQL queries and results.
+; "level = INFO" logs SQL queries.
+; "level = WARN" logs neither.  (Recommended for production systems.)
 class = StreamHandler
-args = (sys.stderr,)
+args = (sys.stderr, )
 level = WARN
 formatter = generic
 
-################
-## FORMATTERS ##
-################
+; ##########
+; FORMATTERS
+; ##########
 
 [formatter_generic]
 class = rhodecode.lib.logging_formatter.ExceptionAwareFormatter
-format = %(asctime)s.%(msecs)03d %(levelname)-5.5s [%(name)s] %(message)s
+format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s
 datefmt = %Y-%m-%d %H:%M:%S
 
 [formatter_color_formatter]
 class = rhodecode.lib.logging_formatter.ColorFormatter
-format = %(asctime)s.%(msecs)03d %(levelname)-5.5s [%(name)s] %(message)s
+format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s
 datefmt = %Y-%m-%d %H:%M:%S
 
 [formatter_color_formatter_sql]
 class = rhodecode.lib.logging_formatter.ColorFormatterSql
-format = %(asctime)s.%(msecs)03d %(levelname)-5.5s [%(name)s] %(message)s
+format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s
 datefmt = %Y-%m-%d %H:%M:%S
+
+[formatter_json]
+format = %(message)s
+class = rhodecode.lib._vendor.jsonlogger.JsonFormatter
  No newline at end of file