config: major update for the code to make it be almost fully controllable via env for new docker based installer.

configs/logging.ini

@@ -0,0 +1,98 b''
+; #####################
+; LOGGING CONFIGURATION
+; #####################
+[loggers]
+keys = root, sqlalchemy, beaker, celery, rhodecode, ssh_wrapper
+
+[handlers]
+keys = console, console_sql
+
+[formatters]
+keys = generic, json, color_formatter, color_formatter_sql
+
+; #######
+; LOGGERS
+; #######
+[logger_root]
+level = NOTSET
+handlers = console
+
+[logger_sqlalchemy]
+level = $RC_LOGGING_LEVEL
+handlers = console_sql
+qualname = sqlalchemy.engine
+propagate = 0
+
+[logger_beaker]
+level = $RC_LOGGING_LEVEL
+handlers =
+qualname = beaker.container
+propagate = 1
+
+[logger_rhodecode]
+level = $RC_LOGGING_LEVEL
+handlers =
+qualname = rhodecode
+propagate = 1
+
+[logger_ssh_wrapper]
+level = $RC_LOGGING_LEVEL
+handlers =
+qualname = ssh_wrapper
+propagate = 1
+
+[logger_celery]
+level = $RC_LOGGING_LEVEL
+handlers =
+qualname = celery
+
+
+; ########
+; HANDLERS
+; ########
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr, )
+level = $RC_LOGGING_LEVEL
+; To enable JSON formatted logs replace generic with json
+; This allows sending properly formatted logs to grafana loki or elasticsearch
+#formatter = json
+#formatter = generic
+formatter = $RC_LOGGING_FORMATTER
+
+[handler_console_sql]
+; "level = DEBUG" logs SQL queries and results.
+; "level = INFO" logs SQL queries.
+; "level = WARN" logs neither.  (Recommended for production systems.)
+class = StreamHandler
+args = (sys.stderr, )
+level = WARN
+; To enable JSON formatted logs replace generic with json
+; This allows sending properly formatted logs to grafana loki or elasticsearch
+#formatter = json
+#formatter = generic
+formatter = $RC_LOGGING_FORMATTER
+
+; ##########
+; FORMATTERS
+; ##########
+
+[formatter_generic]
+class = rhodecode.lib.logging_formatter.ExceptionAwareFormatter
+format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s
+datefmt = %Y-%m-%d %H:%M:%S
+
+[formatter_color_formatter]
+class = rhodecode.lib.logging_formatter.ColorFormatter
+format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s
+datefmt = %Y-%m-%d %H:%M:%S
+
+[formatter_color_formatter_sql]
+class = rhodecode.lib.logging_formatter.ColorFormatterSql
+format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s
+datefmt = %Y-%m-%d %H:%M:%S
+
+[formatter_json]
+format = %(message)s
+class = rhodecode.lib._vendor.jsonlogger.JsonFormatter
  No newline at end of file

rhodecode/config/settings_maker.py

@@ -0,0 +1,182 b''
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2010-2020 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+
+import os
+import textwrap
+import string
+import functools
+import logging
+import tempfile
+import logging.config
+log = logging.getLogger(__name__)
+
+
+def str2bool(_str):
+    """
+    returns True/False value from given string, it tries to translate the
+    string into boolean
+
+    :param _str: string value to translate into boolean
+    :rtype: boolean
+    :returns: boolean from given string
+    """
+    if _str is None:
+        return False
+    if _str in (True, False):
+        return _str
+    _str = str(_str).strip().lower()
+    return _str in ('t', 'true', 'y', 'yes', 'on', '1')
+
+
+def aslist(obj, sep=None, strip=True):
+    """
+    Returns given string separated by sep as list
+
+    :param obj:
+    :param sep:
+    :param strip:
+    """
+    if isinstance(obj, (basestring,)):
+        if obj in ['', ""]:
+            return []
+
+        lst = obj.split(sep)
+        if strip:
+            lst = [v.strip() for v in lst]
+        return lst
+    elif isinstance(obj, (list, tuple)):
+        return obj
+    elif obj is None:
+        return []
+    else:
+        return [obj]
+
+
+class SettingsMaker(object):
+
+    def __init__(self, app_settings):
+        self.settings = app_settings
+
+    @classmethod
+    def _bool_func(cls, input_val):
+        if isinstance(input_val, unicode):
+            input_val = input_val.encode('utf8')
+        return str2bool(input_val)
+
+    @classmethod
+    def _int_func(cls, input_val):
+        return int(input_val)
+
+    @classmethod
+    def _list_func(cls, input_val, sep=','):
+        return aslist(input_val, sep=sep)
+
+    @classmethod
+    def _string_func(cls, input_val, lower=True):
+        if lower:
+            input_val = input_val.lower()
+        return input_val
+
+    @classmethod
+    def _float_func(cls, input_val):
+        return float(input_val)
+
+    @classmethod
+    def _dir_func(cls, input_val, ensure_dir=False, mode=0o755):
+
+        # ensure we have our dir created
+        if not os.path.isdir(input_val) and ensure_dir:
+            os.makedirs(input_val, mode=mode)
+
+        if not os.path.isdir(input_val):
+            raise Exception('Dir at {} does not exist'.format(input_val))
+        return input_val
+
+    @classmethod
+    def _file_path_func(cls, input_val, ensure_dir=False, mode=0o755):
+        dirname = os.path.dirname(input_val)
+        cls._dir_func(dirname, ensure_dir=ensure_dir)
+        return input_val
+
+    @classmethod
+    def _key_transformator(cls, key):
+        return "{}_{}".format('RC'.upper(), key.upper().replace('.', '_').replace('-', '_'))
+
+    def enable_logging(self, logging_conf=None, level='INFO', formatter='generic'):
+        """
+        Helper to enable debug on running instance
+        :return:
+        """
+
+        if not str2bool(self.settings.get('logging.autoconfigure')):
+            log.info('logging configuration based on main .ini file')
+            return
+
+        if logging_conf is None:
+            logging_conf = self.settings.get('logging.logging_conf_file') or ''
+
+        if not os.path.isfile(logging_conf):
+            log.error('Unable to setup logging based on %s, file does not exist...', logging_conf)
+            return
+
+        with open(logging_conf, 'rb') as f:
+            ini_template = textwrap.dedent(f.read())
+            ini_template = string.Template(ini_template).safe_substitute(
+                RC_LOGGING_LEVEL=os.environ.get('RC_LOGGING_LEVEL', '') or level,
+                RC_LOGGING_FORMATTER=os.environ.get('RC_LOGGING_FORMATTER', '') or formatter
+            )
+
+        with tempfile.NamedTemporaryFile(prefix='rc_logging_', suffix='.ini', delete=False) as f:
+            log.info('Saved Temporary LOGGING config at %s', f.name)
+            f.write(ini_template)
+
+        logging.config.fileConfig(f.name)
+        os.remove(f.name)
+
+    def make_setting(self, key, default, lower=False, default_when_empty=False, parser=None):
+
+        input_val = self.settings.get(key, default)
+
+        if default_when_empty and not input_val:
+            # use default value when value is set in the config but it is empty
+            input_val = default
+
+        parser_func = {
+            'bool': self._bool_func,
+            'int': self._int_func,
+            'list': self._list_func,
+            'list:newline': functools.partial(self._list_func, sep='/n'),
+            'list:spacesep': functools.partial(self._list_func, sep=' '),
+            'string': functools.partial(self._string_func, lower=lower),
+            'dir': self._dir_func,
+            'dir:ensured': functools.partial(self._dir_func, ensure_dir=True),
+            'file': self._file_path_func,
+            'file:ensured': functools.partial(self._file_path_func, ensure_dir=True),
+            None: lambda i: i
+        }[parser]
+
+        # now maybe we have this KEY in env, search and use the value with higher priority.
+        transformed_key = self._key_transformator(key)
+        envvar_value = os.environ.get(transformed_key)
+        if envvar_value:
+            log.debug('using `%s` key instead of `%s` key for config', transformed_key, key)
+            input_val = envvar_value
+        self.settings[key] = parser_func(input_val)
+        return self.settings[key]

configs/gunicorn_config.py

@@ -1,269 +1,272 b''
 """
 Gunicorn config extension and hooks. This config file adds some extra settings and memory management.
 Gunicorn configuration should be managed by .ini files entries of RhodeCode or VCSServer
 """
 
 import gc
 import os
 import sys
 import math
 import time
 import threading
 import traceback
 import random
 from gunicorn.glogging import Logger
 
 
 def get_workers():
     import multiprocessing
     return multiprocessing.cpu_count() * 2 + 1
 
 # GLOBAL
 errorlog = '-'
 accesslog = '-'
 
 
 # SERVER MECHANICS
 # None == system temp dir
 # worker_tmp_dir is recommended to be set to some tmpfs
 worker_tmp_dir = None
 tmp_upload_dir = None
 
 # Custom log format
-access_log_format = (
-    '%(t)s %(p)s INFO  [GNCRN] %(h)-15s rqt:%(L)s %(s)s %(b)-6s "%(m)s:%(U)s %(q)s" usr:%(u)s "%(f)s" "%(a)s"')
+#access_log_format = (
+#    '%(t)s %(p)s INFO  [GNCRN] %(h)-15s rqt:%(L)s %(s)s %(b)-6s "%(m)s:%(U)s %(q)s" usr:%(u)s "%(f)s" "%(a)s"')
 
 # loki format for easier parsing in grafana
-#access_log_format = (
-#        'time="%(t)s" pid=%(p)s level="INFO" type="[GNCRN]" ip="%(h)-15s" rqt="%(L)s" response_code="%(s)s" response_bytes="%(b)-6s" uri="%(m)s:%(U)s %(q)s" user=":%(u)s" user_agent="%(a)s"')
+access_log_format = (
+    'time="%(t)s" pid=%(p)s level="INFO" type="[GNCRN]" ip="%(h)-15s" rqt="%(L)s" response_code="%(s)s" response_bytes="%(b)-6s" uri="%(m)s:%(U)s %(q)s" user=":%(u)s" user_agent="%(a)s"')
 
 # self adjust workers based on CPU count
 # workers = get_workers()
 
 
 def _get_process_rss(pid=None):
     try:
         import psutil
         if pid:
             proc = psutil.Process(pid)
         else:
             proc = psutil.Process()
         return proc.memory_info().rss
     except Exception:
         return None
 
 
 def _get_config(ini_path):
 
     try:
         import configparser
     except ImportError:
         import ConfigParser as configparser
     try:
         config = configparser.RawConfigParser()
         config.read(ini_path)
         return config
     except Exception:
         return None
 
 
 def _time_with_offset(memory_usage_check_interval):
     return time.time() - random.randint(0, memory_usage_check_interval/2.0)
 
 
 def pre_fork(server, worker):
     pass
 
 
 def post_fork(server, worker):
 
     # memory spec defaults
     _memory_max_usage = 0
     _memory_usage_check_interval = 60
     _memory_usage_recovery_threshold = 0.8
 
     ini_path = os.path.abspath(server.cfg.paste)
     conf = _get_config(ini_path)
 
     section = 'server:main'
     if conf and conf.has_section(section):
 
         if conf.has_option(section, 'memory_max_usage'):
             _memory_max_usage = conf.getint(section, 'memory_max_usage')
 
         if conf.has_option(section, 'memory_usage_check_interval'):
             _memory_usage_check_interval = conf.getint(section, 'memory_usage_check_interval')
 
         if conf.has_option(section, 'memory_usage_recovery_threshold'):
             _memory_usage_recovery_threshold = conf.getfloat(section, 'memory_usage_recovery_threshold')
 
-    worker._memory_max_usage = _memory_max_usage
-    worker._memory_usage_check_interval = _memory_usage_check_interval
-    worker._memory_usage_recovery_threshold = _memory_usage_recovery_threshold
+    worker._memory_max_usage = int(os.environ.get('RC_GUNICORN_MEMORY_MAX_USAGE', '')
+                                   or _memory_max_usage)
+    worker._memory_usage_check_interval = int(os.environ.get('RC_GUNICORN_MEMORY_USAGE_CHECK_INTERVAL', '')
+                                              or _memory_usage_check_interval)
+    worker._memory_usage_recovery_threshold = float(os.environ.get('RC_GUNICORN_MEMORY_USAGE_RECOVERY_THRESHOLD', '')
+                                                    or _memory_usage_recovery_threshold)
 
     # register memory last check time, with some random offset so we don't recycle all
     # at once
     worker._last_memory_check_time = _time_with_offset(_memory_usage_check_interval)
 
     if _memory_max_usage:
         server.log.info("[%-10s] WORKER spawned with max memory set at %s", worker.pid,
                         _format_data_size(_memory_max_usage))
     else:
         server.log.info("[%-10s] WORKER spawned", worker.pid)
 
 
 def pre_exec(server):
     server.log.info("Forked child, re-executing.")
 
 
 def on_starting(server):
     server_lbl = '{} {}'.format(server.proc_name, server.address)
     server.log.info("Server %s is starting.", server_lbl)
 
 
 def when_ready(server):
     server.log.info("Server %s is ready. Spawning workers", server)
 
 
 def on_reload(server):
     pass
 
 
 def _format_data_size(size, unit="B", precision=1, binary=True):
     """Format a number using SI units (kilo, mega, etc.).
 
     ``size``: The number as a float or int.
 
     ``unit``: The unit name in plural form. Examples: "bytes", "B".
 
     ``precision``: How many digits to the right of the decimal point. Default
     is 1.  0 suppresses the decimal point.
 
     ``binary``: If false, use base-10 decimal prefixes (kilo = K = 1000).
     If true, use base-2 binary prefixes (kibi = Ki = 1024).
 
     ``full_name``: If false (default), use the prefix abbreviation ("k" or
     "Ki").  If true, use the full prefix ("kilo" or "kibi"). If false,
     use abbreviation ("k" or "Ki").
 
     """
 
     if not binary:
         base = 1000
         multiples = ('', 'k', 'M', 'G', 'T', 'P', 'E', 'Z', 'Y')
     else:
         base = 1024
         multiples = ('', 'Ki', 'Mi', 'Gi', 'Ti', 'Pi', 'Ei', 'Zi', 'Yi')
 
     sign = ""
     if size > 0:
         m = int(math.log(size, base))
     elif size < 0:
         sign = "-"
         size = -size
         m = int(math.log(size, base))
     else:
         m = 0
     if m > 8:
         m = 8
 
     if m == 0:
         precision = '%.0f'
     else:
         precision = '%%.%df' % precision
 
     size = precision % (size / math.pow(base, m))
 
     return '%s%s %s%s' % (sign, size.strip(), multiples[m], unit)
 
 
 def _check_memory_usage(worker):
     memory_max_usage = worker._memory_max_usage
     if not memory_max_usage:
         return
 
     memory_usage_check_interval = worker._memory_usage_check_interval
     memory_usage_recovery_threshold = memory_max_usage * worker._memory_usage_recovery_threshold
 
     elapsed = time.time() - worker._last_memory_check_time
     if elapsed > memory_usage_check_interval:
         mem_usage = _get_process_rss()
         if mem_usage and mem_usage > memory_max_usage:
             worker.log.info(
                 "memory usage %s > %s, forcing gc",
                 _format_data_size(mem_usage), _format_data_size(memory_max_usage))
             # Try to clean it up by forcing a full collection.
             gc.collect()
             mem_usage = _get_process_rss()
             if mem_usage > memory_usage_recovery_threshold:
                 # Didn't clean up enough, we'll have to terminate.
                 worker.log.warning(
                     "memory usage %s > %s after gc, quitting",
                     _format_data_size(mem_usage), _format_data_size(memory_max_usage))
                 # This will cause worker to auto-restart itself
                 worker.alive = False
         worker._last_memory_check_time = time.time()
 
 
 def worker_int(worker):
     worker.log.info("[%-10s] worker received INT or QUIT signal", worker.pid)
 
     # get traceback info, on worker crash
     id2name = dict([(th.ident, th.name) for th in threading.enumerate()])
     code = []
     for thread_id, stack in sys._current_frames().items():
         code.append(
             "\n# Thread: %s(%d)" % (id2name.get(thread_id, ""), thread_id))
         for fname, lineno, name, line in traceback.extract_stack(stack):
             code.append('File: "%s", line %d, in %s' % (fname, lineno, name))
             if line:
                 code.append("  %s" % (line.strip()))
     worker.log.debug("\n".join(code))
 
 
 def worker_abort(worker):
     worker.log.info("[%-10s] worker received SIGABRT signal", worker.pid)
 
 
 def worker_exit(server, worker):
     worker.log.info("[%-10s] worker exit", worker.pid)
 
 
 def child_exit(server, worker):
     worker.log.info("[%-10s] worker child exit", worker.pid)
 
 
 def pre_request(worker, req):
     worker.start_time = time.time()
     worker.log.debug(
         "GNCRN PRE  WORKER [cnt:%s]: %s %s", worker.nr, req.method, req.path)
 
 
 def post_request(worker, req, environ, resp):
     total_time = time.time() - worker.start_time
     # Gunicorn sometimes has problems with reading the status_code
     status_code = getattr(resp, 'status_code', '')
     worker.log.debug(
         "GNCRN POST WORKER [cnt:%s]: %s %s resp: %s, Load Time: %.4fs",
         worker.nr, req.method, req.path, status_code, total_time)
     _check_memory_usage(worker)
 
 
 class RhodeCodeLogger(Logger):
     """
     Custom Logger that allows some customization that gunicorn doesn't allow
     """
 
     datefmt = r"%Y-%m-%d %H:%M:%S"
 
     def __init__(self, cfg):
         Logger.__init__(self, cfg)
 
     def now(self):
         """ return date in RhodeCode Log format """
         now = time.time()
         msecs = int((now - long(now)) * 1000)
         return time.strftime(self.datefmt, time.localtime(now)) + '.{0:03d}'.format(msecs)
 
 
 logger_class = RhodeCodeLogger

configs/production.ini

@@ -1,816 +1,825 b''
 ## -*- coding: utf-8 -*-
 
 ; #########################################
 ; RHODECODE COMMUNITY EDITION CONFIGURATION
 ; #########################################
 
 [DEFAULT]
 ; Debug flag sets all loggers to debug, and enables request tracking
 debug = false
 
 ; ########################################################################
 ; EMAIL CONFIGURATION
 ; These settings will be used by the RhodeCode mailing system
 ; ########################################################################
 
 ; prefix all emails subjects with given prefix, helps filtering out emails
 #email_prefix = [RhodeCode]
 
 ; email FROM address all mails will be sent
 #app_email_from = rhodecode-noreply@localhost
 
 #smtp_server = mail.server.com
 #smtp_username =
 #smtp_password =
 #smtp_port =
 #smtp_use_tls = false
 #smtp_use_ssl = true
 
 [server:main]
 ; COMMON HOST/IP CONFIG
 host = 127.0.0.1
 port = 5000
 
 
 ; ###########################
 ; GUNICORN APPLICATION SERVER
 ; ###########################
 
 ; run with gunicorn --log-config rhodecode.ini --paste rhodecode.ini
 
 ; Module to use, this setting shouldn't be changed
 use = egg:gunicorn#main
 
 ; Sets the number of process workers. More workers means more concurrent connections
 ; RhodeCode can handle at the same time. Each additional worker also it increases
 ; memory usage as each has it's own set of caches.
 ; Recommended value is (2 * NUMBER_OF_CPUS + 1), eg 2CPU = 5 workers, but no more
 ; than 8-10 unless for really big deployments .e.g 700-1000 users.
 ; `instance_id = *` must be set in the [app:main] section below (which is the default)
 ; when using more than 1 worker.
 workers = 2
 
 ; Gunicorn access log level
 loglevel = info
 
 ; Process name visible in process list
 proc_name = rhodecode
 
 ; Type of worker class, one of `sync`, `gevent`
 ; Recommended type is `gevent`
 worker_class = gevent
 
 ; The maximum number of simultaneous clients per worker. Valid only for gevent
 worker_connections = 10
 
 ; Max number of requests that worker will handle before being gracefully restarted.
 ; Prevents memory leaks, jitter adds variability so not all workers are restarted at once.
 max_requests = 1000
 max_requests_jitter = 30
 
 ; Amount of time a worker can spend with handling a request before it
 ; gets killed and restarted. By default set to 21600 (6hrs)
 ; Examples: 1800 (30min), 3600 (1hr), 7200 (2hr), 43200 (12h)
 timeout = 21600
 
 ; The maximum size of HTTP request line in bytes.
 ; 0 for unlimited
 limit_request_line = 0
 
 ; Limit the number of HTTP headers fields in a request.
 ; By default this value is 100 and can't be larger than 32768.
 limit_request_fields = 32768
 
 ; Limit the allowed size of an HTTP request header field.
 ; Value is a positive number or 0.
 ; Setting it to 0 will allow unlimited header field sizes.
 limit_request_field_size = 0
 
 ; Timeout for graceful workers restart.
 ; After receiving a restart signal, workers have this much time to finish
 ; serving requests. Workers still alive after the timeout (starting from the
 ; receipt of the restart signal) are force killed.
 ; Examples: 1800 (30min), 3600 (1hr), 7200 (2hr), 43200 (12h)
 graceful_timeout = 3600
 
 # The number of seconds to wait for requests on a Keep-Alive connection.
 # Generally set in the 1-5 seconds range.
 keepalive = 2
 
 ; Maximum memory usage that each worker can use before it will receive a
 ; graceful restart signal 0 = memory monitoring is disabled
 ; Examples: 268435456 (256MB), 536870912 (512MB)
 ; 1073741824 (1GB), 2147483648 (2GB), 4294967296 (4GB)
 memory_max_usage = 0
 
 ; How often in seconds to check for memory usage for each gunicorn worker
 memory_usage_check_interval = 60
 
 ; Threshold value for which we don't recycle worker if GarbageCollection
 ; frees up enough resources. Before each restart we try to run GC on worker
 ; in case we get enough free memory after that, restart will not happen.
 memory_usage_recovery_threshold = 0.8
 
 
 ; Prefix middleware for RhodeCode.
 ; recommended when using proxy setup.
 ; allows to set RhodeCode under a prefix in server.
 ; eg https://server.com/custom_prefix. Enable `filter-with =` option below as well.
 ; And set your prefix like: `prefix = /custom_prefix`
 ; be sure to also set beaker.session.cookie_path = /custom_prefix if you need
 ; to make your cookies only work on prefix url
 [filter:proxy-prefix]
 use = egg:PasteDeploy#prefix
 prefix = /
 
 [app:main]
-; The %(here)s variable will be replaced with the absolute path of parent directory
-; of this file
-; In addition ENVIRONMENT variables usage is possible, e.g
-; sqlalchemy.db1.url = {ENV_RC_DB_URL}
 
 use = egg:rhodecode-enterprise-ce
 
 ; enable proxy prefix middleware, defined above
 #filter-with = proxy-prefix
 
 ; encryption key used to encrypt social plugin tokens,
 ; remote_urls with credentials etc, if not set it defaults to
 ; `beaker.session.secret`
 #rhodecode.encrypted_values.secret =
 
 ; decryption strict mode (enabled by default). It controls if decryption raises
 ; `SignatureVerificationError` in case of wrong key, or damaged encryption data.
 #rhodecode.encrypted_values.strict = false
 
 ; Pick algorithm for encryption. Either fernet (more secure) or aes (default)
 ; fernet is safer, and we strongly recommend switching to it.
 ; Due to backward compatibility aes is used as default.
 #rhodecode.encrypted_values.algorithm = fernet
 
 ; Return gzipped responses from RhodeCode (static files/application)
 gzip_responses = false
 
 ; Auto-generate javascript routes file on startup
 generate_js_files = false
 
 ; System global default language.
 ; All available languages: en (default), be, de, es, fr, it, ja, pl, pt, ru, zh
 lang = en
 
 ; Perform a full repository scan and import on each server start.
 ; Settings this to true could lead to very long startup time.
 startup.import_repos = false
 
 ; Uncomment and set this path to use archive download cache.
 ; Once enabled, generated archives will be cached at this location
 ; and served from the cache during subsequent requests for the same archive of
 ; the repository.
 #archive_cache_dir = /tmp/tarballcache
 
 ; URL at which the application is running. This is used for Bootstrapping
 ; requests in context when no web request is available. Used in ishell, or
 ; SSH calls. Set this for events to receive proper url for SSH calls.
 app.base_url = http://rhodecode.local
 
 ; Unique application ID. Should be a random unique string for security.
 app_instance_uuid = rc-production
 
 ; Cut off limit for large diffs (size in bytes). If overall diff size on
 ; commit, or pull request exceeds this limit this diff will be displayed
 ; partially. E.g 512000 == 512Kb
 cut_off_limit_diff = 512000
 
 ; Cut off limit for large files inside diffs (size in bytes). Each individual
 ; file inside diff which exceeds this limit will be displayed partially.
 ;  E.g 128000 == 128Kb
 cut_off_limit_file = 128000
 
 ; Use cached version of vcs repositories everywhere. Recommended to be `true`
 vcs_full_cache = true
 
 ; Force https in RhodeCode, fixes https redirects, assumes it's always https.
 ; Normally this is controlled by proper flags sent from http server such as Nginx or Apache
 force_https = false
 
 ; use Strict-Transport-Security headers
 use_htsts = false
 
 ; Set to true if your repos are exposed using the dumb protocol
 git_update_server_info = false
 
 ; RSS/ATOM feed options
 rss_cut_off_limit = 256000
 rss_items_per_page = 10
 rss_include_diff = false
 
 ; gist URL alias, used to create nicer urls for gist. This should be an
 ; url that does rewrites to _admin/gists/{gistid}.
 ; example: http://gist.rhodecode.org/{gistid}. Empty means use the internal
 ; RhodeCode url, ie. http[s]://rhodecode.server/_admin/gists/{gistid}
 gist_alias_url =
 
 ; List of views (using glob pattern syntax) that AUTH TOKENS could be
 ; used for access.
 ; Adding ?auth_token=TOKEN_HASH to the url authenticates this request as if it
 ; came from the the logged in user who own this authentication token.
 ; Additionally @TOKEN syntax can be used to bound the view to specific
 ; authentication token. Such view would be only accessible when used together
 ; with this authentication token
 ; list of all views can be found under `/_admin/permissions/auth_token_access`
 ; The list should be "," separated and on a single line.
 ; Most common views to enable:
 
 #    RepoCommitsView:repo_commit_download
 #    RepoCommitsView:repo_commit_patch
 #    RepoCommitsView:repo_commit_raw
 #    RepoCommitsView:repo_commit_raw@TOKEN
 #    RepoFilesView:repo_files_diff
 #    RepoFilesView:repo_archivefile
 #    RepoFilesView:repo_file_raw
 #    GistView:*
 api_access_controllers_whitelist =
 
 ; Default encoding used to convert from and to unicode
 ; can be also a comma separated list of encoding in case of mixed encodings
 default_encoding = UTF-8
 
 ; instance-id prefix
 ; a prefix key for this instance used for cache invalidation when running
 ; multiple instances of RhodeCode, make sure it's globally unique for
 ; all running RhodeCode instances. Leave empty if you don't use it
 instance_id =
 
 ; Fallback authentication plugin. Set this to a plugin ID to force the usage
 ; of an authentication plugin also if it is disabled by it's settings.
 ; This could be useful if you are unable to log in to the system due to broken
 ; authentication settings. Then you can enable e.g. the internal RhodeCode auth
 ; module to log in again and fix the settings.
 ; Available builtin plugin IDs (hash is part of the ID):
 ; egg:rhodecode-enterprise-ce#rhodecode
 ; egg:rhodecode-enterprise-ce#pam
 ; egg:rhodecode-enterprise-ce#ldap
 ; egg:rhodecode-enterprise-ce#jasig_cas
 ; egg:rhodecode-enterprise-ce#headers
 ; egg:rhodecode-enterprise-ce#crowd
 
 #rhodecode.auth_plugin_fallback = egg:rhodecode-enterprise-ce#rhodecode
 
 ; Flag to control loading of legacy plugins in py:/path format
 auth_plugin.import_legacy_plugins = true
 
 ; alternative return HTTP header for failed authentication. Default HTTP
 ; response is 401 HTTPUnauthorized. Currently HG clients have troubles with
 ; handling that causing a series of failed authentication calls.
 ; Set this variable to 403 to return HTTPForbidden, or any other HTTP code
 ; This will be served instead of default 401 on bad authentication
 auth_ret_code =
 
 ; use special detection method when serving auth_ret_code, instead of serving
 ; ret_code directly, use 401 initially (Which triggers credentials prompt)
 ; and then serve auth_ret_code to clients
 auth_ret_code_detection = false
 
 ; locking return code. When repository is locked return this HTTP code. 2XX
 ; codes don't break the transactions while 4XX codes do
 lock_ret_code = 423
 
 ; allows to change the repository location in settings page
 allow_repo_location_change = true
 
 ; allows to setup custom hooks in settings page
 allow_custom_hooks_settings = true
 
 ; Generated license token required for EE edition license.
 ; New generated token value can be found in Admin > settings > license page.
 license_token =
 
 ; This flag hides sensitive information on the license page such as token, and license data
 license.hide_license_info = false
 
 ; supervisor connection uri, for managing supervisor and logs.
 supervisor.uri =
 
 ; supervisord group name/id we only want this RC instance to handle
 supervisor.group_id = prod
 
 ; Display extended labs settings
 labs_settings_active = true
 
 ; Custom exception store path, defaults to TMPDIR
 ; This is used to store exception from RhodeCode in shared directory
 #exception_tracker.store_path =
 
 ; Send email with exception details when it happens
 #exception_tracker.send_email = false
 
 ; Comma separated list of recipients for exception emails,
 ; e.g [e-mail],[e-mail]
 ; Can be left empty, then emails will be sent to ALL super-admins
 #exception_tracker.send_email_recipients =
 
 ; optional prefix to Add to email Subject
 #exception_tracker.email_prefix = [RHODECODE ERROR]
 
 ; File store configuration. This is used to store and serve uploaded files
 file_store.enabled = true
 
 ; Storage backend, available options are: local
 file_store.backend = local
 
 ; path to store the uploaded binaries
 file_store.storage_path = %(here)s/data/file_store
 
 
 ; #############
 ; CELERY CONFIG
 ; #############
 
 ; manually run celery: /path/to/celery worker -E --beat --app rhodecode.lib.celerylib.loader --scheduler rhodecode.lib.celerylib.scheduler.RcScheduler --loglevel DEBUG --ini /path/to/rhodecode.ini
 
 use_celery = false
 
+; path to store schedule database
+#celerybeat-schedule.path =
+
 ; connection url to the message broker (default redis)
 celery.broker_url = redis://localhost:6379/8
 
 ; rabbitmq example
 #celery.broker_url = amqp://rabbitmq:qweqwe@localhost:5672/rabbitmqhost
 
 ; maximum tasks to execute before worker restart
 celery.max_tasks_per_child = 100
 
 ; tasks will never be sent to the queue, but executed locally instead.
 celery.task_always_eager = false
 
 ; #############
 ; DOGPILE CACHE
 ; #############
 
 ; Default cache dir for caches. Putting this into a ramdisk can boost performance.
 ; eg. /tmpfs/data_ramdisk, however this directory might require large amount of space
 cache_dir = %(here)s/data
 
 ; *********************************************
 ; `sql_cache_short` cache for heavy SQL queries
 ; Only supported backend is `memory_lru`
 ; *********************************************
 rc_cache.sql_cache_short.backend = dogpile.cache.rc.memory_lru
 rc_cache.sql_cache_short.expiration_time = 30
 
 
 ; *****************************************************
 ; `cache_repo_longterm` cache for repo object instances
 ; Only supported backend is `memory_lru`
 ; *****************************************************
 rc_cache.cache_repo_longterm.backend = dogpile.cache.rc.memory_lru
 ; by default we use 30 Days, cache is still invalidated on push
 rc_cache.cache_repo_longterm.expiration_time = 2592000
 ; max items in LRU cache, set to smaller number to save memory, and expire last used caches
 rc_cache.cache_repo_longterm.max_size = 10000
 
 
 ; *************************************************
 ; `cache_perms` cache for permission tree, auth TTL
 ; *************************************************
 rc_cache.cache_perms.backend = dogpile.cache.rc.file_namespace
 rc_cache.cache_perms.expiration_time = 300
 ; file cache store path. Defaults to `cache_dir =` value or tempdir if both values are not set
 #rc_cache.cache_perms.arguments.filename = /tmp/cache_perms.db
 
 ; alternative `cache_perms` redis backend with distributed lock
 #rc_cache.cache_perms.backend = dogpile.cache.rc.redis
 #rc_cache.cache_perms.expiration_time = 300
 
 ; redis_expiration_time needs to be greater then expiration_time
 #rc_cache.cache_perms.arguments.redis_expiration_time = 7200
 
 #rc_cache.cache_perms.arguments.host = localhost
 #rc_cache.cache_perms.arguments.port = 6379
 #rc_cache.cache_perms.arguments.db = 0
 #rc_cache.cache_perms.arguments.socket_timeout = 30
 ; more Redis options: https://dogpilecache.sqlalchemy.org/en/latest/api.html#redis-backends
 #rc_cache.cache_perms.arguments.distributed_lock = true
 
 ; auto-renew lock to prevent stale locks, slower but safer. Use only if problems happen
 #rc_cache.cache_perms.arguments.lock_auto_renewal = true
 
 ; ***************************************************
 ; `cache_repo` cache for file tree, Readme, RSS FEEDS
 ; ***************************************************
 rc_cache.cache_repo.backend = dogpile.cache.rc.file_namespace
 rc_cache.cache_repo.expiration_time = 2592000
 ; file cache store path. Defaults to `cache_dir =` value or tempdir if both values are not set
 #rc_cache.cache_repo.arguments.filename = /tmp/cache_repo.db
 
 ; alternative `cache_repo` redis backend with distributed lock
 #rc_cache.cache_repo.backend = dogpile.cache.rc.redis
 #rc_cache.cache_repo.expiration_time = 2592000
 
 ; redis_expiration_time needs to be greater then expiration_time
 #rc_cache.cache_repo.arguments.redis_expiration_time = 2678400
 
 #rc_cache.cache_repo.arguments.host = localhost
 #rc_cache.cache_repo.arguments.port = 6379
 #rc_cache.cache_repo.arguments.db = 1
 #rc_cache.cache_repo.arguments.socket_timeout = 30
 ; more Redis options: https://dogpilecache.sqlalchemy.org/en/latest/api.html#redis-backends
 #rc_cache.cache_repo.arguments.distributed_lock = true
 
 ; auto-renew lock to prevent stale locks, slower but safer. Use only if problems happen
 #rc_cache.cache_repo.arguments.lock_auto_renewal = true
 
 ; ##############
 ; BEAKER SESSION
 ; ##############
 
 ; beaker.session.type is type of storage options for the logged users sessions. Current allowed
 ; types are file, ext:redis, ext:database, ext:memcached, and memory (default if not specified).
 ; Fastest ones are Redis and ext:database
 beaker.session.type = file
 beaker.session.data_dir = %(here)s/data/sessions
 
 ; Redis based sessions
 #beaker.session.type = ext:redis
 #beaker.session.url = redis://127.0.0.1:6379/2
 
 ; DB based session, fast, and allows easy management over logged in users
 #beaker.session.type = ext:database
 #beaker.session.table_name = db_session
 #beaker.session.sa.url = postgresql://postgres:secret@localhost/rhodecode
 #beaker.session.sa.url = mysql://root:secret@127.0.0.1/rhodecode
 #beaker.session.sa.pool_recycle = 3600
 #beaker.session.sa.echo = false
 
 beaker.session.key = rhodecode
 beaker.session.secret = production-rc-uytcxaz
 beaker.session.lock_dir = %(here)s/data/sessions/lock
 
 ; Secure encrypted cookie. Requires AES and AES python libraries
 ; you must disable beaker.session.secret to use this
 #beaker.session.encrypt_key = key_for_encryption
 #beaker.session.validate_key = validation_key
 
 ; Sets session as invalid (also logging out user) if it haven not been
 ; accessed for given amount of time in seconds
 beaker.session.timeout = 2592000
 beaker.session.httponly = true
 
 ; Path to use for the cookie. Set to prefix if you use prefix middleware
 #beaker.session.cookie_path = /custom_prefix
 
 ; Set https secure cookie
 beaker.session.secure = false
 
 ; default cookie expiration time in seconds, set to `true` to set expire
 ; at browser close
 #beaker.session.cookie_expires = 3600
 
 ; #############################
 ; SEARCH INDEXING CONFIGURATION
 ; #############################
 
 ; Full text search indexer is available in rhodecode-tools under
 ; `rhodecode-tools index` command
 
 ; WHOOSH Backend, doesn't require additional services to run
 ; it works good with few dozen repos
 search.module = rhodecode.lib.index.whoosh
 search.location = %(here)s/data/index
 
 ; ####################
 ; CHANNELSTREAM CONFIG
 ; ####################
 
 ; channelstream enables persistent connections and live notification
 ; in the system. It's also used by the chat system
 
 channelstream.enabled = false
 
 ; server address for channelstream server on the backend
 channelstream.server = 127.0.0.1:9800
 
 ; location of the channelstream server from outside world
 ; use ws:// for http or wss:// for https. This address needs to be handled
 ; by external HTTP server such as Nginx or Apache
 ; see Nginx/Apache configuration examples in our docs
 channelstream.ws_url = ws://rhodecode.yourserver.com/_channelstream
 channelstream.secret = secret
 channelstream.history.location = %(here)s/channelstream_history
 
 ; Internal application path that Javascript uses to connect into.
 ; If you use proxy-prefix the prefix should be added before /_channelstream
 channelstream.proxy_path = /_channelstream
 
 
 ; ##############################
 ; MAIN RHODECODE DATABASE CONFIG
 ; ##############################
 
 #sqlalchemy.db1.url = sqlite:///%(here)s/rhodecode.db?timeout=30
 #sqlalchemy.db1.url = postgresql://postgres:qweqwe@localhost/rhodecode
 #sqlalchemy.db1.url = mysql://root:qweqwe@localhost/rhodecode?charset=utf8
 ; pymysql is an alternative driver for MySQL, use in case of problems with default one
 #sqlalchemy.db1.url = mysql+pymysql://root:qweqwe@localhost/rhodecode
 
 sqlalchemy.db1.url = postgresql://postgres:qweqwe@localhost/rhodecode
 
 ; see sqlalchemy docs for other advanced settings
 ; print the sql statements to output
 sqlalchemy.db1.echo = false
 
 ; recycle the connections after this amount of seconds
 sqlalchemy.db1.pool_recycle = 3600
 sqlalchemy.db1.convert_unicode = true
 
 ; the number of connections to keep open inside the connection pool.
 ; 0 indicates no limit
 #sqlalchemy.db1.pool_size = 5
 
 ; The number of connections to allow in connection pool "overflow", that is
 ; connections that can be opened above and beyond the pool_size setting,
 ; which defaults to five.
 #sqlalchemy.db1.max_overflow = 10
 
 ; Connection check ping, used to detect broken database connections
 ; could be enabled to better handle cases if MySQL has gone away errors
 #sqlalchemy.db1.ping_connection = true
 
 ; ##########
 ; VCS CONFIG
 ; ##########
 vcs.server.enable = true
 vcs.server = localhost:9900
 
 ; Web server connectivity protocol, responsible for web based VCS operations
 ; Available protocols are:
 ; `http` - use http-rpc backend (default)
 vcs.server.protocol = http
 
 ; Push/Pull operations protocol, available options are:
 ; `http` - use http-rpc backend (default)
 vcs.scm_app_implementation = http
 
 ; Push/Pull operations hooks protocol, available options are:
 ; `http` - use http-rpc backend (default)
 vcs.hooks.protocol = http
 
 ; Host on which this instance is listening for hooks. If vcsserver is in other location
 ; this should be adjusted.
 vcs.hooks.host = 127.0.0.1
 
 ; Start VCSServer with this instance as a subprocess, useful for development
 vcs.start_server = false
 
 ; List of enabled VCS backends, available options are:
 ; `hg`  - mercurial
 ; `git` - git
 ; `svn` - subversion
 vcs.backends = hg, git, svn
 
 ; Wait this number of seconds before killing connection to the vcsserver
 vcs.connection_timeout = 3600
 
 ; Compatibility version when creating SVN repositories. Defaults to newest version when commented out.
 ; Set a numeric version for your current SVN e.g 1.8, or 1.12
 ; Legacy available options are: pre-1.4-compatible, pre-1.5-compatible, pre-1.6-compatible, pre-1.8-compatible, pre-1.9-compatible
 #vcs.svn.compatible_version = 1.8
 
+; Cache flag to cache vcsserver remote calls locally
+; It uses cache_region `cache_repo`
+vcs.methods.cache = true
 
 ; ####################################################
 ; Subversion proxy support (mod_dav_svn)
 ; Maps RhodeCode repo groups into SVN paths for Apache
 ; ####################################################
 
 ; Enable or disable the config file generation.
 svn.proxy.generate_config = false
 
 ; Generate config file with `SVNListParentPath` set to `On`.
 svn.proxy.list_parent_path = true
 
 ; Set location and file name of generated config file.
 svn.proxy.config_file_path = %(here)s/mod_dav_svn.conf
 
 ; alternative mod_dav config template. This needs to be a valid mako template
 ; Example template can be found in the source code:
 ; rhodecode/apps/svn_support/templates/mod-dav-svn.conf.mako
 #svn.proxy.config_template = ~/.rccontrol/enterprise-1/custom_svn_conf.mako
 
 ; Used as a prefix to the `Location` block in the generated config file.
 ; In most cases it should be set to `/`.
 svn.proxy.location_root = /
 
 ; Command to reload the mod dav svn configuration on change.
 ; Example: `/etc/init.d/apache2 reload` or /home/USER/apache_reload.sh
 ; Make sure user who runs RhodeCode process is allowed to reload Apache
 #svn.proxy.reload_cmd = /etc/init.d/apache2 reload
 
 ; If the timeout expires before the reload command finishes, the command will
 ; be killed. Setting it to zero means no timeout. Defaults to 10 seconds.
 #svn.proxy.reload_timeout = 10
 
 ; ####################
 ; SSH Support Settings
 ; ####################
 
 ; Defines if a custom authorized_keys file should be created and written on
 ; any change user ssh keys. Setting this to false also disables possibility
 ; of adding SSH keys by users from web interface. Super admins can still
 ; manage SSH Keys.
 ssh.generate_authorized_keyfile = false
 
 ; Options for ssh, default is `no-pty,no-port-forwarding,no-X11-forwarding,no-agent-forwarding`
 # ssh.authorized_keys_ssh_opts =
 
 ; Path to the authorized_keys file where the generate entries are placed.
 ; It is possible to have multiple key files specified in `sshd_config` e.g.
 ; AuthorizedKeysFile %h/.ssh/authorized_keys %h/.ssh/authorized_keys_rhodecode
 ssh.authorized_keys_file_path = ~/.ssh/authorized_keys_rhodecode
 
 ; Command to execute the SSH wrapper. The binary is available in the
 ; RhodeCode installation directory.
 ; e.g ~/.rccontrol/community-1/profile/bin/rc-ssh-wrapper
 ssh.wrapper_cmd = ~/.rccontrol/community-1/rc-ssh-wrapper
 
 ; Allow shell when executing the ssh-wrapper command
 ssh.wrapper_cmd_allow_shell = false
 
 ; Enables logging, and detailed output send back to the client during SSH
 ; operations. Useful for debugging, shouldn't be used in production.
 ssh.enable_debug_logging = false
 
 ; Paths to binary executable, by default they are the names, but we can
 ; override them if we want to use a custom one
 ssh.executable.hg = ~/.rccontrol/vcsserver-1/profile/bin/hg
 ssh.executable.git = ~/.rccontrol/vcsserver-1/profile/bin/git
 ssh.executable.svn = ~/.rccontrol/vcsserver-1/profile/bin/svnserve
 
 ; Enables SSH key generator web interface. Disabling this still allows users
 ; to add their own keys.
 ssh.enable_ui_key_generator = true
 
 
 ; #################
 ; APPENLIGHT CONFIG
 ; #################
 
 ; Appenlight is tailored to work with RhodeCode, see
 ; http://appenlight.rhodecode.com for details how to obtain an account
 
 ; Appenlight integration enabled
-appenlight = false
+#appenlight = false
 
-appenlight.server_url = https://api.appenlight.com
-appenlight.api_key = YOUR_API_KEY
+#appenlight.server_url = https://api.appenlight.com
+#appenlight.api_key = YOUR_API_KEY
 #appenlight.transport_config = https://api.appenlight.com?threaded=1&timeout=5
 
 ; used for JS client
-appenlight.api_public_key = YOUR_API_PUBLIC_KEY
+#appenlight.api_public_key = YOUR_API_PUBLIC_KEY
 
 ; TWEAK AMOUNT OF INFO SENT HERE
 
 ; enables 404 error logging (default False)
-appenlight.report_404 = false
+#appenlight.report_404 = false
 
 ; time in seconds after request is considered being slow (default 1)
-appenlight.slow_request_time = 1
+#appenlight.slow_request_time = 1
 
 ; record slow requests in application
 ; (needs to be enabled for slow datastore recording and time tracking)
-appenlight.slow_requests = true
+#appenlight.slow_requests = true
 
 ; enable hooking to application loggers
-appenlight.logging = true
+#appenlight.logging = true
 
 ; minimum log level for log capture
-appenlight.logging.level = WARNING
+#ppenlight.logging.level = WARNING
 
 ; send logs only from erroneous/slow requests
 ; (saves API quota for intensive logging)
-appenlight.logging_on_error = false
+#appenlight.logging_on_error = false
 
 ; list of additional keywords that should be grabbed from environ object
 ; can be string with comma separated list of words in lowercase
 ; (by default client will always send following info:
 ; 'REMOTE_USER', 'REMOTE_ADDR', 'SERVER_NAME', 'CONTENT_TYPE' + all keys that
 ; start with HTTP* this list be extended with additional keywords here
-appenlight.environ_keys_whitelist =
+#appenlight.environ_keys_whitelist =
 
 ; list of keywords that should be blanked from request object
 ; can be string with comma separated list of words in lowercase
 ; (by default client will always blank keys that contain following words
 ; 'password', 'passwd', 'pwd', 'auth_tkt', 'secret', 'csrf'
 ; this list be extended with additional keywords set here
-appenlight.request_keys_blacklist =
+#appenlight.request_keys_blacklist =
 
 ; list of namespaces that should be ignores when gathering log entries
 ; can be string with comma separated list of namespaces
 ; (by default the client ignores own entries: appenlight_client.client)
-appenlight.log_namespace_blacklist =
+#appenlight.log_namespace_blacklist =
 
 ; Statsd client config, this is used to send metrics to statsd
 ; We recommend setting statsd_exported and scrape them using Promethues
 #statsd.enabled = false
 #statsd.statsd_host = 0.0.0.0
 #statsd.statsd_port = 8125
 #statsd.statsd_prefix =
 #statsd.statsd_ipv6 = false
 
 
+; configure logging automatically at server startup set to false
+; to use the below custom logging config.
+#logging.autoconfigure = true
+
+; specify your own custom logging config file to configure logging
+#logging.logging_conf_file = /path/to/custom_logging.ini
+
 ; Dummy marker to add new entries after.
 ; Add any custom entries below. Please don't remove this marker.
 custom.conf = 1
 
 
 ; #####################
 ; LOGGING CONFIGURATION
 ; #####################
 [loggers]
 keys = root, sqlalchemy, beaker, celery, rhodecode, ssh_wrapper
 
 [handlers]
 keys = console, console_sql
 
 [formatters]
 keys = generic, color_formatter, color_formatter_sql
 
 ; #######
 ; LOGGERS
 ; #######
 [logger_root]
 level = NOTSET
 handlers = console
 
 [logger_sqlalchemy]
 level = INFO
 handlers = console_sql
 qualname = sqlalchemy.engine
 propagate = 0
 
 [logger_beaker]
 level = DEBUG
 handlers =
 qualname = beaker.container
 propagate = 1
 
 [logger_rhodecode]
 level = DEBUG
 handlers =
 qualname = rhodecode
 propagate = 1
 
 [logger_ssh_wrapper]
 level = DEBUG
 handlers =
 qualname = ssh_wrapper
 propagate = 1
 
 [logger_celery]
 level = DEBUG
 handlers =
 qualname = celery
 
 
 ; ########
 ; HANDLERS
 ; ########
 
 [handler_console]
 class = StreamHandler
 args = (sys.stderr, )
 level = INFO
 formatter = generic
 ; To enable JSON formatted logs replace generic with json
 ; This allows sending properly formatted logs to grafana loki or elasticsearch
 #formatter = json
 
 [handler_console_sql]
 ; "level = DEBUG" logs SQL queries and results.
 ; "level = INFO" logs SQL queries.
 ; "level = WARN" logs neither.  (Recommended for production systems.)
 class = StreamHandler
 args = (sys.stderr, )
 level = WARN
 formatter = generic
 
 ; ##########
 ; FORMATTERS
 ; ##########
 
 [formatter_generic]
 class = rhodecode.lib.logging_formatter.ExceptionAwareFormatter
 format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s
 datefmt = %Y-%m-%d %H:%M:%S
 
 [formatter_color_formatter]
 class = rhodecode.lib.logging_formatter.ColorFormatter
 format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s
 datefmt = %Y-%m-%d %H:%M:%S
 
 [formatter_color_formatter_sql]
 class = rhodecode.lib.logging_formatter.ColorFormatterSql
 format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s
 datefmt = %Y-%m-%d %H:%M:%S
 
 [formatter_json]
 format = %(message)s
 class = rhodecode.lib._vendor.jsonlogger.JsonFormatter
  No newline at end of file

rhodecode/apps/file_store/__init__.py

@@ -1,65 +1,66 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2016-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 import os
 from rhodecode.apps.file_store import config_keys
-from rhodecode.config.middleware import _bool_setting, _string_setting
+from rhodecode.config.settings_maker import SettingsMaker
 
 
 def _sanitize_settings_and_apply_defaults(settings):
     """
     Set defaults, convert to python types and validate settings.
     """
-    _bool_setting(settings, config_keys.enabled, 'true')
+    settings_maker = SettingsMaker(settings)
 
-    _string_setting(settings, config_keys.backend, 'local')
+    settings_maker.make_setting(config_keys.enabled, True, parser='bool')
+    settings_maker.make_setting(config_keys.backend, 'local')
 
     default_store = os.path.join(os.path.dirname(settings['__file__']), 'upload_store')
-    _string_setting(settings, config_keys.store_path, default_store)
+    settings_maker.make_setting(config_keys.store_path, default_store)
 
 
 def includeme(config):
     from rhodecode.apps.file_store.views import FileStoreView
 
     settings = config.registry.settings
     _sanitize_settings_and_apply_defaults(settings)
 
     config.add_route(
         name='upload_file',
         pattern='/_file_store/upload')
     config.add_view(
         FileStoreView,
         attr='upload_file',
         route_name='upload_file', request_method='POST', renderer='json_ext')
 
     config.add_route(
         name='download_file',
         pattern='/_file_store/download/{fid:.*}')
     config.add_view(
         FileStoreView,
         attr='download_file',
         route_name='download_file')
 
     config.add_route(
         name='download_file_by_token',
         pattern='/_file_store/token-download/{_auth_token}/{fid:.*}')
     config.add_view(
         FileStoreView,
         attr='download_file_by_token',
         route_name='download_file_by_token')

rhodecode/apps/ssh_support/__init__.py

@@ -1,67 +1,59 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2016-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import logging
 
 from . import config_keys
 from .events import SshKeyFileChangeEvent
 from .subscribers import generate_ssh_authorized_keys_file_subscriber
 
-from rhodecode.config.middleware import _bool_setting, _string_setting
+from rhodecode.config.settings_maker import SettingsMaker
 
 log = logging.getLogger(__name__)
 
 
 def _sanitize_settings_and_apply_defaults(settings):
     """
     Set defaults, convert to python types and validate settings.
     """
-    _bool_setting(settings, config_keys.generate_authorized_keyfile, 'false')
-    _bool_setting(settings, config_keys.wrapper_allow_shell, 'false')
-    _bool_setting(settings, config_keys.enable_debug_logging, 'false')
-    _bool_setting(settings, config_keys.ssh_key_generator_enabled, 'true')
+    settings_maker = SettingsMaker(settings)
+
+    settings_maker.make_setting(config_keys.generate_authorized_keyfile, False, parser='bool')
+    settings_maker.make_setting(config_keys.wrapper_allow_shell, False, parser='bool')
+    settings_maker.make_setting(config_keys.enable_debug_logging, False, parser='bool')
+    settings_maker.make_setting(config_keys.ssh_key_generator_enabled, True, parser='bool')
 
-    _string_setting(settings, config_keys.authorized_keys_file_path,
-                    '~/.ssh/authorized_keys_rhodecode',
-                    lower=False)
-    _string_setting(settings, config_keys.wrapper_cmd, '',
-                    lower=False)
-    _string_setting(settings, config_keys.authorized_keys_line_ssh_opts, '',
-                    lower=False)
+    settings_maker.make_setting(config_keys.authorized_keys_file_path, '~/.ssh/authorized_keys_rhodecode')
+    settings_maker.make_setting(config_keys.wrapper_cmd, '')
+    settings_maker.make_setting(config_keys.authorized_keys_line_ssh_opts, '')
 
-    _string_setting(settings, config_keys.ssh_hg_bin,
-                    '~/.rccontrol/vcsserver-1/profile/bin/hg',
-                    lower=False)
-    _string_setting(settings, config_keys.ssh_git_bin,
-                    '~/.rccontrol/vcsserver-1/profile/bin/git',
-                    lower=False)
-    _string_setting(settings, config_keys.ssh_svn_bin,
-                    '~/.rccontrol/vcsserver-1/profile/bin/svnserve',
-                    lower=False)
+    settings_maker.make_setting(config_keys.ssh_hg_bin, '~/.rccontrol/vcsserver-1/profile/bin/hg')
+    settings_maker.make_setting(config_keys.ssh_git_bin, '~/.rccontrol/vcsserver-1/profile/bin/git')
+    settings_maker.make_setting(config_keys.ssh_svn_bin, '~/.rccontrol/vcsserver-1/profile/bin/svnserve')
 
 
 def includeme(config):
     settings = config.registry.settings
     _sanitize_settings_and_apply_defaults(settings)
 
     # if we have enable generation of file, subscribe to event
     if settings[config_keys.generate_authorized_keyfile]:
         config.add_subscriber(
             generate_ssh_authorized_keys_file_subscriber, SshKeyFileChangeEvent)

rhodecode/apps/svn_support/__init__.py

@@ -1,90 +1,89 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2016-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 import os
 import logging
-import shlex
 from pyramid import compat
 
 # Do not use `from rhodecode import events` here, it will be overridden by the
 # events module in this package due to pythons import mechanism.
 from rhodecode.events import RepoGroupEvent
 from rhodecode.subscribers import AsyncSubprocessSubscriber
-from rhodecode.config.middleware import (
-    _bool_setting, _string_setting, _int_setting)
+from rhodecode.config.settings_maker import SettingsMaker
 
 from .events import ModDavSvnConfigChange
 from .subscribers import generate_config_subscriber
 from . import config_keys
 
 
 log = logging.getLogger(__name__)
 
 
 def _sanitize_settings_and_apply_defaults(settings):
     """
     Set defaults, convert to python types and validate settings.
     """
-    _bool_setting(settings, config_keys.generate_config, 'false')
-    _bool_setting(settings, config_keys.list_parent_path, 'true')
-    _int_setting(settings, config_keys.reload_timeout, 10)
-    _string_setting(settings, config_keys.config_file_path, '', lower=False)
-    _string_setting(settings, config_keys.location_root, '/', lower=False)
-    _string_setting(settings, config_keys.reload_command, '', lower=False)
-    _string_setting(settings, config_keys.template, '', lower=False)
+    settings_maker = SettingsMaker(settings)
+    settings_maker.make_setting(config_keys.generate_config, False, parser='bool')
+    settings_maker.make_setting(config_keys.list_parent_path, True, parser='bool')
+    settings_maker.make_setting(config_keys.reload_timeout, 10, parser='bool')
+    settings_maker.make_setting(config_keys.config_file_path, '')
+    settings_maker.make_setting(config_keys.location_root, '/')
+    settings_maker.make_setting(config_keys.reload_command, '')
+    settings_maker.make_setting(config_keys.template, '')
 
     # Convert negative timeout values to zero.
     if settings[config_keys.reload_timeout] < 0:
         settings[config_keys.reload_timeout] = 0
 
     # Append path separator to location root.
     settings[config_keys.location_root] = _append_path_sep(
         settings[config_keys.location_root])
 
     # Validate settings.
     if settings[config_keys.generate_config]:
         assert len(settings[config_keys.config_file_path]) > 0
 
 
 def _append_path_sep(path):
     """
     Append the path separator if missing.
     """
     if isinstance(path, compat.string_types) and not path.endswith(os.path.sep):
         path += os.path.sep
     return path
 
 
 def includeme(config):
     settings = config.registry.settings
     _sanitize_settings_and_apply_defaults(settings)
 
     if settings[config_keys.generate_config]:
         # Add subscriber to generate the Apache mod dav svn configuration on
         # repository group events.
         config.add_subscriber(generate_config_subscriber, RepoGroupEvent)
 
         # If a reload command is set add a subscriber to execute it on
         # configuration changes.
         reload_cmd = settings[config_keys.reload_command]
         if reload_cmd:
             reload_timeout = settings[config_keys.reload_timeout] or None
             reload_subscriber = AsyncSubprocessSubscriber(
                 cmd=reload_cmd, timeout=reload_timeout)
             config.add_subscriber(reload_subscriber, ModDavSvnConfigChange)

rhodecode/config/environment.py

@@ -1,122 +1,87 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import os
 import logging
 import rhodecode
 
 from rhodecode.config import utils
 
 from rhodecode.lib.utils import load_rcextensions
 from rhodecode.lib.utils2 import str2bool
 from rhodecode.lib.vcs import connect_vcs
 
 log = logging.getLogger(__name__)
 
 
 def load_pyramid_environment(global_config, settings):
     # Some parts of the code expect a merge of global and app settings.
     settings_merged = global_config.copy()
     settings_merged.update(settings)
 
     # TODO(marcink): probably not required anymore
     # configure channelstream,
     settings_merged['channelstream_config'] = {
         'enabled': str2bool(settings_merged.get('channelstream.enabled', False)),
         'server': settings_merged.get('channelstream.server'),
         'secret': settings_merged.get('channelstream.secret')
     }
 
     # If this is a test run we prepare the test environment like
     # creating a test database, test search index and test repositories.
     # This has to be done before the database connection is initialized.
     if settings['is_test']:
         rhodecode.is_test = True
         rhodecode.disable_error_handler = True
         from rhodecode import authentication
         authentication.plugin_default_auth_ttl = 0
 
         utils.initialize_test_environment(settings_merged)
 
     # Initialize the database connection.
     utils.initialize_database(settings_merged)
 
     load_rcextensions(root_path=settings_merged['here'])
 
     # Limit backends to `vcs.backends` from configuration, and preserve the order
     for alias in rhodecode.BACKENDS.keys():
         if alias not in settings['vcs.backends']:
             del rhodecode.BACKENDS[alias]
 
     _sorted_backend = sorted(rhodecode.BACKENDS.items(),
                              key=lambda item: settings['vcs.backends'].index(item[0]))
     rhodecode.BACKENDS = rhodecode.OrderedDict(_sorted_backend)
 
     log.info('Enabled VCS backends: %s', rhodecode.BACKENDS.keys())
 
     # initialize vcs client and optionally run the server if enabled
     vcs_server_uri = settings['vcs.server']
     vcs_server_enabled = settings['vcs.server.enable']
 
     utils.configure_vcs(settings)
 
     # Store the settings to make them available to other modules.
 
     rhodecode.PYRAMID_SETTINGS = settings_merged
     rhodecode.CONFIG = settings_merged
     rhodecode.CONFIG['default_user_id'] = utils.get_default_user_id()
 
     if vcs_server_enabled:
         connect_vcs(vcs_server_uri, utils.get_vcs_server_protocol(settings))
-
-
-def get_rc_env_settings(prefix='ENV_{}'):
-    return {'ENV_{}'.format(key): value for key, value in os.environ.items()}
-
-
-def substitute_values(mapping, substitutions):
-    result = {}
-
-    try:
-        for key, value in mapping.items():
-            # initialize without substitution first
-            result[key] = value
-
-            # Note: Cannot use regular replacements, since they would clash
-            # with the implementation of ConfigParser. Using "format" instead.
-            try:
-                result[key] = value.format(**substitutions)
-            except KeyError as e:
-                env_var = '{}'.format(e.args[0])
-
-                msg = 'Failed to substitute: `{key}={{{var}}}` with environment entry. ' \
-                      'Make sure your environment has {var} set, or remove this ' \
-                      'variable from config file'.format(key=key, var=env_var)
-
-                if env_var.startswith('ENV_'):
-                    raise ValueError(msg)
-                else:
-                    log.warning(msg)
-
-    except ValueError as e:
-        log.warning('Failed to substitute ENV variable: %s', e)
-        result = mapping
-
-    return result
  No newline at end of file

rhodecode/config/middleware.py

@@ -1,761 +1,611 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import os
 import sys
-import logging
 import collections
 import tempfile
 import time
+import logging.config
 
 from paste.gzipper import make_gzip_middleware
 import pyramid.events
 from pyramid.wsgi import wsgiapp
 from pyramid.authorization import ACLAuthorizationPolicy
 from pyramid.config import Configurator
 from pyramid.settings import asbool, aslist
 from pyramid.httpexceptions import (
     HTTPException, HTTPError, HTTPInternalServerError, HTTPFound, HTTPNotFound)
 from pyramid.renderers import render_to_response
 
 from rhodecode.model import meta
 from rhodecode.config import patches
 from rhodecode.config import utils as config_utils
-from rhodecode.config.environment import load_pyramid_environment, substitute_values, get_rc_env_settings
+from rhodecode.config.settings_maker import SettingsMaker
+from rhodecode.config.environment import load_pyramid_environment
 
 import rhodecode.events
 from rhodecode.lib.middleware.vcs import VCSMiddleware
 from rhodecode.lib.request import Request
 from rhodecode.lib.vcs import VCSCommunicationError
 from rhodecode.lib.exceptions import VCSServerUnavailable
 from rhodecode.lib.middleware.appenlight import wrap_in_appenlight_if_enabled
 from rhodecode.lib.middleware.https_fixup import HttpsFixup
 from rhodecode.lib.plugins.utils import register_rhodecode_plugin
-from rhodecode.lib.utils2 import aslist as rhodecode_aslist, AttributeDict
+from rhodecode.lib.utils2 import AttributeDict
 from rhodecode.lib.exc_tracking import store_exception
 from rhodecode.subscribers import (
     scan_repositories_if_enabled, write_js_routes_if_enabled,
     write_metadata_if_needed, write_usage_data)
 from rhodecode.lib.statsd_client import StatsdClient
 
 log = logging.getLogger(__name__)
 
 
 def is_http_error(response):
     # error which should have traceback
     return response.status_code > 499
 
 
 def should_load_all():
     """
     Returns if all application components should be loaded. In some cases it's
     desired to skip apps loading for faster shell script execution
     """
     ssh_cmd = os.environ.get('RC_CMD_SSH_WRAPPER')
     if ssh_cmd:
         return False
 
     return True
 
 
 def make_pyramid_app(global_config, **settings):
     """
     Constructs the WSGI application based on Pyramid.
 
     Specials:
 
     * The application can also be integrated like a plugin via the call to
       `includeme`. This is accompanied with the other utility functions which
       are called. Changing this should be done with great care to not break
       cases when these fragments are assembled from another place.
 
     """
-
-    # Allows to use format style "{ENV_NAME}" placeholders in the configuration. It
-    # will be replaced by the value of the environment variable "NAME" in this case.
     start_time = time.time()
     log.info('Pyramid app config starting')
 
-    global_config = substitute_values(global_config, get_rc_env_settings())
-    settings = substitute_values(settings, get_rc_env_settings())
+    sanitize_settings_and_apply_defaults(global_config, settings)
 
     # init and bootstrap StatsdClient
     StatsdClient.setup(settings)
 
-    debug = asbool(global_config.get('debug'))
-    if debug:
-        enable_debug()
-
-    sanitize_settings_and_apply_defaults(global_config, settings)
-
     config = Configurator(settings=settings)
     # Init our statsd at very start
     config.registry.statsd = StatsdClient.statsd
 
     # Apply compatibility patches
     patches.inspect_getargspec()
 
     load_pyramid_environment(global_config, settings)
 
     # Static file view comes first
     includeme_first(config)
 
     includeme(config)
 
     pyramid_app = config.make_wsgi_app()
     pyramid_app = wrap_app_in_wsgi_middlewares(pyramid_app, config)
     pyramid_app.config = config
 
-    config.configure_celery(global_config['__file__'])
+    celery_settings = get_celery_config(settings)
+    config.configure_celery(celery_settings)
 
     # creating the app uses a connection - return it after we are done
     meta.Session.remove()
 
     total_time = time.time() - start_time
     log.info('Pyramid app `%s` created and configured in %.2fs',
-             pyramid_app.func_name, total_time)
+             getattr(pyramid_app, 'func_name', 'pyramid_app'), total_time)
     return pyramid_app
 
 
+def get_celery_config(settings):
+    """
+    Converts basic ini configuration into celery 4.X options
+    """
+
+    def key_converter(key_name):
+        pref = 'celery.'
+        if key_name.startswith(pref):
+            return key_name[len(pref):].replace('.', '_').lower()
+
+    def type_converter(parsed_key, value):
+        # cast to int
+        if value.isdigit():
+            return int(value)
+
+        # cast to bool
+        if value.lower() in ['true', 'false', 'True', 'False']:
+            return value.lower() == 'true'
+        return value
+
+    celery_config = {}
+    for k, v in settings.items():
+        pref = 'celery.'
+        if k.startswith(pref):
+            celery_config[key_converter(k)] = type_converter(key_converter(k), v)
+
+    # TODO:rethink if we want to support celerybeat based file config, probably NOT
+    # beat_config = {}
+    # for section in parser.sections():
+    #     if section.startswith('celerybeat:'):
+    #         name = section.split(':', 1)[1]
+    #         beat_config[name] = get_beat_config(parser, section)
+
+    # final compose of settings
+    celery_settings = {}
+
+    if celery_config:
+        celery_settings.update(celery_config)
+    # if beat_config:
+    #     celery_settings.update({'beat_schedule': beat_config})
+
+    return celery_settings
+
+
 def not_found_view(request):
     """
     This creates the view which should be registered as not-found-view to
     pyramid.
     """
 
     if not getattr(request, 'vcs_call', None):
         # handle like regular case with our error_handler
         return error_handler(HTTPNotFound(), request)
 
     # handle not found view as a vcs call
     settings = request.registry.settings
     ae_client = getattr(request, 'ae_client', None)
     vcs_app = VCSMiddleware(
         HTTPNotFound(), request.registry, settings,
         appenlight_client=ae_client)
 
     return wsgiapp(vcs_app)(None, request)
 
 
 def error_handler(exception, request):
     import rhodecode
     from rhodecode.lib import helpers
     from rhodecode.lib.utils2 import str2bool
 
     rhodecode_title = rhodecode.CONFIG.get('rhodecode_title') or 'RhodeCode'
 
     base_response = HTTPInternalServerError()
     # prefer original exception for the response since it may have headers set
     if isinstance(exception, HTTPException):
         base_response = exception
     elif isinstance(exception, VCSCommunicationError):
         base_response = VCSServerUnavailable()
 
     if is_http_error(base_response):
         log.exception(
             'error occurred handling this request for path: %s', request.path)
 
     error_explanation = base_response.explanation or str(base_response)
     if base_response.status_code == 404:
         error_explanation += " Optionally you don't have permission to access this page."
     c = AttributeDict()
     c.error_message = base_response.status
     c.error_explanation = error_explanation
     c.visual = AttributeDict()
 
     c.visual.rhodecode_support_url = (
         request.registry.settings.get('rhodecode_support_url') or
         request.route_url('rhodecode_support')
     )
     c.redirect_time = 0
     c.rhodecode_name = rhodecode_title
     if not c.rhodecode_name:
         c.rhodecode_name = 'Rhodecode'
 
     c.causes = []
     if is_http_error(base_response):
         c.causes.append('Server is overloaded.')
         c.causes.append('Server database connection is lost.')
         c.causes.append('Server expected unhandled error.')
 
     if hasattr(base_response, 'causes'):
         c.causes = base_response.causes
 
     c.messages = helpers.flash.pop_messages(request=request)
 
     exc_info = sys.exc_info()
     c.exception_id = id(exc_info)
     c.show_exception_id = isinstance(base_response, VCSServerUnavailable) \
                           or base_response.status_code > 499
     c.exception_id_url = request.route_url(
         'admin_settings_exception_tracker_show', exception_id=c.exception_id)
 
     if c.show_exception_id:
         store_exception(c.exception_id, exc_info)
     c.exception_debug = str2bool(rhodecode.CONFIG.get('debug'))
     c.exception_config_ini = rhodecode.CONFIG.get('__file__')
 
     response = render_to_response(
         '/errors/error_document.mako', {'c': c, 'h': helpers}, request=request,
         response=base_response)
 
     statsd = request.registry.statsd
     if statsd and base_response.status_code > 499:
         exc_type = "{}.{}".format(exception.__class__.__module__, exception.__class__.__name__)
         statsd.incr('rhodecode_exception_total',
                     tags=["exc_source:web",
                           "http_code:{}".format(base_response.status_code),
                           "type:{}".format(exc_type)])
 
     return response
 
 
 def includeme_first(config):
     # redirect automatic browser favicon.ico requests to correct place
     def favicon_redirect(context, request):
         return HTTPFound(
             request.static_path('rhodecode:public/images/favicon.ico'))
 
     config.add_view(favicon_redirect, route_name='favicon')
     config.add_route('favicon', '/favicon.ico')
 
     def robots_redirect(context, request):
         return HTTPFound(
             request.static_path('rhodecode:public/robots.txt'))
 
     config.add_view(robots_redirect, route_name='robots')
     config.add_route('robots', '/robots.txt')
 
     config.add_static_view(
         '_static/deform', 'deform:static')
     config.add_static_view(
         '_static/rhodecode', path='rhodecode:public', cache_max_age=3600 * 24)
 
 
 def includeme(config, auth_resources=None):
     from rhodecode.lib.celerylib.loader import configure_celery
     log.debug('Initializing main includeme from %s', os.path.basename(__file__))
     settings = config.registry.settings
     config.set_request_factory(Request)
 
     # plugin information
     config.registry.rhodecode_plugins = collections.OrderedDict()
 
     config.add_directive(
         'register_rhodecode_plugin', register_rhodecode_plugin)
 
     config.add_directive('configure_celery', configure_celery)
 
-    if asbool(settings.get('appenlight', 'false')):
+    if settings.get('appenlight', False):
         config.include('appenlight_client.ext.pyramid_tween')
 
     load_all = should_load_all()
 
     # Includes which are required. The application would fail without them.
     config.include('pyramid_mako')
     config.include('rhodecode.lib.rc_beaker')
     config.include('rhodecode.lib.rc_cache')
     config.include('rhodecode.apps._base.navigation')
     config.include('rhodecode.apps._base.subscribers')
     config.include('rhodecode.tweens')
     config.include('rhodecode.authentication')
 
     if load_all:
         ce_auth_resources = [
             'rhodecode.authentication.plugins.auth_crowd',
             'rhodecode.authentication.plugins.auth_headers',
             'rhodecode.authentication.plugins.auth_jasig_cas',
             'rhodecode.authentication.plugins.auth_ldap',
             'rhodecode.authentication.plugins.auth_pam',
             'rhodecode.authentication.plugins.auth_rhodecode',
             'rhodecode.authentication.plugins.auth_token',
         ]
 
         # load CE authentication plugins
 
         if auth_resources:
             ce_auth_resources.extend(auth_resources)
 
         for resource in ce_auth_resources:
             config.include(resource)
 
         # Auto discover authentication plugins and include their configuration.
         if asbool(settings.get('auth_plugin.import_legacy_plugins', 'true')):
             from rhodecode.authentication import discover_legacy_plugins
             discover_legacy_plugins(config)
 
     # apps
     if load_all:
         config.include('rhodecode.api')
         config.include('rhodecode.apps._base')
         config.include('rhodecode.apps.hovercards')
         config.include('rhodecode.apps.ops')
         config.include('rhodecode.apps.channelstream')
         config.include('rhodecode.apps.file_store')
         config.include('rhodecode.apps.admin')
         config.include('rhodecode.apps.login')
         config.include('rhodecode.apps.home')
         config.include('rhodecode.apps.journal')
 
         config.include('rhodecode.apps.repository')
         config.include('rhodecode.apps.repo_group')
         config.include('rhodecode.apps.user_group')
         config.include('rhodecode.apps.search')
         config.include('rhodecode.apps.user_profile')
         config.include('rhodecode.apps.user_group_profile')
         config.include('rhodecode.apps.my_account')
         config.include('rhodecode.apps.gist')
 
         config.include('rhodecode.apps.svn_support')
         config.include('rhodecode.apps.ssh_support')
         config.include('rhodecode.apps.debug_style')
 
     if load_all:
         config.include('rhodecode.integrations')
 
     config.add_route('rhodecode_support', 'https://rhodecode.com/help/', static=True)
     config.add_translation_dirs('rhodecode:i18n/')
     settings['default_locale_name'] = settings.get('lang', 'en')
 
     # Add subscribers.
     if load_all:
         config.add_subscriber(scan_repositories_if_enabled,
                               pyramid.events.ApplicationCreated)
         config.add_subscriber(write_metadata_if_needed,
                               pyramid.events.ApplicationCreated)
         config.add_subscriber(write_usage_data,
                               pyramid.events.ApplicationCreated)
         config.add_subscriber(write_js_routes_if_enabled,
                               pyramid.events.ApplicationCreated)
 
     # request custom methods
     config.add_request_method(
         'rhodecode.lib.partial_renderer.get_partial_renderer',
         'get_partial_renderer')
 
     config.add_request_method(
         'rhodecode.lib.request_counter.get_request_counter',
         'request_count')
 
     # Set the authorization policy.
     authz_policy = ACLAuthorizationPolicy()
     config.set_authorization_policy(authz_policy)
 
     # Set the default renderer for HTML templates to mako.
     config.add_mako_renderer('.html')
 
     config.add_renderer(
         name='json_ext',
         factory='rhodecode.lib.ext_json_renderer.pyramid_ext_json')
 
     config.add_renderer(
         name='string_html',
         factory='rhodecode.lib.string_renderer.html')
 
     # include RhodeCode plugins
     includes = aslist(settings.get('rhodecode.includes', []))
     for inc in includes:
         config.include(inc)
 
     # custom not found view, if our pyramid app doesn't know how to handle
     # the request pass it to potential VCS handling ap
     config.add_notfound_view(not_found_view)
     if not settings.get('debugtoolbar.enabled', False):
         # disabled debugtoolbar handle all exceptions via the error_handlers
         config.add_view(error_handler, context=Exception)
 
     # all errors including 403/404/50X
     config.add_view(error_handler, context=HTTPError)
 
 
 def wrap_app_in_wsgi_middlewares(pyramid_app, config):
     """
     Apply outer WSGI middlewares around the application.
     """
     registry = config.registry
     settings = registry.settings
 
     # enable https redirects based on HTTP_X_URL_SCHEME set by proxy
     pyramid_app = HttpsFixup(pyramid_app, settings)
 
     pyramid_app, _ae_client = wrap_in_appenlight_if_enabled(
         pyramid_app, settings)
     registry.ae_client = _ae_client
 
     if settings['gzip_responses']:
         pyramid_app = make_gzip_middleware(
             pyramid_app, settings, compress_level=1)
 
     # this should be the outer most middleware in the wsgi stack since
     # middleware like Routes make database calls
     def pyramid_app_with_cleanup(environ, start_response):
         try:
             return pyramid_app(environ, start_response)
         finally:
             # Dispose current database session and rollback uncommitted
             # transactions.
             meta.Session.remove()
 
             # In a single threaded mode server, on non sqlite db we should have
             # '0 Current Checked out connections' at the end of a request,
             # if not, then something, somewhere is leaving a connection open
             pool = meta.Base.metadata.bind.engine.pool
             log.debug('sa pool status: %s', pool.status())
             log.debug('Request processing finalized')
 
     return pyramid_app_with_cleanup
 
 
 def sanitize_settings_and_apply_defaults(global_config, settings):
     """
     Applies settings defaults and does all type conversion.
 
     We would move all settings parsing and preparation into this place, so that
     we have only one place left which deals with this part. The remaining parts
     of the application would start to rely fully on well prepared settings.
 
     This piece would later be split up per topic to avoid a big fat monster
     function.
     """
 
-    settings.setdefault('rhodecode.edition', 'Community Edition')
-    settings.setdefault('rhodecode.edition_id', 'CE')
+    global_settings_maker = SettingsMaker(global_config)
+    global_settings_maker.make_setting('debug', default=False, parser='bool')
+    debug_enabled = asbool(global_config.get('debug'))
+
+    settings_maker = SettingsMaker(settings)
+
+    settings_maker.make_setting(
+        'logging.autoconfigure',
+        default=True,
+        parser='bool')
+
+    logging_conf = os.path.join(os.path.dirname(global_config.get('__file__')), 'logging.ini')
+    settings_maker.enable_logging(logging_conf, level='INFO' if debug_enabled else 'DEBUG')
+
+    # Default includes, possible to change as a user
+    pyramid_includes = settings_maker.make_setting('pyramid.includes', [], parser='list:newline')
+    log.debug(
+        "Using the following pyramid.includes: %s",
+        pyramid_includes)
+
+    settings_maker.make_setting('rhodecode.edition', 'Community Edition')
+    settings_maker.make_setting('rhodecode.edition_id', 'CE')
 
     if 'mako.default_filters' not in settings:
         # set custom default filters if we don't have it defined
         settings['mako.imports'] = 'from rhodecode.lib.base import h_filter'
         settings['mako.default_filters'] = 'h_filter'
 
     if 'mako.directories' not in settings:
         mako_directories = settings.setdefault('mako.directories', [
             # Base templates of the original application
             'rhodecode:templates',
         ])
         log.debug(
             "Using the following Mako template directories: %s",
             mako_directories)
 
     # NOTE(marcink): fix redis requirement for schema of connection since 3.X
     if 'beaker.session.type' in settings and settings['beaker.session.type'] == 'ext:redis':
         raw_url = settings['beaker.session.url']
         if not raw_url.startswith(('redis://', 'rediss://', 'unix://')):
             settings['beaker.session.url'] = 'redis://' + raw_url
 
-    # Default includes, possible to change as a user
-    pyramid_includes = settings.setdefault('pyramid.includes', [])
-    log.debug(
-        "Using the following pyramid.includes: %s",
-        pyramid_includes)
+    settings_maker.make_setting('__file__', global_config.get('__file__'))
 
     # TODO: johbo: Re-think this, usually the call to config.include
     # should allow to pass in a prefix.
-    settings.setdefault('rhodecode.api.url', '/_admin/api')
-    settings.setdefault('__file__', global_config.get('__file__'))
+    settings_maker.make_setting('rhodecode.api.url', '/_admin/api')
 
     # Sanitize generic settings.
-    _list_setting(settings, 'default_encoding', 'UTF-8')
-    _bool_setting(settings, 'is_test', 'false')
-    _bool_setting(settings, 'gzip_responses', 'false')
+    settings_maker.make_setting('default_encoding', 'UTF-8', parser='list')
+    settings_maker.make_setting('is_test', False, parser='bool')
+    settings_maker.make_setting('gzip_responses', False, parser='bool')
 
-    # Call split out functions that sanitize settings for each topic.
-    _sanitize_appenlight_settings(settings)
-    _sanitize_vcs_settings(settings)
+    settings_maker.make_setting('vcs.svn.compatible_version', '')
+    settings_maker.make_setting('vcs.hooks.protocol', 'http')
+    settings_maker.make_setting('vcs.hooks.host', '127.0.0.1')
+    settings_maker.make_setting('vcs.scm_app_implementation', 'http')
+    settings_maker.make_setting('vcs.server', '')
+    settings_maker.make_setting('vcs.server.protocol', 'http')
+    settings_maker.make_setting('startup.import_repos', 'false', parser='bool')
+    settings_maker.make_setting('vcs.hooks.direct_calls', 'false', parser='bool')
+    settings_maker.make_setting('vcs.server.enable', 'true', parser='bool')
+    settings_maker.make_setting('vcs.start_server', 'false', parser='bool')
+    settings_maker.make_setting('vcs.backends', 'hg, git, svn', parser='list')
+    settings_maker.make_setting('vcs.connection_timeout', 3600, parser='int')
+
+    settings_maker.make_setting('vcs.methods.cache', True, parser='bool')
+
+    # Support legacy values of vcs.scm_app_implementation. Legacy
+    # configurations may use 'rhodecode.lib.middleware.utils.scm_app_http', or
+    # disabled since 4.13 'vcsserver.scm_app' which is now mapped to 'http'.
+    scm_app_impl = settings['vcs.scm_app_implementation']
+    if scm_app_impl in ['rhodecode.lib.middleware.utils.scm_app_http', 'vcsserver.scm_app']:
+        settings['vcs.scm_app_implementation'] = 'http'
+
+    settings_maker.make_setting('appenlight', False, parser='bool')
+
     _sanitize_cache_settings(settings)
 
     # configure instance id
     config_utils.set_instance_id(settings)
 
     return settings
 
 
-def enable_debug():
-    """
-    Helper to enable debug on running instance
-    :return:
-    """
-    import tempfile
-    import textwrap
-    import logging.config
-
-    ini_template = textwrap.dedent("""
-    #####################################
-    ### DEBUG LOGGING CONFIGURATION  ####
-    #####################################
-    [loggers]
-    keys = root, sqlalchemy, beaker, celery, rhodecode, ssh_wrapper
-
-    [handlers]
-    keys = console, console_sql
-
-    [formatters]
-    keys = generic, color_formatter, color_formatter_sql
-
-    #############
-    ## LOGGERS ##
-    #############
-    [logger_root]
-    level = NOTSET
-    handlers = console
-
-    [logger_sqlalchemy]
-    level = INFO
-    handlers = console_sql
-    qualname = sqlalchemy.engine
-    propagate = 0
-
-    [logger_beaker]
-    level = DEBUG
-    handlers =
-    qualname = beaker.container
-    propagate = 1
-
-    [logger_rhodecode]
-    level = DEBUG
-    handlers =
-    qualname = rhodecode
-    propagate = 1
-
-    [logger_ssh_wrapper]
-    level = DEBUG
-    handlers =
-    qualname = ssh_wrapper
-    propagate = 1
-
-    [logger_celery]
-    level = DEBUG
-    handlers =
-    qualname = celery
-
-
-    ##############
-    ## HANDLERS ##
-    ##############
-
-    [handler_console]
-    class = StreamHandler
-    args = (sys.stderr, )
-    level = DEBUG
-    formatter = color_formatter
+def _sanitize_cache_settings(settings):
+    settings_maker = SettingsMaker(settings)
 
-    [handler_console_sql]
-    # "level = DEBUG" logs SQL queries and results.
-    # "level = INFO" logs SQL queries.
-    # "level = WARN" logs neither.  (Recommended for production systems.)
-    class = StreamHandler
-    args = (sys.stderr, )
-    level = WARN
-    formatter = color_formatter_sql
-
-    ################
-    ## FORMATTERS ##
-    ################
-
-    [formatter_generic]
-    class = rhodecode.lib.logging_formatter.ExceptionAwareFormatter
-    format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s | %(req_id)s
-    datefmt = %Y-%m-%d %H:%M:%S
-
-    [formatter_color_formatter]
-    class = rhodecode.lib.logging_formatter.ColorRequestTrackingFormatter
-    format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s | %(req_id)s
-    datefmt = %Y-%m-%d %H:%M:%S
-
-    [formatter_color_formatter_sql]
-    class = rhodecode.lib.logging_formatter.ColorFormatterSql
-    format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s
-    datefmt = %Y-%m-%d %H:%M:%S    
-    """)
-
-    with tempfile.NamedTemporaryFile(prefix='rc_debug_logging_', suffix='.ini',
-                                     delete=False) as f:
-        log.info('Saved Temporary DEBUG config at %s', f.name)
-        f.write(ini_template)
-
-    logging.config.fileConfig(f.name)
-    log.debug('DEBUG MODE ON')
-    os.remove(f.name)
-
-
-def _sanitize_appenlight_settings(settings):
-    _bool_setting(settings, 'appenlight', 'false')
-
-
-def _sanitize_vcs_settings(settings):
-    """
-    Applies settings defaults and does type conversion for all VCS related
-    settings.
-    """
-    _string_setting(settings, 'vcs.svn.compatible_version', '')
-    _string_setting(settings, 'vcs.hooks.protocol', 'http')
-    _string_setting(settings, 'vcs.hooks.host', '127.0.0.1')
-    _string_setting(settings, 'vcs.scm_app_implementation', 'http')
-    _string_setting(settings, 'vcs.server', '')
-    _string_setting(settings, 'vcs.server.protocol', 'http')
-    _bool_setting(settings, 'startup.import_repos', 'false')
-    _bool_setting(settings, 'vcs.hooks.direct_calls', 'false')
-    _bool_setting(settings, 'vcs.server.enable', 'true')
-    _bool_setting(settings, 'vcs.start_server', 'false')
-    _list_setting(settings, 'vcs.backends', 'hg, git, svn')
-    _int_setting(settings, 'vcs.connection_timeout', 3600)
-
-    # Support legacy values of vcs.scm_app_implementation. Legacy
-    # configurations may use 'rhodecode.lib.middleware.utils.scm_app_http', or
-    # disabled since 4.13 'vcsserver.scm_app' which is now mapped to 'http'.
-    scm_app_impl = settings['vcs.scm_app_implementation']
-    if scm_app_impl in ['rhodecode.lib.middleware.utils.scm_app_http', 'vcsserver.scm_app']:
-        settings['vcs.scm_app_implementation'] = 'http'
-
-
-def _sanitize_cache_settings(settings):
     temp_store = tempfile.gettempdir()
     default_cache_dir = os.path.join(temp_store, 'rc_cache')
 
     # save default, cache dir, and use it for all backends later.
-    default_cache_dir = _string_setting(
-        settings,
+    default_cache_dir = settings_maker.make_setting(
         'cache_dir',
-        default_cache_dir, lower=False, default_when_empty=True)
-
-    # ensure we have our dir created
-    if not os.path.isdir(default_cache_dir):
-        os.makedirs(default_cache_dir, mode=0o755)
+        default=default_cache_dir, default_when_empty=True,
+        parser='dir:ensured')
 
     # exception store cache
-    _string_setting(
-        settings,
+    settings_maker.make_setting(
         'exception_tracker.store_path',
-        temp_store, lower=False, default_when_empty=True)
-    _bool_setting(
-        settings,
-        'exception_tracker.send_email',
-        'false')
-    _string_setting(
-        settings,
-        'exception_tracker.email_prefix',
-        '[RHODECODE ERROR]', lower=False, default_when_empty=True)
+        default=os.path.join(default_cache_dir, 'exc_store'), default_when_empty=True,
+        parser='dir:ensured'
+    )
+
+    settings_maker.make_setting(
+        'celerybeat-schedule.path',
+        default=os.path.join(default_cache_dir, 'celerybeat_schedule', 'celerybeat-schedule.db'), default_when_empty=True,
+        parser='file:ensured'
+    )
+
+    settings_maker.make_setting('exception_tracker.send_email', False, parser='bool')
+    settings_maker.make_setting('exception_tracker.email_prefix', '[RHODECODE ERROR]', default_when_empty=True)
 
     # cache_perms
-    _string_setting(
-        settings,
-        'rc_cache.cache_perms.backend',
-        'dogpile.cache.rc.file_namespace', lower=False)
-    _int_setting(
-        settings,
-        'rc_cache.cache_perms.expiration_time',
-        60)
-    _string_setting(
-        settings,
-        'rc_cache.cache_perms.arguments.filename',
-        os.path.join(default_cache_dir, 'rc_cache_1'), lower=False)
+    settings_maker.make_setting('rc_cache.cache_perms.backend', 'dogpile.cache.rc.file_namespace')
+    settings_maker.make_setting('rc_cache.cache_perms.expiration_time', 60, parser='int')
+    settings_maker.make_setting('rc_cache.cache_perms.arguments.filename', os.path.join(default_cache_dir, 'rhodecode_cache_perms.db'))
 
     # cache_repo
-    _string_setting(
-        settings,
-        'rc_cache.cache_repo.backend',
-        'dogpile.cache.rc.file_namespace', lower=False)
-    _int_setting(
-        settings,
-        'rc_cache.cache_repo.expiration_time',
-        60)
-    _string_setting(
-        settings,
-        'rc_cache.cache_repo.arguments.filename',
-        os.path.join(default_cache_dir, 'rc_cache_2'), lower=False)
+    settings_maker.make_setting('rc_cache.cache_repo.backend', 'dogpile.cache.rc.file_namespace')
+    settings_maker.make_setting('rc_cache.cache_repo.expiration_time', 60, parser='int')
+    settings_maker.make_setting('rc_cache.cache_repo.arguments.filename', os.path.join(default_cache_dir, 'rhodecode_cache_repo.db'))
 
     # cache_license
-    _string_setting(
-        settings,
-        'rc_cache.cache_license.backend',
-        'dogpile.cache.rc.file_namespace', lower=False)
-    _int_setting(
-        settings,
-        'rc_cache.cache_license.expiration_time',
-        5*60)
-    _string_setting(
-        settings,
-        'rc_cache.cache_license.arguments.filename',
-        os.path.join(default_cache_dir, 'rc_cache_3'), lower=False)
+    settings_maker.make_setting('rc_cache.cache_license.backend', 'dogpile.cache.rc.file_namespace')
+    settings_maker.make_setting('rc_cache.cache_license.expiration_time', 5*60, parser='int')
+    settings_maker.make_setting('rc_cache.cache_license.arguments.filename', os.path.join(default_cache_dir, 'rhodecode_cache_license.db'))
 
     # cache_repo_longterm memory, 96H
-    _string_setting(
-        settings,
-        'rc_cache.cache_repo_longterm.backend',
-        'dogpile.cache.rc.memory_lru', lower=False)
-    _int_setting(
-        settings,
-        'rc_cache.cache_repo_longterm.expiration_time',
-        345600)
-    _int_setting(
-        settings,
-        'rc_cache.cache_repo_longterm.max_size',
-        10000)
+    settings_maker.make_setting('rc_cache.cache_repo_longterm.backend', 'dogpile.cache.rc.memory_lru')
+    settings_maker.make_setting('rc_cache.cache_repo_longterm.expiration_time', 345600, parser='int')
+    settings_maker.make_setting('rc_cache.cache_repo_longterm.max_size', 10000, parser='int')
 
     # sql_cache_short
-    _string_setting(
-        settings,
-        'rc_cache.sql_cache_short.backend',
-        'dogpile.cache.rc.memory_lru', lower=False)
-    _int_setting(
-        settings,
-        'rc_cache.sql_cache_short.expiration_time',
-        30)
-    _int_setting(
-        settings,
-        'rc_cache.sql_cache_short.max_size',
-        10000)
-
-
-def _int_setting(settings, name, default):
-    settings[name] = int(settings.get(name, default))
-    return settings[name]
-
-
-def _bool_setting(settings, name, default):
-    input_val = settings.get(name, default)
-    if isinstance(input_val, unicode):
-        input_val = input_val.encode('utf8')
-    settings[name] = asbool(input_val)
-    return settings[name]
-
-
-def _list_setting(settings, name, default):
-    raw_value = settings.get(name, default)
-
-    old_separator = ','
-    if old_separator in raw_value:
-        # If we get a comma separated list, pass it to our own function.
-        settings[name] = rhodecode_aslist(raw_value, sep=old_separator)
-    else:
-        # Otherwise we assume it uses pyramids space/newline separation.
-        settings[name] = aslist(raw_value)
-    return settings[name]
-
-
-def _string_setting(settings, name, default, lower=True, default_when_empty=False):
-    value = settings.get(name, default)
-
-    if default_when_empty and not value:
-        # use default value when value is empty
-        value = default
-
-    if lower:
-        value = value.lower()
-    settings[name] = value
-    return settings[name]
+    settings_maker.make_setting('rc_cache.sql_cache_short.backend', 'dogpile.cache.rc.memory_lru')
+    settings_maker.make_setting('rc_cache.sql_cache_short.expiration_time', 30, parser='int')
+    settings_maker.make_setting('rc_cache.sql_cache_short.max_size', 10000, parser='int')

rhodecode/lib/base.py

@@ -1,627 +1,627 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 """
 The base Controller API
 Provides the BaseController class for subclassing. And usage in different
 controllers
 """
 
 import logging
 import socket
 
 import markupsafe
 import ipaddress
 
 from paste.auth.basic import AuthBasicAuthenticator
 from paste.httpexceptions import HTTPUnauthorized, HTTPForbidden, get_exception
 from paste.httpheaders import WWW_AUTHENTICATE, AUTHORIZATION
 
 import rhodecode
 from rhodecode.apps._base import TemplateArgs
 from rhodecode.authentication.base import VCS_TYPE
 from rhodecode.lib import auth, utils2
 from rhodecode.lib import helpers as h
 from rhodecode.lib.auth import AuthUser, CookieStoreWrapper
 from rhodecode.lib.exceptions import UserCreationError
 from rhodecode.lib.utils import (password_changed, get_enabled_hook_classes)
 from rhodecode.lib.utils2 import (
     str2bool, safe_unicode, AttributeDict, safe_int, sha1, aslist, safe_str)
 from rhodecode.model.db import Repository, User, ChangesetComment, UserBookmark
 from rhodecode.model.notification import NotificationModel
 from rhodecode.model.settings import VcsSettingsModel, SettingsModel
 
 log = logging.getLogger(__name__)
 
 
 def _filter_proxy(ip):
     """
     Passed in IP addresses in HEADERS can be in a special format of multiple
     ips. Those comma separated IPs are passed from various proxies in the
     chain of request processing. The left-most being the original client.
     We only care about the first IP which came from the org. client.
 
     :param ip: ip string from headers
     """
     if ',' in ip:
         _ips = ip.split(',')
         _first_ip = _ips[0].strip()
         log.debug('Got multiple IPs %s, using %s', ','.join(_ips), _first_ip)
         return _first_ip
     return ip
 
 
 def _filter_port(ip):
     """
     Removes a port from ip, there are 4 main cases to handle here.
     - ipv4 eg. 127.0.0.1
     - ipv6 eg. ::1
     - ipv4+port eg. 127.0.0.1:8080
     - ipv6+port eg. [::1]:8080
 
     :param ip:
     """
     def is_ipv6(ip_addr):
         if hasattr(socket, 'inet_pton'):
             try:
                 socket.inet_pton(socket.AF_INET6, ip_addr)
             except socket.error:
                 return False
         else:
             # fallback to ipaddress
             try:
                 ipaddress.IPv6Address(safe_unicode(ip_addr))
             except Exception:
                 return False
         return True
 
     if ':' not in ip:  # must be ipv4 pure ip
         return ip
 
     if '[' in ip and ']' in ip:  # ipv6 with port
         return ip.split(']')[0][1:].lower()
 
     # must be ipv6 or ipv4 with port
     if is_ipv6(ip):
         return ip
     else:
         ip, _port = ip.split(':')[:2]  # means ipv4+port
         return ip
 
 
 def get_ip_addr(environ):
     proxy_key = 'HTTP_X_REAL_IP'
     proxy_key2 = 'HTTP_X_FORWARDED_FOR'
     def_key = 'REMOTE_ADDR'
     _filters = lambda x: _filter_port(_filter_proxy(x))
 
     ip = environ.get(proxy_key)
     if ip:
         return _filters(ip)
 
     ip = environ.get(proxy_key2)
     if ip:
         return _filters(ip)
 
     ip = environ.get(def_key, '0.0.0.0')
     return _filters(ip)
 
 
 def get_server_ip_addr(environ, log_errors=True):
     hostname = environ.get('SERVER_NAME')
     try:
         return socket.gethostbyname(hostname)
     except Exception as e:
         if log_errors:
             # in some cases this lookup is not possible, and we don't want to
             # make it an exception in logs
             log.exception('Could not retrieve server ip address: %s', e)
         return hostname
 
 
 def get_server_port(environ):
     return environ.get('SERVER_PORT')
 
 
 def get_access_path(environ):
     path = environ.get('PATH_INFO')
     org_req = environ.get('pylons.original_request')
     if org_req:
         path = org_req.environ.get('PATH_INFO')
     return path
 
 
 def get_user_agent(environ):
     return environ.get('HTTP_USER_AGENT')
 
 
 def vcs_operation_context(
         environ, repo_name, username, action, scm, check_locking=True,
         is_shadow_repo=False, check_branch_perms=False, detect_force_push=False):
     """
     Generate the context for a vcs operation, e.g. push or pull.
 
     This context is passed over the layers so that hooks triggered by the
     vcs operation know details like the user, the user's IP address etc.
 
     :param check_locking: Allows to switch of the computation of the locking
         data. This serves mainly the need of the simplevcs middleware to be
         able to disable this for certain operations.
 
     """
     # Tri-state value: False: unlock, None: nothing, True: lock
     make_lock = None
     locked_by = [None, None, None]
     is_anonymous = username == User.DEFAULT_USER
     user = User.get_by_username(username)
     if not is_anonymous and check_locking:
         log.debug('Checking locking on repository "%s"', repo_name)
         repo = Repository.get_by_repo_name(repo_name)
         make_lock, __, locked_by = repo.get_locking_state(
             action, user.user_id)
     user_id = user.user_id
     settings_model = VcsSettingsModel(repo=repo_name)
     ui_settings = settings_model.get_ui_settings()
 
     # NOTE(marcink): This should be also in sync with
     # rhodecode/apps/ssh_support/lib/backends/base.py:update_environment scm_data
     store = [x for x in ui_settings if x.key == '/']
     repo_store = ''
     if store:
         repo_store = store[0].value
 
     scm_data = {
         'ip': get_ip_addr(environ),
         'username': username,
         'user_id': user_id,
         'action': action,
         'repository': repo_name,
         'scm': scm,
         'config': rhodecode.CONFIG['__file__'],
         'repo_store': repo_store,
         'make_lock': make_lock,
         'locked_by': locked_by,
         'server_url': utils2.get_server_url(environ),
         'user_agent': get_user_agent(environ),
         'hooks': get_enabled_hook_classes(ui_settings),
         'is_shadow_repo': is_shadow_repo,
         'detect_force_push': detect_force_push,
         'check_branch_perms': check_branch_perms,
     }
     return scm_data
 
 
 class BasicAuth(AuthBasicAuthenticator):
 
     def __init__(self, realm, authfunc, registry, auth_http_code=None,
                  initial_call_detection=False, acl_repo_name=None, rc_realm=''):
         self.realm = realm
         self.rc_realm = rc_realm
         self.initial_call = initial_call_detection
         self.authfunc = authfunc
         self.registry = registry
         self.acl_repo_name = acl_repo_name
         self._rc_auth_http_code = auth_http_code
 
     def _get_response_from_code(self, http_code):
         try:
             return get_exception(safe_int(http_code))
         except Exception:
             log.exception('Failed to fetch response for code %s', http_code)
             return HTTPForbidden
 
     def get_rc_realm(self):
         return safe_str(self.rc_realm)
 
     def build_authentication(self):
         head = WWW_AUTHENTICATE.tuples('Basic realm="%s"' % self.realm)
         if self._rc_auth_http_code and not self.initial_call:
             # return alternative HTTP code if alternative http return code
             # is specified in RhodeCode config, but ONLY if it's not the
             # FIRST call
             custom_response_klass = self._get_response_from_code(
                 self._rc_auth_http_code)
             return custom_response_klass(headers=head)
         return HTTPUnauthorized(headers=head)
 
     def authenticate(self, environ):
         authorization = AUTHORIZATION(environ)
         if not authorization:
             return self.build_authentication()
         (authmeth, auth) = authorization.split(' ', 1)
         if 'basic' != authmeth.lower():
             return self.build_authentication()
         auth = auth.strip().decode('base64')
         _parts = auth.split(':', 1)
         if len(_parts) == 2:
             username, password = _parts
             auth_data = self.authfunc(
                     username, password, environ, VCS_TYPE,
                     registry=self.registry, acl_repo_name=self.acl_repo_name)
             if auth_data:
                 return {'username': username, 'auth_data': auth_data}
             if username and password:
                 # we mark that we actually executed authentication once, at
                 # that point we can use the alternative auth code
                 self.initial_call = False
 
         return self.build_authentication()
 
     __call__ = authenticate
 
 
 def calculate_version_hash(config):
     return sha1(
         config.get('beaker.session.secret', '') +
         rhodecode.__version__)[:8]
 
 
 def get_current_lang(request):
     # NOTE(marcink): remove after pyramid move
     try:
         return translation.get_lang()[0]
     except:
         pass
 
     return getattr(request, '_LOCALE_', request.locale_name)
 
 
 def attach_context_attributes(context, request, user_id=None, is_api=None):
     """
     Attach variables into template context called `c`.
     """
     config = request.registry.settings
 
     rc_config = SettingsModel().get_all_settings(cache=True, from_request=False)
     context.rc_config = rc_config
     context.rhodecode_version = rhodecode.__version__
     context.rhodecode_edition = config.get('rhodecode.edition')
     context.rhodecode_edition_id = config.get('rhodecode.edition_id')
     # unique secret + version does not leak the version but keep consistency
     context.rhodecode_version_hash = calculate_version_hash(config)
 
     # Default language set for the incoming request
     context.language = get_current_lang(request)
 
     # Visual options
     context.visual = AttributeDict({})
 
     # DB stored Visual Items
     context.visual.show_public_icon = str2bool(
         rc_config.get('rhodecode_show_public_icon'))
     context.visual.show_private_icon = str2bool(
         rc_config.get('rhodecode_show_private_icon'))
     context.visual.stylify_metatags = str2bool(
         rc_config.get('rhodecode_stylify_metatags'))
     context.visual.dashboard_items = safe_int(
         rc_config.get('rhodecode_dashboard_items', 100))
     context.visual.admin_grid_items = safe_int(
         rc_config.get('rhodecode_admin_grid_items', 100))
     context.visual.show_revision_number = str2bool(
         rc_config.get('rhodecode_show_revision_number', True))
     context.visual.show_sha_length = safe_int(
         rc_config.get('rhodecode_show_sha_length', 100))
     context.visual.repository_fields = str2bool(
         rc_config.get('rhodecode_repository_fields'))
     context.visual.show_version = str2bool(
         rc_config.get('rhodecode_show_version'))
     context.visual.use_gravatar = str2bool(
         rc_config.get('rhodecode_use_gravatar'))
     context.visual.gravatar_url = rc_config.get('rhodecode_gravatar_url')
     context.visual.default_renderer = rc_config.get(
         'rhodecode_markup_renderer', 'rst')
     context.visual.comment_types = ChangesetComment.COMMENT_TYPES
     context.visual.rhodecode_support_url = \
         rc_config.get('rhodecode_support_url') or h.route_url('rhodecode_support')
 
     context.visual.affected_files_cut_off = 60
 
     context.pre_code = rc_config.get('rhodecode_pre_code')
     context.post_code = rc_config.get('rhodecode_post_code')
     context.rhodecode_name = rc_config.get('rhodecode_title')
     context.default_encodings = aslist(config.get('default_encoding'), sep=',')
     # if we have specified default_encoding in the request, it has more
     # priority
     if request.GET.get('default_encoding'):
         context.default_encodings.insert(0, request.GET.get('default_encoding'))
     context.clone_uri_tmpl = rc_config.get('rhodecode_clone_uri_tmpl')
     context.clone_uri_id_tmpl = rc_config.get('rhodecode_clone_uri_id_tmpl')
     context.clone_uri_ssh_tmpl = rc_config.get('rhodecode_clone_uri_ssh_tmpl')
 
     # INI stored
     context.labs_active = str2bool(
         config.get('labs_settings_active', 'false'))
     context.ssh_enabled = str2bool(
         config.get('ssh.generate_authorized_keyfile', 'false'))
     context.ssh_key_generator_enabled = str2bool(
         config.get('ssh.enable_ui_key_generator', 'true'))
 
     context.visual.allow_repo_location_change = str2bool(
         config.get('allow_repo_location_change', True))
     context.visual.allow_custom_hooks_settings = str2bool(
         config.get('allow_custom_hooks_settings', True))
     context.debug_style = str2bool(config.get('debug_style', False))
 
     context.rhodecode_instanceid = config.get('instance_id')
 
     context.visual.cut_off_limit_diff = safe_int(
         config.get('cut_off_limit_diff'))
     context.visual.cut_off_limit_file = safe_int(
         config.get('cut_off_limit_file'))
 
     context.license = AttributeDict({})
     context.license.hide_license_info = str2bool(
         config.get('license.hide_license_info', False))
 
     # AppEnlight
-    context.appenlight_enabled = str2bool(config.get('appenlight', 'false'))
+    context.appenlight_enabled = config.get('appenlight', False)
     context.appenlight_api_public_key = config.get(
         'appenlight.api_public_key', '')
     context.appenlight_server_url = config.get('appenlight.server_url', '')
 
     diffmode = {
         "unified": "unified",
         "sideside": "sideside"
     }.get(request.GET.get('diffmode'))
 
     if is_api is not None:
         is_api = hasattr(request, 'rpc_user')
     session_attrs = {
         # defaults
         "clone_url_format": "http",
         "diffmode": "sideside",
         "license_fingerprint": request.session.get('license_fingerprint')
     }
 
     if not is_api:
         # don't access pyramid session for API calls
         if diffmode and diffmode != request.session.get('rc_user_session_attr.diffmode'):
             request.session['rc_user_session_attr.diffmode'] = diffmode
 
         # session settings per user
 
         for k, v in request.session.items():
             pref = 'rc_user_session_attr.'
             if k and k.startswith(pref):
                 k = k[len(pref):]
                 session_attrs[k] = v
 
     context.user_session_attrs = session_attrs
 
     # JS template context
     context.template_context = {
         'repo_name': None,
         'repo_type': None,
         'repo_landing_commit': None,
         'rhodecode_user': {
             'username': None,
             'email': None,
             'notification_status': False
         },
         'session_attrs': session_attrs,
         'visual': {
             'default_renderer': None
         },
         'commit_data': {
             'commit_id': None
         },
         'pull_request_data': {'pull_request_id': None},
         'timeago': {
             'refresh_time': 120 * 1000,
             'cutoff_limit': 1000 * 60 * 60 * 24 * 7
         },
         'pyramid_dispatch': {
 
         },
         'extra': {'plugins': {}}
     }
     # END CONFIG VARS
     if is_api:
         csrf_token = None
     else:
         csrf_token = auth.get_csrf_token(session=request.session)
 
     context.csrf_token = csrf_token
     context.backends = rhodecode.BACKENDS.keys()
 
     unread_count = 0
     user_bookmark_list = []
     if user_id:
         unread_count = NotificationModel().get_unread_cnt_for_user(user_id)
         user_bookmark_list = UserBookmark.get_bookmarks_for_user(user_id)
     context.unread_notifications = unread_count
     context.bookmark_items = user_bookmark_list
 
     # web case
     if hasattr(request, 'user'):
         context.auth_user = request.user
         context.rhodecode_user = request.user
 
     # api case
     if hasattr(request, 'rpc_user'):
         context.auth_user = request.rpc_user
         context.rhodecode_user = request.rpc_user
 
     # attach the whole call context to the request
     request.call_context = context
 
 
 def get_auth_user(request):
     environ = request.environ
     session = request.session
 
     ip_addr = get_ip_addr(environ)
 
     # make sure that we update permissions each time we call controller
     _auth_token = (
             # ?auth_token=XXX
             request.GET.get('auth_token', '')
             # ?api_key=XXX !LEGACY
             or request.GET.get('api_key', '')
             # or headers....
             or request.headers.get('X-Rc-Auth-Token', '')
     )
     if not _auth_token and request.matchdict:
         url_auth_token = request.matchdict.get('_auth_token')
         _auth_token = url_auth_token
         if _auth_token:
             log.debug('Using URL extracted auth token `...%s`', _auth_token[-4:])
 
     if _auth_token:
         # when using API_KEY we assume user exists, and
         # doesn't need auth based on cookies.
         auth_user = AuthUser(api_key=_auth_token, ip_addr=ip_addr)
         authenticated = False
     else:
         cookie_store = CookieStoreWrapper(session.get('rhodecode_user'))
         try:
             auth_user = AuthUser(user_id=cookie_store.get('user_id', None),
                                  ip_addr=ip_addr)
         except UserCreationError as e:
             h.flash(e, 'error')
             # container auth or other auth functions that create users
             # on the fly can throw this exception signaling that there's
             # issue with user creation, explanation should be provided
             # in Exception itself. We then create a simple blank
             # AuthUser
             auth_user = AuthUser(ip_addr=ip_addr)
 
         # in case someone changes a password for user it triggers session
         # flush and forces a re-login
         if password_changed(auth_user, session):
             session.invalidate()
             cookie_store = CookieStoreWrapper(session.get('rhodecode_user'))
             auth_user = AuthUser(ip_addr=ip_addr)
 
         authenticated = cookie_store.get('is_authenticated')
 
     if not auth_user.is_authenticated and auth_user.is_user_object:
         # user is not authenticated and not empty
         auth_user.set_authenticated(authenticated)
 
     return auth_user, _auth_token
 
 
 def h_filter(s):
     """
     Custom filter for Mako templates. Mako by standard uses `markupsafe.escape`
     we wrap this with additional functionality that converts None to empty
     strings
     """
     if s is None:
         return markupsafe.Markup()
     return markupsafe.escape(s)
 
 
 def add_events_routes(config):
     """
     Adds routing that can be used in events. Because some events are triggered
     outside of pyramid context, we need to bootstrap request with some
     routing registered
     """
 
     from rhodecode.apps._base import ADMIN_PREFIX
 
     config.add_route(name='home', pattern='/')
     config.add_route(name='main_page_repos_data', pattern='/_home_repos')
     config.add_route(name='main_page_repo_groups_data', pattern='/_home_repo_groups')
 
     config.add_route(name='login', pattern=ADMIN_PREFIX + '/login')
     config.add_route(name='logout', pattern=ADMIN_PREFIX + '/logout')
     config.add_route(name='repo_summary', pattern='/{repo_name}')
     config.add_route(name='repo_summary_explicit', pattern='/{repo_name}/summary')
     config.add_route(name='repo_group_home', pattern='/{repo_group_name}')
 
     config.add_route(name='pullrequest_show',
                      pattern='/{repo_name}/pull-request/{pull_request_id}')
     config.add_route(name='pull_requests_global',
                      pattern='/pull-request/{pull_request_id}')
 
     config.add_route(name='repo_commit',
                      pattern='/{repo_name}/changeset/{commit_id}')
     config.add_route(name='repo_files',
                      pattern='/{repo_name}/files/{commit_id}/{f_path}')
 
     config.add_route(name='hovercard_user',
                      pattern='/_hovercard/user/{user_id}')
 
     config.add_route(name='hovercard_user_group',
                      pattern='/_hovercard/user_group/{user_group_id}')
 
     config.add_route(name='hovercard_pull_request',
                      pattern='/_hovercard/pull_request/{pull_request_id}')
 
     config.add_route(name='hovercard_repo_commit',
                      pattern='/_hovercard/commit/{repo_name}/{commit_id}')
 
 
 def bootstrap_config(request):
     import pyramid.testing
     registry = pyramid.testing.Registry('RcTestRegistry')
 
     config = pyramid.testing.setUp(registry=registry, request=request)
 
     # allow pyramid lookup in testing
     config.include('pyramid_mako')
     config.include('rhodecode.lib.rc_beaker')
     config.include('rhodecode.lib.rc_cache')
 
     add_events_routes(config)
 
     return config
 
 
 def bootstrap_request(**kwargs):
     import pyramid.testing
 
     class TestRequest(pyramid.testing.DummyRequest):
         application_url = kwargs.pop('application_url', 'http://example.com')
         host = kwargs.pop('host', 'example.com:80')
         domain = kwargs.pop('domain', 'example.com')
 
         def translate(self, msg):
             return msg
 
         def plularize(self, singular, plural, n):
             return singular
 
         def get_partial_renderer(self, tmpl_name):
 
             from rhodecode.lib.partial_renderer import get_partial_renderer
             return get_partial_renderer(request=self, tmpl_name=tmpl_name)
 
         _call_context = TemplateArgs()
         _call_context.visual = TemplateArgs()
         _call_context.visual.show_sha_length = 12
         _call_context.visual.show_revision_number = True
 
         @property
         def call_context(self):
             return self._call_context
 
     class TestDummySession(pyramid.testing.DummySession):
         def save(*arg, **kw):
             pass
 
     request = TestRequest(**kwargs)
     request.session = TestDummySession()
 
     return request
 

rhodecode/lib/celerylib/loader.py

@@ -1,312 +1,313 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 """
 Celery loader, run with::
 
     celery worker \
         --beat \
         --app rhodecode.lib.celerylib.loader \
         --scheduler rhodecode.lib.celerylib.scheduler.RcScheduler \
         --loglevel DEBUG --ini=._dev/dev.ini
 """
 import os
 import logging
 import importlib
 
 from celery import Celery
 from celery import signals
 from celery import Task
 from celery import exceptions  # pragma: no cover
 from kombu.serialization import register
 from pyramid.threadlocal import get_current_request
 
 import rhodecode
 
 from rhodecode.lib.auth import AuthUser
-from rhodecode.lib.celerylib.utils import get_ini_config, parse_ini_vars, ping_db
+from rhodecode.lib.celerylib.utils import parse_ini_vars, ping_db
 from rhodecode.lib.ext_json import json
 from rhodecode.lib.pyramid_utils import bootstrap, setup_logging, prepare_request
 from rhodecode.lib.utils2 import str2bool
 from rhodecode.model import meta
 
 
 register('json_ext', json.dumps, json.loads,
          content_type='application/x-json-ext',
          content_encoding='utf-8')
 
 log = logging.getLogger('celery.rhodecode.loader')
 
 
 def add_preload_arguments(parser):
     parser.add_argument(
         '--ini', default=None,
         help='Path to ini configuration file.'
     )
     parser.add_argument(
         '--ini-var', default=None,
         help='Comma separated list of key=value to pass to ini.'
     )
 
 
 def get_logger(obj):
     custom_log = logging.getLogger(
         'rhodecode.task.{}'.format(obj.__class__.__name__))
 
     if rhodecode.CELERY_ENABLED:
         try:
             custom_log = obj.get_logger()
         except Exception:
             pass
 
     return custom_log
 
 
 imports = ['rhodecode.lib.celerylib.tasks']
 
 try:
     # try if we have EE tasks available
     importlib.import_module('rc_ee')
     imports.append('rc_ee.lib.celerylib.tasks')
 except ImportError:
     pass
 
 
 base_celery_config = {
     'result_backend': 'rpc://',
     'result_expires': 60 * 60 * 24,
     'result_persistent': True,
     'imports': imports,
     'worker_max_tasks_per_child': 100,
     'accept_content': ['json_ext'],
     'task_serializer': 'json_ext',
     'result_serializer': 'json_ext',
     'worker_hijack_root_logger': False,
     'database_table_names': {
         'task': 'beat_taskmeta',
         'group': 'beat_groupmeta',
     }
 }
 # init main celery app
 celery_app = Celery()
 celery_app.user_options['preload'].add(add_preload_arguments)
 ini_file_glob = None
 
 
 @signals.setup_logging.connect
 def setup_logging_callback(**kwargs):
     setup_logging(ini_file_glob)
 
 
 @signals.user_preload_options.connect
 def on_preload_parsed(options, **kwargs):
+    from rhodecode.config.middleware import get_celery_config
+
     ini_location = options['ini']
     ini_vars = options['ini_var']
     celery_app.conf['INI_PYRAMID'] = options['ini']
 
     if ini_location is None:
         print('You must provide the paste --ini argument')
         exit(-1)
 
     options = None
     if ini_vars is not None:
         options = parse_ini_vars(ini_vars)
 
     global ini_file_glob
     ini_file_glob = ini_location
 
     log.debug('Bootstrapping RhodeCode application...')
     env = bootstrap(ini_location, options=options)
 
+    celery_settings = get_celery_config(env['registry'].settings)
     setup_celery_app(
         app=env['app'], root=env['root'], request=env['request'],
         registry=env['registry'], closer=env['closer'],
-        ini_location=ini_location)
+        celery_settings=celery_settings)
 
     # fix the global flag even if it's disabled via .ini file because this
     # is a worker code that doesn't need this to be disabled.
     rhodecode.CELERY_ENABLED = True
 
 
 @signals.task_prerun.connect
 def task_prerun_signal(task_id, task, args, **kwargs):
     ping_db()
 
 
 @signals.task_success.connect
 def task_success_signal(result, **kwargs):
     meta.Session.commit()
     closer = celery_app.conf['PYRAMID_CLOSER']
     if closer:
         closer()
 
 
 @signals.task_retry.connect
 def task_retry_signal(
         request, reason, einfo, **kwargs):
     meta.Session.remove()
     closer = celery_app.conf['PYRAMID_CLOSER']
     if closer:
         closer()
 
 
 @signals.task_failure.connect
 def task_failure_signal(
         task_id, exception, args, kwargs, traceback, einfo, **kargs):
     from rhodecode.lib.exc_tracking import store_exception
     from rhodecode.lib.statsd_client import StatsdClient
 
     meta.Session.remove()
 
     # simulate sys.exc_info()
     exc_info = (einfo.type, einfo.exception, einfo.tb)
     store_exception(id(exc_info), exc_info, prefix='rhodecode-celery')
     statsd = StatsdClient.statsd
     if statsd:
         exc_type = "{}.{}".format(einfo.__class__.__module__, einfo.__class__.__name__)
         statsd.incr('rhodecode_exception_total',
                     tags=["exc_source:celery", "type:{}".format(exc_type)])
 
     closer = celery_app.conf['PYRAMID_CLOSER']
     if closer:
         closer()
 
 
 @signals.task_revoked.connect
 def task_revoked_signal(
         request, terminated, signum, expired, **kwargs):
     closer = celery_app.conf['PYRAMID_CLOSER']
     if closer:
         closer()
 
 
-def setup_celery_app(app, root, request, registry, closer, ini_location):
-    ini_dir = os.path.dirname(os.path.abspath(ini_location))
+def setup_celery_app(app, root, request, registry, closer, celery_settings):
+    log.debug('Got custom celery conf: %s', celery_settings)
     celery_config = base_celery_config
     celery_config.update({
         # store celerybeat scheduler db where the .ini file is
-        'beat_schedule_filename': os.path.join(ini_dir, 'celerybeat-schedule'),
+        'beat_schedule_filename': registry.settings['celerybeat-schedule.path'],
     })
-    ini_settings = get_ini_config(ini_location)
-    log.debug('Got custom celery conf: %s', ini_settings)
 
-    celery_config.update(ini_settings)
+    celery_config.update(celery_settings)
     celery_app.config_from_object(celery_config)
 
     celery_app.conf.update({'PYRAMID_APP': app})
     celery_app.conf.update({'PYRAMID_ROOT': root})
     celery_app.conf.update({'PYRAMID_REQUEST': request})
     celery_app.conf.update({'PYRAMID_REGISTRY': registry})
     celery_app.conf.update({'PYRAMID_CLOSER': closer})
 
 
-def configure_celery(config, ini_location):
+def configure_celery(config, celery_settings):
     """
     Helper that is called from our application creation logic. It gives
     connection info into running webapp and allows execution of tasks from
     RhodeCode itself
     """
     # store some globals into rhodecode
     rhodecode.CELERY_ENABLED = str2bool(
         config.registry.settings.get('use_celery'))
     if rhodecode.CELERY_ENABLED:
-        log.info('Configuring celery based on `%s` file', ini_location)
+        log.info('Configuring celery based on `%s` settings', celery_settings)
         setup_celery_app(
             app=None, root=None, request=None, registry=config.registry,
-            closer=None, ini_location=ini_location)
+            closer=None, celery_settings=celery_settings)
 
 
 def maybe_prepare_env(req):
     environ = {}
     try:
         environ.update({
             'PATH_INFO': req.environ['PATH_INFO'],
             'SCRIPT_NAME': req.environ['SCRIPT_NAME'],
-            'HTTP_HOST':req.environ.get('HTTP_HOST', req.environ['SERVER_NAME']),
+            'HTTP_HOST': req.environ.get('HTTP_HOST', req.environ['SERVER_NAME']),
             'SERVER_NAME': req.environ['SERVER_NAME'],
             'SERVER_PORT': req.environ['SERVER_PORT'],
             'wsgi.url_scheme': req.environ['wsgi.url_scheme'],
         })
     except Exception:
         pass
 
     return environ
 
 
 class RequestContextTask(Task):
     """
     This is a celery task which will create a rhodecode app instance context
     for the task, patch pyramid with the original request
     that created the task and also add the user to the context.
     """
 
     def apply_async(self, args=None, kwargs=None, task_id=None, producer=None,
                     link=None, link_error=None, shadow=None, **options):
         """ queue the job to run (we are in web request context here) """
 
         req = get_current_request()
 
         # web case
         if hasattr(req, 'user'):
             ip_addr = req.user.ip_addr
             user_id = req.user.user_id
 
         # api case
         elif hasattr(req, 'rpc_user'):
             ip_addr = req.rpc_user.ip_addr
             user_id = req.rpc_user.user_id
         else:
             raise Exception(
                 'Unable to fetch required data from request: {}. \n'
                 'This task is required to be executed from context of '
                 'request in a webapp'.format(repr(req)))
 
         if req:
             # we hook into kwargs since it is the only way to pass our data to
             # the celery worker
             environ = maybe_prepare_env(req)
             options['headers'] = options.get('headers', {})
             options['headers'].update({
                 'rhodecode_proxy_data': {
                     'environ': environ,
                     'auth_user': {
                         'ip_addr': ip_addr,
                         'user_id': user_id
                     },
                 }
             })
 
         return super(RequestContextTask, self).apply_async(
             args, kwargs, task_id, producer, link, link_error, shadow, **options)
 
     def __call__(self, *args, **kwargs):
         """ rebuild the context and then run task on celery worker """
 
         proxy_data = getattr(self.request, 'rhodecode_proxy_data', None)
         if not proxy_data:
             return super(RequestContextTask, self).__call__(*args, **kwargs)
 
         log.debug('using celery proxy data to run task: %r', proxy_data)
         # re-inject and register threadlocals for proper routing support
         request = prepare_request(proxy_data['environ'])
         request.user = AuthUser(user_id=proxy_data['auth_user']['user_id'],
                                 ip_addr=proxy_data['auth_user']['ip_addr'])
 
         return super(RequestContextTask, self).__call__(*args, **kwargs)
 

rhodecode/lib/celerylib/utils.py

@@ -1,187 +1,141 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2010-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 import os
 import json
 import logging
 import datetime
 import time
 
 from functools import partial
 
 from pyramid.compat import configparser
 from celery.result import AsyncResult
 import celery.loaders.base
 import celery.schedules
 
 log = logging.getLogger(__name__)
 
 
 def get_task_id(task):
     task_id = None
     if isinstance(task, AsyncResult):
         task_id = task.task_id
 
     return task_id
 
 
 def crontab(value):
     return celery.schedules.crontab(**value)
 
 
 def timedelta(value):
     return datetime.timedelta(**value)
 
 
 def safe_json(get, section, key):
     value = ''
     try:
         value = get(key)
         json_value = json.loads(value)
     except ValueError:
         msg = 'The %s=%s is not valid json in section %s' % (
             key, value, section
         )
         raise ValueError(msg)
 
     return json_value
 
 
 def raw_2_schedule(schedule_value, schedule_type):
     schedule_type_map = {
         'crontab': crontab,
         'timedelta': timedelta,
         'integer': int
     }
     scheduler_cls = schedule_type_map.get(schedule_type)
 
     if scheduler_cls is None:
         raise ValueError(
             'schedule type %s in section is invalid' % (
                 schedule_type,
             )
         )
     try:
         schedule = scheduler_cls(schedule_value)
     except TypeError:
         log.exception('Failed to compose a schedule from value: %r', schedule_value)
         schedule = None
     return schedule
 
 
 def get_beat_config(parser, section):
 
     get = partial(parser.get, section)
     has_option = partial(parser.has_option, section)
 
     schedule_type = get('type')
     schedule_value = safe_json(get, section, 'schedule')
 
     config = {
         'schedule_type': schedule_type,
         'schedule_value': schedule_value,
         'task': get('task'),
     }
     schedule = raw_2_schedule(schedule_value, schedule_type)
     if schedule:
         config['schedule'] = schedule
 
     if has_option('args'):
         config['args'] = safe_json(get, section, 'args')
 
     if has_option('kwargs'):
         config['kwargs'] = safe_json(get, section, 'kwargs')
 
     if has_option('force_update'):
         config['force_update'] = get('force_update')
 
     return config
 
 
-def get_ini_config(ini_location):
-    """
-    Converts basic ini configuration into celery 4.X options
-    """
-    def key_converter(key_name):
-        pref = 'celery.'
-        if key_name.startswith(pref):
-            return key_name[len(pref):].replace('.', '_').lower()
-
-    def type_converter(parsed_key, value):
-        # cast to int
-        if value.isdigit():
-            return int(value)
-
-        # cast to bool
-        if value.lower() in ['true', 'false', 'True', 'False']:
-            return value.lower() == 'true'
-        return value
-
-    parser = configparser.SafeConfigParser(
-        defaults={'here': os.path.abspath(ini_location)})
-    parser.read(ini_location)
-
-    ini_config = {}
-    for k, v in parser.items('app:main'):
-        pref = 'celery.'
-        if k.startswith(pref):
-            ini_config[key_converter(k)] = type_converter(key_converter(k), v)
-
-    beat_config = {}
-    for section in parser.sections():
-        if section.startswith('celerybeat:'):
-            name = section.split(':', 1)[1]
-            beat_config[name] = get_beat_config(parser, section)
-
-    # final compose of settings
-    celery_settings = {}
-
-    if ini_config:
-        celery_settings.update(ini_config)
-    if beat_config:
-        celery_settings.update({'beat_schedule': beat_config})
-
-    return celery_settings
-
-
 def parse_ini_vars(ini_vars):
     options = {}
     for pairs in ini_vars.split(','):
         key, value = pairs.split('=')
         options[key] = value
     return options
 
 
 def ping_db():
     from rhodecode.model import meta
     from rhodecode.model.db import DbMigrateVersion
     log.info('Testing DB connection...')
 
     for test in range(10):
         try:
             scalar = DbMigrateVersion.query().scalar()
             log.debug('DB PING %s@%s', scalar, scalar.version)
             break
         except Exception:
             retry = 1
             log.debug('DB not ready, next try in %ss', retry)
             time.sleep(retry)
         finally:
             meta.Session.remove()

rhodecode/tests/config/test_sanitize_settings.py

@@ -1,203 +1,205 b''
 # -*- coding: utf-8 -*-
 
 # Copyright (C) 2016-2020 RhodeCode GmbH
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License, version 3
 # (only), as published by the Free Software Foundation.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
 # This program is dual-licensed. If you wish to learn more about the
 # RhodeCode Enterprise Edition, including its added features, Support services,
 # and proprietary license terms, please see https://rhodecode.com/licenses/
 
 
 import pytest
 
 from rhodecode.tests import no_newline_id_generator
-from rhodecode.config.middleware import (
-    _sanitize_vcs_settings, _bool_setting, _string_setting, _list_setting,
-    _int_setting)
+from rhodecode.config.middleware import sanitize_settings_and_apply_defaults
+from rhodecode.config.settings_maker import SettingsMaker
 
 
 class TestHelperFunctions(object):
     @pytest.mark.parametrize('raw, expected', [
         ('true', True), (u'true', True),
         ('yes', True), (u'yes', True),
         ('on', True), (u'on', True),
         ('false', False), (u'false', False),
         ('no', False), (u'no', False),
         ('off', False), (u'off', False),
         ('invalid-bool-value', False),
         ('invalid-∫øø@-√å@¨€', False),
         (u'invalid-∫øø@-√å@¨€', False),
     ])
-    def test_bool_setting_helper(self, raw, expected):
-        key = 'dummy-key'
-        settings = {key: raw}
-        _bool_setting(settings, key, None)
-        assert settings[key] is expected
+    def test_bool_func_helper(self, raw, expected):
+        val = SettingsMaker._bool_func(raw)
+        assert val == expected
 
     @pytest.mark.parametrize('raw, expected', [
         ('', ''),
         ('test-string', 'test-string'),
         ('CaSe-TeSt', 'case-test'),
         ('test-string-烩€', 'test-string-烩€'),
         (u'test-string-烩€', u'test-string-烩€'),
     ])
-    def test_string_setting_helper(self, raw, expected):
-        key = 'dummy-key'
-        settings = {key: raw}
-        _string_setting(settings, key, None)
-        assert settings[key] == expected
+    def test_string_func_helper(self, raw, expected):
+        val = SettingsMaker._string_func(raw)
+        assert val == expected
 
     @pytest.mark.parametrize('raw, expected', [
         ('', []),
         ('test', ['test']),
         ('CaSe-TeSt', ['CaSe-TeSt']),
         ('test-string-烩€', ['test-string-烩€']),
         (u'test-string-烩€', [u'test-string-烩€']),
-        ('hg git svn', ['hg', 'git', 'svn']),
         ('hg,git,svn', ['hg', 'git', 'svn']),
         ('hg, git, svn', ['hg', 'git', 'svn']),
-        ('hg\ngit\nsvn', ['hg', 'git', 'svn']),
-        (' hg\n git\n svn ', ['hg', 'git', 'svn']),
+
         (', hg , git , svn , ', ['', 'hg', 'git', 'svn', '']),
         ('cheese,free node,other', ['cheese', 'free node', 'other']),
     ], ids=no_newline_id_generator)
     def test_list_setting_helper(self, raw, expected):
-        key = 'dummy-key'
-        settings = {key: raw}
-        _list_setting(settings, key, None)
-        assert settings[key] == expected
+        val = SettingsMaker._list_func(raw)
+        assert val == expected
+
+    @pytest.mark.parametrize('raw, expected', [
+        ('hg git svn', ['hg', 'git', 'svn']),
+    ], ids=no_newline_id_generator)
+    def test_list_setting_spaces_helper(self, raw, expected):
+        val = SettingsMaker._list_func(raw, sep=' ')
+        assert val == expected
+
+    @pytest.mark.parametrize('raw, expected', [
+        ('hg\ngit\nsvn', ['hg', 'git', 'svn']),
+        (' hg\n git\n svn ', ['hg', 'git', 'svn']),
+    ], ids=no_newline_id_generator)
+    def test_list_setting_newlines_helper(self, raw, expected):
+        val = SettingsMaker._list_func(raw, sep='\n')
+        assert val == expected
 
     @pytest.mark.parametrize('raw, expected', [
         ('0', 0),
         ('-0', 0),
         ('12345', 12345),
         ('-12345', -12345),
         (u'-12345', -12345),
     ])
     def test_int_setting_helper(self, raw, expected):
-        key = 'dummy-key'
-        settings = {key: raw}
-        _int_setting(settings, key, None)
-        assert settings[key] == expected
+        val = SettingsMaker._int_func(raw)
+        assert val == expected
 
     @pytest.mark.parametrize('raw', [
         ('0xff'),
         (''),
         ('invalid-int'),
         ('invalid-⁄~†'),
         (u'invalid-⁄~†'),
     ])
     def test_int_setting_helper_invalid_input(self, raw):
-        key = 'dummy-key'
-        settings = {key: raw}
         with pytest.raises(Exception):
-            _int_setting(settings, key, None)
+            SettingsMaker._int_func(raw)
 
 
 class TestSanitizeVcsSettings(object):
-    _bool_settings = [
+    _bool_funcs = [
         ('vcs.hooks.direct_calls', False),
         ('vcs.server.enable', True),
         ('vcs.start_server', False),
         ('startup.import_repos', False),
     ]
 
-    _string_settings = [
+    _string_funcs = [
         ('vcs.svn.compatible_version', ''),
         ('vcs.hooks.protocol', 'http'),
         ('vcs.hooks.host', '127.0.0.1'),
         ('vcs.scm_app_implementation', 'http'),
         ('vcs.server', ''),
         ('vcs.server.protocol', 'http'),
     ]
 
     _list_settings = [
         ('vcs.backends', 'hg git'),
     ]
 
-    @pytest.mark.parametrize('key, default', _list_settings)
-    def test_list_setting_spacesep_list(self, key, default):
-        test_list = ['test', 'list', 'values', 'for', key]
-        input_value = ' '.join(test_list)
-        settings = {key: input_value}
-        _sanitize_vcs_settings(settings)
-        assert settings[key] == test_list
-
-    @pytest.mark.parametrize('key, default', _list_settings)
-    def test_list_setting_newlinesep_list(self, key, default):
-        test_list = ['test', 'list', 'values', 'for', key]
-        input_value = '\n'.join(test_list)
-        settings = {key: input_value}
-        _sanitize_vcs_settings(settings)
-        assert settings[key] == test_list
+    # @pytest.mark.parametrize('key, default', _list_settings)
+    # def test_list_setting_spacesep_list(self, key, default):
+    #     test_list = ['test', 'list', 'values', 'for', key]
+    #     input_value = ' '.join(test_list)
+    #     settings = {key: input_value}
+    #     sanitize_settings_and_apply_defaults({'__file__': ''}, settings)
+    #     assert settings[key] == test_list
+    #
+    # @pytest.mark.parametrize('key, default', _list_settings)
+    # def test_list_setting_newlinesep_list(self, key, default):
+    #     test_list = ['test', 'list', 'values', 'for', key]
+    #     input_value = '\n'.join(test_list)
+    #     settings = {key: input_value}
+    #     sanitize_settings_and_apply_defaults({'__file__': ''}, settings)
+    #     assert settings[key] == test_list
 
     @pytest.mark.parametrize('key, default', _list_settings)
     def test_list_setting_commasep_list(self, key, default):
         test_list = ['test', 'list', 'values', 'for', key]
         input_value = ','.join(test_list)
         settings = {key: input_value}
-        _sanitize_vcs_settings(settings)
+        sanitize_settings_and_apply_defaults({'__file__': ''}, settings)
         assert settings[key] == test_list
 
     @pytest.mark.parametrize('key, default', _list_settings)
     def test_list_setting_comma_and_space_sep_list(self, key, default):
         test_list = ['test', 'list', 'values', 'for', key]
         input_value = ', '.join(test_list)
         settings = {key: input_value}
-        _sanitize_vcs_settings(settings)
+        sanitize_settings_and_apply_defaults({'__file__': ''}, settings)
         assert settings[key] == test_list
 
-    @pytest.mark.parametrize('key, default', _string_settings)
-    def test_string_setting_string(self, key, default):
+    @pytest.mark.parametrize('key, default', _string_funcs)
+    def test_string_func_string(self, key, default):
         test_value = 'test-string-for-{}'.format(key)
         settings = {key: test_value}
-        _sanitize_vcs_settings(settings)
+        sanitize_settings_and_apply_defaults({'__file__': ''}, settings)
         assert settings[key] == test_value
 
-    @pytest.mark.parametrize('key, default', _string_settings)
-    def test_string_setting_default(self, key, default):
+    @pytest.mark.parametrize('key, default', _string_funcs)
+    def test_string_func_default(self, key, default):
         settings = {}
-        _sanitize_vcs_settings(settings)
+        sanitize_settings_and_apply_defaults({'__file__': ''}, settings)
         assert settings[key] == default
 
-    @pytest.mark.parametrize('key, default', _string_settings)
-    def test_string_setting_lowercase(self, key, default):
-        test_value = 'Test-String-For-{}'.format(key)
-        settings = {key: test_value}
-        _sanitize_vcs_settings(settings)
-        assert settings[key] == test_value.lower()
+    # @pytest.mark.parametrize('key, default', _string_funcs)
+    # def test_string_func_lowercase(self, key, default):
+    #     test_value = 'Test-String-For-{}'.format(key)
+    #     settings = {key: test_value}
+    #     sanitize_settings_and_apply_defaults({'__file__': ''}, settings)
+    #     assert settings[key] == test_value.lower()
 
-    @pytest.mark.parametrize('key, default', _bool_settings)
-    def test_bool_setting_true(self, key, default):
+    @pytest.mark.parametrize('key, default', _bool_funcs)
+    def test_bool_func_true(self, key, default):
         settings = {key: 'true'}
-        _sanitize_vcs_settings(settings)
+        sanitize_settings_and_apply_defaults({'__file__': ''}, settings)
         assert settings[key] is True
 
-    @pytest.mark.parametrize('key, default', _bool_settings)
-    def test_bool_setting_false(self, key, default):
+    @pytest.mark.parametrize('key, default', _bool_funcs)
+    def test_bool_func_false(self, key, default):
         settings = {key: 'false'}
-        _sanitize_vcs_settings(settings)
+        sanitize_settings_and_apply_defaults({'__file__': ''}, settings)
         assert settings[key] is False
 
-    @pytest.mark.parametrize('key, default', _bool_settings)
-    def test_bool_setting_invalid_string(self, key, default):
+    @pytest.mark.parametrize('key, default', _bool_funcs)
+    def test_bool_func_invalid_string(self, key, default):
         settings = {key: 'no-bool-val-string'}
-        _sanitize_vcs_settings(settings)
+        sanitize_settings_and_apply_defaults({'__file__': ''}, settings)
         assert settings[key] is False
 
-    @pytest.mark.parametrize('key, default', _bool_settings)
-    def test_bool_setting_default(self, key, default):
+    @pytest.mark.parametrize('key, default', _bool_funcs)
+    def test_bool_func_default(self, key, default):
         settings = {}
-        _sanitize_vcs_settings(settings)
+        sanitize_settings_and_apply_defaults({'__file__': ''}, settings)
         assert settings[key] is default

rhodecode/tests/rhodecode.ini

@@ -1,664 +1,658 b''
 
-
-################################################################################
-##                 RHODECODE COMMUNITY EDITION CONFIGURATION                  ##
-# The %(here)s variable will be replaced with the parent directory of this file#
-################################################################################
+; #########################################
+; RHODECODE COMMUNITY EDITION CONFIGURATION
+; #########################################
 
 [DEFAULT]
+; Debug flag sets all loggers to debug, and enables request tracking
 debug = true
 
-################################################################################
-##                            EMAIL CONFIGURATION                             ##
-## Uncomment and replace with the email address which should receive          ##
-## any error reports after an application crash                               ##
-## Additionally these settings will be used by the RhodeCode mailing system   ##
-################################################################################
+; ########################################################################
+; EMAIL CONFIGURATION
+; These settings will be used by the RhodeCode mailing system
+; ########################################################################
 
-## prefix all emails subjects with given prefix, helps filtering out emails
+; prefix all emails subjects with given prefix, helps filtering out emails
 #email_prefix = [RhodeCode]
 
-## email FROM address all mails will be sent
+; email FROM address all mails will be sent
 #app_email_from = rhodecode-noreply@localhost
 
-## Uncomment and replace with the address which should receive any error report
-## note: using appenlight for error handling doesn't need this to be uncommented
-#email_to = admin@localhost
-
 #smtp_server = mail.server.com
 #smtp_username =
 #smtp_password =
 #smtp_port =
 #smtp_use_tls = false
 #smtp_use_ssl = true
 
 [server:main]
-## COMMON ##
+; COMMON HOST/IP CONFIG
 host = 0.0.0.0
 port = 5000
 
-##########################
-## GUNICORN WSGI SERVER ##
-##########################
-## run with gunicorn --log-config rhodecode.ini --paste rhodecode.ini
+
+; ###########################
+; GUNICORN APPLICATION SERVER
+; ###########################
 
+; run with gunicorn --log-config rhodecode.ini --paste rhodecode.ini
+
+; Module to use, this setting shouldn't be changed
 use = egg:gunicorn#main
+
 ## Sets the number of process workers. You must set `instance_id = *`
 ## when this option is set to more than one worker, recommended
 ## value is (2 * NUMBER_OF_CPUS + 1), eg 2CPU = 5 workers
 ## The `instance_id = *` must be set in the [app:main] section below
 #workers = 2
 ## number of threads for each of the worker, must be set to 1 for gevent
 ## generally recommened to be at 1
 #threads = 1
 ## process name
 #proc_name = rhodecode
 ## type of worker class, one of sync, gevent
 ## recommended for bigger setup is using of of other than sync one
 #worker_class = sync
 ## The maximum number of simultaneous clients. Valid only for Gevent
 #worker_connections = 10
 ## max number of requests that worker will handle before being gracefully
 ## restarted, could prevent memory leaks
 #max_requests = 1000
 #max_requests_jitter = 30
 ## amount of time a worker can spend with handling a request before it
 ## gets killed and restarted. Set to 6hrs
 #timeout = 21600
 
 ## prefix middleware for RhodeCode.
 ## recommended when using proxy setup.
 ## allows to set RhodeCode under a prefix in server.
 ## eg https://server.com/custom_prefix. Enable `filter-with =` option below as well.
 ## And set your prefix like: `prefix = /custom_prefix`
 ## be sure to also set beaker.session.cookie_path = /custom_prefix if you need
 ## to make your cookies only work on prefix url
 [filter:proxy-prefix]
 use = egg:PasteDeploy#prefix
 prefix = /
 
 [app:main]
 is_test = True
 use = egg:rhodecode-enterprise-ce
 
-## enable proxy prefix middleware, defined above
+; enable proxy prefix middleware, defined above
 #filter-with = proxy-prefix
 
 
 ## RHODECODE PLUGINS ##
 rhodecode.includes = rhodecode.api
 
 # api prefix url
 rhodecode.api.url = /_admin/api
 
 
 ## END RHODECODE PLUGINS ##
 
 ## encryption key used to encrypt social plugin tokens,
 ## remote_urls with credentials etc, if not set it defaults to
 ## `beaker.session.secret`
 #rhodecode.encrypted_values.secret =
 
-## decryption strict mode (enabled by default). It controls if decryption raises
-## `SignatureVerificationError` in case of wrong key, or damaged encryption data.
+; decryption strict mode (enabled by default). It controls if decryption raises
+; `SignatureVerificationError` in case of wrong key, or damaged encryption data.
 #rhodecode.encrypted_values.strict = false
 
-## return gzipped responses from Rhodecode (static files/application)
+; Pick algorithm for encryption. Either fernet (more secure) or aes (default)
+; fernet is safer, and we strongly recommend switching to it.
+; Due to backward compatibility aes is used as default.
+#rhodecode.encrypted_values.algorithm = fernet
+
+; Return gzipped responses from RhodeCode (static files/application)
 gzip_responses = false
 
-## autogenerate javascript routes file on startup
+; Auto-generate javascript routes file on startup
 generate_js_files = false
 
-## Optional Languages
-## en(default), be, de, es, fr, it, ja, pl, pt, ru, zh
+; System global default language.
+; All available languages: en (default), be, de, es, fr, it, ja, pl, pt, ru, zh
 lang = en
 
 ## perform a full repository scan on each server start, this should be
 ## set to false after first startup, to allow faster server restarts.
 startup.import_repos = true
 
-## Uncomment and set this path to use archive download cache.
-## Once enabled, generated archives will be cached at this location
-## and served from the cache during subsequent requests for the same archive of
-## the repository.
+; Uncomment and set this path to use archive download cache.
+; Once enabled, generated archives will be cached at this location
+; and served from the cache during subsequent requests for the same archive of
+; the repository.
 #archive_cache_dir = /tmp/tarballcache
 
-## URL at which the application is running. This is used for bootstraping
-## requests in context when no web request is available. Used in ishell, or
-## SSH calls. Set this for events to receive proper url for SSH calls.
+; URL at which the application is running. This is used for Bootstrapping
+; requests in context when no web request is available. Used in ishell, or
+; SSH calls. Set this for events to receive proper url for SSH calls.
 app.base_url = http://rhodecode.local
 
-## change this to unique ID for security
+; Unique application ID. Should be a random unique string for security.
 app_instance_uuid = rc-production
 
 ## cut off limit for large diffs (size in bytes)
 cut_off_limit_diff = 1024000
 cut_off_limit_file = 256000
 
-## use cache version of scm repo everywhere
+; Use cached version of vcs repositories everywhere. Recommended to be `true`
 vcs_full_cache = false
 
-## force https in RhodeCode, fixes https redirects, assumes it's always https
-## Normally this is controlled by proper http flags sent from http server
+; Force https in RhodeCode, fixes https redirects, assumes it's always https.
+; Normally this is controlled by proper flags sent from http server such as Nginx or Apache
 force_https = false
 
-## use Strict-Transport-Security headers
+; use Strict-Transport-Security headers
 use_htsts = false
 
-# Set to true if your repos are exposed using the dumb protocol
+; Set to true if your repos are exposed using the dumb protocol
 git_update_server_info = false
 
-## RSS/ATOM feed options
+; RSS/ATOM feed options
 rss_cut_off_limit = 256000
 rss_items_per_page = 10
 rss_include_diff = false
 
-## gist URL alias, used to create nicer urls for gist. This should be an
-## url that does rewrites to _admin/gists/{gistid}.
-## example: http://gist.rhodecode.org/{gistid}. Empty means use the internal
-## RhodeCode url, ie. http[s]://rhodecode.server/_admin/gists/{gistid}
+; gist URL alias, used to create nicer urls for gist. This should be an
+; url that does rewrites to _admin/gists/{gistid}.
+; example: http://gist.rhodecode.org/{gistid}. Empty means use the internal
+; RhodeCode url, ie. http[s]://rhodecode.server/_admin/gists/{gistid}
 gist_alias_url =
 
 ## List of views (using glob pattern syntax) that AUTH TOKENS could be
 ## used for access.
 ## Adding ?auth_token=TOKEN_HASH to the url authenticates this request as if it
 ## came from the the logged in user who own this authentication token.
 ## Additionally @TOKEN syntaxt can be used to bound the view to specific
 ## authentication token. Such view would be only accessible when used together
 ## with this authentication token
 ##
 ## list of all views can be found under `/_admin/permissions/auth_token_access`
 ## The list should be "," separated and on a single line.
 ##
 ## Most common views to enable:
 #    RepoCommitsView:repo_commit_download
 #    RepoCommitsView:repo_commit_patch
 #    RepoCommitsView:repo_commit_raw
 #    RepoCommitsView:repo_commit_raw@TOKEN
 #    RepoFilesView:repo_files_diff
 #    RepoFilesView:repo_archivefile
 #    RepoFilesView:repo_file_raw
 #    GistView:*
 api_access_controllers_whitelist =
 
-## default encoding used to convert from and to unicode
-## can be also a comma separated list of encoding in case of mixed encodings
+; Default encoding used to convert from and to unicode
+; can be also a comma separated list of encoding in case of mixed encodings
 default_encoding = UTF-8
 
-## instance-id prefix
-## a prefix key for this instance used for cache invalidation when running
-## multiple instances of rhodecode, make sure it's globally unique for
-## all running rhodecode instances. Leave empty if you don't use it
+; instance-id prefix
+; a prefix key for this instance used for cache invalidation when running
+; multiple instances of RhodeCode, make sure it's globally unique for
+; all running RhodeCode instances. Leave empty if you don't use it
 instance_id =
 
-## Fallback authentication plugin. Set this to a plugin ID to force the usage
-## of an authentication plugin also if it is disabled by it's settings.
-## This could be useful if you are unable to log in to the system due to broken
-## authentication settings. Then you can enable e.g. the internal rhodecode auth
-## module to log in again and fix the settings.
-##
-## Available builtin plugin IDs (hash is part of the ID):
-## egg:rhodecode-enterprise-ce#rhodecode
-## egg:rhodecode-enterprise-ce#pam
-## egg:rhodecode-enterprise-ce#ldap
-## egg:rhodecode-enterprise-ce#jasig_cas
-## egg:rhodecode-enterprise-ce#headers
-## egg:rhodecode-enterprise-ce#crowd
+; Fallback authentication plugin. Set this to a plugin ID to force the usage
+; of an authentication plugin also if it is disabled by it's settings.
+; This could be useful if you are unable to log in to the system due to broken
+; authentication settings. Then you can enable e.g. the internal RhodeCode auth
+; module to log in again and fix the settings.
+; Available builtin plugin IDs (hash is part of the ID):
+; egg:rhodecode-enterprise-ce#rhodecode
+; egg:rhodecode-enterprise-ce#pam
+; egg:rhodecode-enterprise-ce#ldap
+; egg:rhodecode-enterprise-ce#jasig_cas
+; egg:rhodecode-enterprise-ce#headers
+; egg:rhodecode-enterprise-ce#crowd
+
 #rhodecode.auth_plugin_fallback = egg:rhodecode-enterprise-ce#rhodecode
 
-## alternative return HTTP header for failed authentication. Default HTTP
-## response is 401 HTTPUnauthorized. Currently HG clients have troubles with
-## handling that causing a series of failed authentication calls.
-## Set this variable to 403 to return HTTPForbidden, or any other HTTP code
-## This will be served instead of default 401 on bad authnetication
+; Flag to control loading of legacy plugins in py:/path format
+auth_plugin.import_legacy_plugins = true
+
+; alternative return HTTP header for failed authentication. Default HTTP
+; response is 401 HTTPUnauthorized. Currently HG clients have troubles with
+; handling that causing a series of failed authentication calls.
+; Set this variable to 403 to return HTTPForbidden, or any other HTTP code
+; This will be served instead of default 401 on bad authentication
 auth_ret_code =
 
 ## use special detection method when serving auth_ret_code, instead of serving
 ## ret_code directly, use 401 initially (Which triggers credentials prompt)
 ## and then serve auth_ret_code to clients
 auth_ret_code_detection = false
 
 ## locking return code. When repository is locked return this HTTP code. 2XX
 ## codes don't break the transactions while 4XX codes do
 lock_ret_code = 423
 
 ## allows to change the repository location in settings page
 allow_repo_location_change = true
 
 ## allows to setup custom hooks in settings page
 allow_custom_hooks_settings = true
 
 ## generated license token, goto license page in RhodeCode settings to obtain
 ## new token
 license_token = abra-cada-bra1-rce3
 
 ## supervisor connection uri, for managing supervisor and logs.
 supervisor.uri =
 ## supervisord group name/id we only want this RC instance to handle
 supervisor.group_id = dev
 
 ## Display extended labs settings
 labs_settings_active = true
 
-####################################
-###        CELERY CONFIG        ####
-####################################
+; #############
+; CELERY CONFIG
+; #############
+
+; manually run celery: /path/to/celery worker -E --beat --app rhodecode.lib.celerylib.loader --scheduler rhodecode.lib.celerylib.scheduler.RcScheduler --loglevel DEBUG --ini /path/to/rhodecode.ini
+
 use_celery = false
-broker.host = localhost
-broker.vhost = rabbitmqhost
-broker.port = 5672
-broker.user = rabbitmq
-broker.password = qweqwe
 
-celery.imports = rhodecode.lib.celerylib.tasks
+; path to store schedule database
+#celerybeat-schedule.path =
 
-celery.result.backend = amqp
-celery.result.dburi = amqp://
-celery.result.serialier = json
+; connection url to the message broker (default redis)
+celery.broker_url = redis://localhost:6379/8
 
-#celery.send.task.error.emails = true
-#celery.amqp.task.result.expires = 18000
+; rabbitmq example
+#celery.broker_url = amqp://rabbitmq:qweqwe@localhost:5672/rabbitmqhost
 
-celeryd.concurrency = 2
-#celeryd.log.file = celeryd.log
-celeryd.log.level = debug
-celeryd.max.tasks.per.child = 1
+; maximum tasks to execute before worker restart
+celery.max_tasks_per_child = 100
 
-## tasks will never be sent to the queue, but executed locally instead.
-celery.always.eager = false
+; tasks will never be sent to the queue, but executed locally instead.
+celery.task_always_eager = false
 
-####################################
-###         BEAKER CACHE        ####
-####################################
-# default cache dir for templates.  Putting this into a ramdisk
-## can boost performance, eg. %(here)s/data_ramdisk
+; #############
+; DOGPILE CACHE
+; #############
+
+; Default cache dir for caches. Putting this into a ramdisk can boost performance.
+; eg. /tmpfs/data_ramdisk, however this directory might require large amount of space
 cache_dir = %(here)s/data
 
 ## locking and default file storage for Beaker. Putting this into a ramdisk
 ## can boost performance, eg. %(here)s/data_ramdisk/cache/beaker_data
 beaker.cache.data_dir = %(here)s/rc/data/cache/beaker_data
 beaker.cache.lock_dir = %(here)s/rc/data/cache/beaker_lock
 
 beaker.cache.regions = long_term
 
 beaker.cache.long_term.type = memory
 beaker.cache.long_term.expire = 36000
 beaker.cache.long_term.key_length = 256
 
 
 #####################################
 ###         DOGPILE CACHE        ####
 #####################################
 
 ## permission tree cache settings
 rc_cache.cache_perms.backend = dogpile.cache.rc.file_namespace
 rc_cache.cache_perms.expiration_time = 0
 rc_cache.cache_perms.arguments.filename = /tmp/rc_cache_1
 
 
 ## cache settings for SQL queries
 rc_cache.sql_cache_short.backend = dogpile.cache.rc.memory_lru
 rc_cache.sql_cache_short.expiration_time = 0
 
 
-####################################
-###       BEAKER SESSION        ####
-####################################
+; ##############
+; BEAKER SESSION
+; ##############
 
-## .session.type is type of storage options for the session, current allowed
-## types are file, ext:memcached, ext:database, and memory (default).
+; beaker.session.type is type of storage options for the logged users sessions. Current allowed
+; types are file, ext:redis, ext:database, ext:memcached, and memory (default if not specified).
+; Fastest ones are Redis and ext:database
 beaker.session.type = file
 beaker.session.data_dir = %(here)s/rc/data/sessions/data
 
-## db based session, fast, and allows easy management over logged in users
+; Redis based sessions
+#beaker.session.type = ext:redis
+#beaker.session.url = redis://127.0.0.1:6379/2
+
+; DB based session, fast, and allows easy management over logged in users
 #beaker.session.type = ext:database
 #beaker.session.table_name = db_session
 #beaker.session.sa.url = postgresql://postgres:secret@localhost/rhodecode
 #beaker.session.sa.url = mysql://root:secret@127.0.0.1/rhodecode
 #beaker.session.sa.pool_recycle = 3600
 #beaker.session.sa.echo = false
 
 beaker.session.key = rhodecode
 beaker.session.secret = test-rc-uytcxaz
 beaker.session.lock_dir = %(here)s/rc/data/sessions/lock
 
-## Secure encrypted cookie. Requires AES and AES python libraries
-## you must disable beaker.session.secret to use this
+; Secure encrypted cookie. Requires AES and AES python libraries
+; you must disable beaker.session.secret to use this
 #beaker.session.encrypt_key = key_for_encryption
 #beaker.session.validate_key = validation_key
 
-## sets session as invalid(also logging out user) if it haven not been
-## accessed for given amount of time in seconds
+; Sets session as invalid (also logging out user) if it haven not been
+; accessed for given amount of time in seconds
 beaker.session.timeout = 2592000
 beaker.session.httponly = true
-## Path to use for the cookie. Set to prefix if you use prefix middleware
+
+; Path to use for the cookie. Set to prefix if you use prefix middleware
 #beaker.session.cookie_path = /custom_prefix
 
-## uncomment for https secure cookie
+; Set https secure cookie
 beaker.session.secure = false
 
 ## auto save the session to not to use .save()
 beaker.session.auto = false
 
 ## default cookie expiration time in seconds, set to `true` to set expire
 ## at browser close
 #beaker.session.cookie_expires = 3600
 
-###################################
-## SEARCH INDEXING CONFIGURATION ##
-###################################
-## Full text search indexer is available in rhodecode-tools under
-## `rhodecode-tools index` command
+; #############################
+; SEARCH INDEXING CONFIGURATION
+; #############################
 
 ## WHOOSH Backend, doesn't require additional services to run
 ## it works good with few dozen repos
 search.module = rhodecode.lib.index.whoosh
 search.location = %(here)s/data/index
 
-########################################
-###    CHANNELSTREAM CONFIG         ####
-########################################
-## channelstream enables persistent connections and live notification
-## in the system. It's also used by the chat system
+; ####################
+; CHANNELSTREAM CONFIG
+; ####################
+
+; channelstream enables persistent connections and live notification
+; in the system. It's also used by the chat system
 
 channelstream.enabled = false
 
-## server address for channelstream server on the backend
+; server address for channelstream server on the backend
 channelstream.server = 127.0.0.1:9800
-## location of the channelstream server from outside world
-## use ws:// for http or wss:// for https. This address needs to be handled
-## by external HTTP server such as Nginx or Apache
-## see nginx/apache configuration examples in our docs
+
+; location of the channelstream server from outside world
+; use ws:// for http or wss:// for https. This address needs to be handled
+; by external HTTP server such as Nginx or Apache
+; see Nginx/Apache configuration examples in our docs
 channelstream.ws_url = ws://rhodecode.yourserver.com/_channelstream
 channelstream.secret = secret
 channelstream.history.location = %(here)s/channelstream_history
 
-## Internal application path that Javascript uses to connect into.
-## If you use proxy-prefix the prefix should be added before /_channelstream
+; Internal application path that Javascript uses to connect into.
+; If you use proxy-prefix the prefix should be added before /_channelstream
 channelstream.proxy_path = /_channelstream
 
 
-###################################
-##       APPENLIGHT CONFIG       ##
-###################################
-
-## Appenlight is tailored to work with RhodeCode, see
-## http://appenlight.com for details how to obtain an account
-
-## appenlight integration enabled
-appenlight = false
-
-appenlight.server_url = https://api.appenlight.com
-appenlight.api_key = YOUR_API_KEY
-#appenlight.transport_config = https://api.appenlight.com?threaded=1&timeout=5
-
-# used for JS client
-appenlight.api_public_key = YOUR_API_PUBLIC_KEY
-
-## TWEAK AMOUNT OF INFO SENT HERE
-
-## enables 404 error logging (default False)
-appenlight.report_404 = false
-
-## time in seconds after request is considered being slow (default 1)
-appenlight.slow_request_time = 1
-
-## record slow requests in application
-## (needs to be enabled for slow datastore recording and time tracking)
-appenlight.slow_requests = true
-
-## enable hooking to application loggers
-appenlight.logging = true
-
-## minimum log level for log capture
-appenlight.logging.level = WARNING
-
-## send logs only from erroneous/slow requests
-## (saves API quota for intensive logging)
-appenlight.logging_on_error = false
+; ##############################
+; MAIN RHODECODE DATABASE CONFIG
+; ##############################
 
-## list of additonal keywords that should be grabbed from environ object
-## can be string with comma separated list of words in lowercase
-## (by default client will always send following info:
-## 'REMOTE_USER', 'REMOTE_ADDR', 'SERVER_NAME', 'CONTENT_TYPE' + all keys that
-## start with HTTP* this list be extended with additional keywords here
-appenlight.environ_keys_whitelist =
-
-## list of keywords that should be blanked from request object
-## can be string with comma separated list of words in lowercase
-## (by default client will always blank keys that contain following words
-## 'password', 'passwd', 'pwd', 'auth_tkt', 'secret', 'csrf'
-## this list be extended with additional keywords set here
-appenlight.request_keys_blacklist =
-
-## list of namespaces that should be ignores when gathering log entries
-## can be string with comma separated list of namespaces
-## (by default the client ignores own entries: appenlight_client.client)
-appenlight.log_namespace_blacklist =
-
+#sqlalchemy.db1.url = sqlite:///%(here)s/rhodecode.db?timeout=30
+#sqlalchemy.db1.url = postgresql://postgres:qweqwe@localhost/rhodecode
+#sqlalchemy.db1.url = mysql://root:qweqwe@localhost/rhodecode?charset=utf8
+; pymysql is an alternative driver for MySQL, use in case of problems with default one
+#sqlalchemy.db1.url = mysql+pymysql://root:qweqwe@localhost/rhodecode
 
-################################################################################
-## WARNING: *THE LINE BELOW MUST BE UNCOMMENTED ON A PRODUCTION ENVIRONMENT*  ##
-## Debug mode will enable the interactive debugging tool, allowing ANYONE to  ##
-## execute malicious code after an exception is raised.                       ##
-################################################################################
-set debug = false
-
-
-##############
-## STYLING  ##
-##############
-debug_style = false
-
-###########################################
-###   MAIN RHODECODE DATABASE CONFIG    ###
-###########################################
-#sqlalchemy.db1.url = sqlite:///%(here)s/rhodecode_test.db?timeout=30
-#sqlalchemy.db1.url = postgresql://postgres:qweqwe@localhost/rhodecode_test
-#sqlalchemy.db1.url = mysql://root:qweqwe@localhost/rhodecode_test
 sqlalchemy.db1.url = sqlite:///%(here)s/rhodecode_test.db?timeout=30
 
-# see sqlalchemy docs for other advanced settings
-
-## print the sql statements to output
+; see sqlalchemy docs for other advanced settings
+; print the sql statements to output
 sqlalchemy.db1.echo = false
-## recycle the connections after this amount of seconds
-sqlalchemy.db1.pool_recycle = 3600
 
-## the number of connections to keep open inside the connection pool.
-## 0 indicates no limit
+; recycle the connections after this amount of seconds
+sqlalchemy.db1.pool_recycle = 3600
+sqlalchemy.db1.convert_unicode = true
+
+; the number of connections to keep open inside the connection pool.
+; 0 indicates no limit
 #sqlalchemy.db1.pool_size = 5
 
-## the number of connections to allow in connection pool "overflow", that is
-## connections that can be opened above and beyond the pool_size setting,
-## which defaults to five.
+; The number of connections to allow in connection pool "overflow", that is
+; connections that can be opened above and beyond the pool_size setting,
+; which defaults to five.
 #sqlalchemy.db1.max_overflow = 10
 
+; Connection check ping, used to detect broken database connections
+; could be enabled to better handle cases if MySQL has gone away errors
+#sqlalchemy.db1.ping_connection = true
 
-##################
-### VCS CONFIG ###
-##################
+; ##########
+; VCS CONFIG
+; ##########
 vcs.server.enable = true
 vcs.server = localhost:9901
 
-## Web server connectivity protocol, responsible for web based VCS operatations
-## Available protocols are:
-## `http` - use http-rpc backend (default)
+; Web server connectivity protocol, responsible for web based VCS operations
+; Available protocols are:
+; `http` - use http-rpc backend (default)
 vcs.server.protocol = http
 
-## Push/Pull operations protocol, available options are:
-## `http` - use http-rpc backend (default)
-## `vcsserver.scm_app` - internal app (EE only)
+; Push/Pull operations protocol, available options are:
+; `http` - use http-rpc backend (default)
 vcs.scm_app_implementation = http
 
-## Push/Pull operations hooks protocol, available options are:
-## `http` - use http-rpc backend (default)
+; Push/Pull operations hooks protocol, available options are:
+; `http` - use http-rpc backend (default)
 vcs.hooks.protocol = http
+
+; Host on which this instance is listening for hooks. If vcsserver is in other location
+; this should be adjusted.
 vcs.hooks.host = 127.0.0.1
 
-
-## Start VCSServer with this instance as a subprocess, Useful for development
+; Start VCSServer with this instance as a subprocess, useful for development
 vcs.start_server = false
 
-## List of enabled VCS backends, available options are:
-## `hg`  - mercurial
-## `git` - git
-## `svn` - subversion
+; List of enabled VCS backends, available options are:
+; `hg`  - mercurial
+; `git` - git
+; `svn` - subversion
 vcs.backends = hg, git, svn
 
+; Wait this number of seconds before killing connection to the vcsserver
 vcs.connection_timeout = 3600
-## Compatibility version when creating SVN repositories. Defaults to newest version when commented out.
-## Available options are: pre-1.4-compatible, pre-1.5-compatible, pre-1.6-compatible, pre-1.8-compatible, pre-1.9-compatible
-#vcs.svn.compatible_version = pre-1.8-compatible
 
+; Compatibility version when creating SVN repositories. Defaults to newest version when commented out.
+; Set a numeric version for your current SVN e.g 1.8, or 1.12
+; Legacy available options are: pre-1.4-compatible, pre-1.5-compatible, pre-1.6-compatible, pre-1.8-compatible, pre-1.9-compatible
+#vcs.svn.compatible_version = 1.8
 
-############################################################
-### Subversion proxy support (mod_dav_svn)               ###
-### Maps RhodeCode repo groups into SVN paths for Apache ###
-############################################################
-## Enable or disable the config file generation.
+; Cache flag to cache vcsserver remote calls locally
+; It uses cache_region `cache_repo`
+vcs.methods.cache = false
+
+; ####################################################
+; Subversion proxy support (mod_dav_svn)
+; Maps RhodeCode repo groups into SVN paths for Apache
+; ####################################################
+
+; Enable or disable the config file generation.
 svn.proxy.generate_config = false
-## Generate config file with `SVNListParentPath` set to `On`.
+
+; Generate config file with `SVNListParentPath` set to `On`.
 svn.proxy.list_parent_path = true
-## Set location and file name of generated config file.
+
+; Set location and file name of generated config file.
 svn.proxy.config_file_path = %(here)s/mod_dav_svn.conf
-## Used as a prefix to the `Location` block in the generated config file.
-## In most cases it should be set to `/`.
+
+; alternative mod_dav config template. This needs to be a valid mako template
+; Example template can be found in the source code:
+; rhodecode/apps/svn_support/templates/mod-dav-svn.conf.mako
+#svn.proxy.config_template = ~/.rccontrol/enterprise-1/custom_svn_conf.mako
+
+; Used as a prefix to the `Location` block in the generated config file.
+; In most cases it should be set to `/`.
 svn.proxy.location_root = /
-## Command to reload the mod dav svn configuration on change.
-## Example: `/etc/init.d/apache2 reload`
+
+; Command to reload the mod dav svn configuration on change.
+; Example: `/etc/init.d/apache2 reload` or /home/USER/apache_reload.sh
+; Make sure user who runs RhodeCode process is allowed to reload Apache
 #svn.proxy.reload_cmd = /etc/init.d/apache2 reload
-## If the timeout expires before the reload command finishes, the command will
-## be killed. Setting it to zero means no timeout. Defaults to 10 seconds.
+
+; If the timeout expires before the reload command finishes, the command will
+; be killed. Setting it to zero means no timeout. Defaults to 10 seconds.
 #svn.proxy.reload_timeout = 10
 
-############################################################
-### SSH Support Settings                                 ###
-############################################################
+; ####################
+; SSH Support Settings
+; ####################
 
-## Defines if the authorized_keys file should be written on any change of
-## user ssh keys, setting this to false also disables posibility of adding
-## ssh keys for users from web interface.
+; Defines if a custom authorized_keys file should be created and written on
+; any change user ssh keys. Setting this to false also disables possibility
+; of adding SSH keys by users from web interface. Super admins can still
+; manage SSH Keys.
 ssh.generate_authorized_keyfile = true
 
-## Options for ssh, default is `no-pty,no-port-forwarding,no-X11-forwarding,no-agent-forwarding`
+; Options for ssh, default is `no-pty,no-port-forwarding,no-X11-forwarding,no-agent-forwarding`
 # ssh.authorized_keys_ssh_opts =
 
-## File to generate the authorized keys together with options
-## It is possible to have multiple key files specified in `sshd_config` e.g.
-## AuthorizedKeysFile %h/.ssh/authorized_keys %h/.ssh/authorized_keys_rhodecode
+; Path to the authorized_keys file where the generate entries are placed.
+; It is possible to have multiple key files specified in `sshd_config` e.g.
+; AuthorizedKeysFile %h/.ssh/authorized_keys %h/.ssh/authorized_keys_rhodecode
 ssh.authorized_keys_file_path = %(here)s/rc/authorized_keys_rhodecode
 
-## Command to execute the SSH wrapper. The binary is available in the
-## rhodecode installation directory.
-## e.g ~/.rccontrol/community-1/profile/bin/rc-ssh-wrapper
+; Command to execute the SSH wrapper. The binary is available in the
+; RhodeCode installation directory.
+; e.g ~/.rccontrol/community-1/profile/bin/rc-ssh-wrapper
 ssh.wrapper_cmd = ~/.rccontrol/community-1/rc-ssh-wrapper
 
-## Allow shell when executing the ssh-wrapper command
+; Allow shell when executing the ssh-wrapper command
 ssh.wrapper_cmd_allow_shell = false
 
-## Enables logging, and detailed output send back to the client. Useful for
-## debugging, shouldn't be used in production.
+; Enables logging, and detailed output send back to the client during SSH
+; operations. Useful for debugging, shouldn't be used in production.
 ssh.enable_debug_logging = false
 
-## Paths to binary executrables, by default they are the names, but we can
-## override them if we want to use a custom one
+; Paths to binary executable, by default they are the names, but we can
+; override them if we want to use a custom one
 ssh.executable.hg = ~/.rccontrol/vcsserver-1/profile/bin/hg
 ssh.executable.git = ~/.rccontrol/vcsserver-1/profile/bin/git
 ssh.executable.svn = ~/.rccontrol/vcsserver-1/profile/bin/svnserve
 
-## Enables SSH key generator web interface. Disabling this still allows users
-## to add their own keys.
+; Enables SSH key generator web interface. Disabling this still allows users
+; to add their own keys.
 ssh.enable_ui_key_generator = true
 
+; Statsd client config, this is used to send metrics to statsd
+; We recommend setting statsd_exported and scrape them using Promethues
+#statsd.enabled = false
+#statsd.statsd_host = 0.0.0.0
+#statsd.statsd_port = 8125
+#statsd.statsd_prefix =
+#statsd.statsd_ipv6 = false
 
-## Dummy marker to add new entries after.
-## Add any custom entries below. Please don't remove.
+
+; configure logging automatically at server startup set to false
+; to use the below custom logging config.
+logging.autoconfigure = false
+
+; specify your own custom logging config file to configure logging
+#logging.logging_conf_file = /path/to/custom_logging.ini
+
+; Dummy marker to add new entries after.
+; Add any custom entries below. Please don't remove this marker.
 custom.conf = 1
 
 
-################################
-### LOGGING CONFIGURATION   ####
-################################
+; #####################
+; LOGGING CONFIGURATION
+; #####################
 [loggers]
 keys = root, sqlalchemy, beaker, rhodecode, ssh_wrapper
 
 [handlers]
 keys = console, console_sql
 
 [formatters]
 keys = generic, color_formatter, color_formatter_sql
 
-#############
-## LOGGERS ##
-#############
+; #######
+; LOGGERS
+; #######
 [logger_root]
 level = NOTSET
 handlers = console
 
 [logger_routes]
 level = DEBUG
 handlers =
 qualname = routes.middleware
 ## "level = DEBUG" logs the route matched and routing variables.
 propagate = 1
 
+[logger_sqlalchemy]
+level = INFO
+handlers = console_sql
+qualname = sqlalchemy.engine
+propagate = 0
+
 [logger_beaker]
 level = DEBUG
 handlers =
 qualname = beaker.container
 propagate = 1
 
 [logger_rhodecode]
 level = DEBUG
 handlers =
 qualname = rhodecode
 propagate = 1
 
-[logger_sqlalchemy]
-level = ERROR
-handlers = console_sql
-qualname = sqlalchemy.engine
-propagate = 0
-
 [logger_ssh_wrapper]
 level = DEBUG
 handlers =
 qualname = ssh_wrapper
 propagate = 1
 
+[logger_celery]
+level = DEBUG
+handlers =
+qualname = celery
 
-##############
-## HANDLERS ##
-##############
+
+; ########
+; HANDLERS
+; ########
 
 [handler_console]
 class = StreamHandler
-args = (sys.stderr,)
+args = (sys.stderr, )
 level = DEBUG
 formatter = generic
+; To enable JSON formatted logs replace generic with json
+; This allows sending properly formatted logs to grafana loki or elasticsearch
+#formatter = json
 
 [handler_console_sql]
+; "level = DEBUG" logs SQL queries and results.
+; "level = INFO" logs SQL queries.
+; "level = WARN" logs neither.  (Recommended for production systems.)
 class = StreamHandler
-args = (sys.stderr,)
+args = (sys.stderr, )
 level = WARN
 formatter = generic
 
-################
-## FORMATTERS ##
-################
+; ##########
+; FORMATTERS
+; ##########
 
 [formatter_generic]
 class = rhodecode.lib.logging_formatter.ExceptionAwareFormatter
-format = %(asctime)s.%(msecs)03d %(levelname)-5.5s [%(name)s] %(message)s
+format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s
 datefmt = %Y-%m-%d %H:%M:%S
 
 [formatter_color_formatter]
 class = rhodecode.lib.logging_formatter.ColorFormatter
-format = %(asctime)s.%(msecs)03d %(levelname)-5.5s [%(name)s] %(message)s
+format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s
 datefmt = %Y-%m-%d %H:%M:%S
 
 [formatter_color_formatter_sql]
 class = rhodecode.lib.logging_formatter.ColorFormatterSql
-format = %(asctime)s.%(msecs)03d %(levelname)-5.5s [%(name)s] %(message)s
+format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s
 datefmt = %Y-%m-%d %H:%M:%S
+
+[formatter_json]
+format = %(message)s
+class = rhodecode.lib._vendor.jsonlogger.JsonFormatter
  No newline at end of file