core: re-implemented the way how configuration can be made

- all controllable via ENV variables that can override .ini file settings
- make logging configurable by extension file
- added more defauts and settings validation via settings_make
- update .ini configs and commented out defaults

configs/logging.ini

@@ -0,0 +1,53 b''
+; #####################
+; LOGGING CONFIGURATION
+; #####################
+; Logging template, used for configure the logging
+; some variables here are replaced by RhodeCode to default values
+
+[loggers]
+keys = root, vcsserver
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic, json
+
+; #######
+; LOGGERS
+; #######
+[logger_root]
+level = NOTSET
+handlers = console
+
+[logger_vcsserver]
+level = $RC_LOGGING_LEVEL
+handlers =
+qualname = vcsserver
+propagate = 1
+
+; ########
+; HANDLERS
+; ########
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr, )
+level = $RC_LOGGING_LEVEL
+; To enable JSON formatted logs replace generic with json
+; This allows sending properly formatted logs to grafana loki or elasticsearch
+#formatter = json
+#formatter = generic
+formatter = $RC_LOGGING_FORMATTER
+
+; ##########
+; FORMATTERS
+; ##########
+
+[formatter_generic]
+format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s
+datefmt = %Y-%m-%d %H:%M:%S
+
+[formatter_json]
+format = %(timestamp)s %(levelname)s %(name)s %(message)s %(req_id)s
+class = vcsserver.lib._vendor.jsonlogger.JsonFormatter

vcsserver/config/settings_maker.py

@@ -0,0 +1,177 b''
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2010-2020 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+
+import os
+import textwrap
+import string
+import functools
+import logging
+import tempfile
+import logging.config
+log = logging.getLogger(__name__)
+
+
+def str2bool(_str):
+    """
+    returns True/False value from given string, it tries to translate the
+    string into boolean
+
+    :param _str: string value to translate into boolean
+    :rtype: boolean
+    :returns: boolean from given string
+    """
+    if _str is None:
+        return False
+    if _str in (True, False):
+        return _str
+    _str = str(_str).strip().lower()
+    return _str in ('t', 'true', 'y', 'yes', 'on', '1')
+
+
+def aslist(obj, sep=None, strip=True):
+    """
+    Returns given string separated by sep as list
+
+    :param obj:
+    :param sep:
+    :param strip:
+    """
+    if isinstance(obj, (basestring,)):
+        lst = obj.split(sep)
+        if strip:
+            lst = [v.strip() for v in lst]
+        return lst
+    elif isinstance(obj, (list, tuple)):
+        return obj
+    elif obj is None:
+        return []
+    else:
+        return [obj]
+
+
+class SettingsMaker(object):
+
+    def __init__(self, app_settings):
+        self.settings = app_settings
+
+    @classmethod
+    def _bool_func(cls, input_val):
+        if isinstance(input_val, unicode):
+            input_val = input_val.encode('utf8')
+        return str2bool(input_val)
+
+    @classmethod
+    def _int_func(cls, input_val):
+        return int(input_val)
+
+    @classmethod
+    def _list_func(cls, input_val, sep=','):
+        return aslist(input_val, sep=sep)
+
+    @classmethod
+    def _string_func(cls, input_val, lower=True):
+        if lower:
+            input_val = input_val.lower()
+        return input_val
+
+    @classmethod
+    def _float_func(cls, input_val):
+        return float(input_val)
+
+    @classmethod
+    def _dir_func(cls, input_val, ensure_dir=False, mode=0o755):
+
+        # ensure we have our dir created
+        if not os.path.isdir(input_val) and ensure_dir:
+            os.makedirs(input_val, mode=mode)
+
+        if not os.path.isdir(input_val):
+            raise Exception('Dir at {} does not exist'.format(input_val))
+        return input_val
+
+    @classmethod
+    def _file_path_func(cls, input_val, ensure_dir=False, mode=0o755):
+        dirname = os.path.dirname(input_val)
+        cls._dir_func(dirname, ensure_dir=ensure_dir)
+        return input_val
+
+    @classmethod
+    def _key_transformator(cls, key):
+        return "{}_{}".format('RC'.upper(), key.upper().replace('.', '_').replace('-', '_'))
+
+    def enable_logging(self, logging_conf=None):
+        """
+        Helper to enable debug on running instance
+        :return:
+        """
+        if not str2bool(self.settings.get('logging.autoconfigure')):
+            log.info('logging configuration based on main .ini file')
+            return
+
+        if logging_conf is None:
+            logging_conf = self.settings.get('logging.logging_conf_file') or ''
+
+        if not os.path.isfile(logging_conf):
+            log.error('Unable to setup logging based on %s, file does not exist...', logging_conf)
+            return
+
+        with open(logging_conf, 'rb') as f:
+            ini_template = textwrap.dedent(f.read())
+            ini_template = string.Template(ini_template).safe_substitute(
+                RC_LOGGING_LEVEL=os.environ.get('RC_LOGGING_LEVEL', '') or 'INFO',
+                RC_LOGGING_FORMATTER=os.environ.get('RC_LOGGING_FORMATTER', '') or 'generic'
+            )
+
+        with tempfile.NamedTemporaryFile(prefix='rc_logging_', suffix='.ini', delete=False) as f:
+            log.info('Saved Temporary LOGGING config at %s', f.name)
+            f.write(ini_template)
+
+        logging.config.fileConfig(f.name)
+        os.remove(f.name)
+
+    def make_setting(self, key, default, lower=False, default_when_empty=False, parser=None):
+
+        input_val = self.settings.get(key, default)
+
+        if default_when_empty and not input_val:
+            # use default value when value is set in the config but it is empty
+            input_val = default
+
+        parser_func = {
+            'bool': self._bool_func,
+            'int': self._int_func,
+            'list': self._list_func,
+            'list:newline': functools.partial(self._list_func, sep='/n'),
+            'string': functools.partial(self._string_func, lower=lower),
+            'dir': self._dir_func,
+            'dir:ensured': functools.partial(self._dir_func, ensure_dir=True),
+            'file': self._file_path_func,
+            'file:ensured': functools.partial(self._file_path_func, ensure_dir=True),
+            None: lambda i: i
+        }[parser]
+
+        # now maybe we have this KEY in env, search and use the value with higher priority.
+        transformed_key = self._key_transformator(key)
+        envvar_value = os.environ.get(transformed_key)
+        if envvar_value:
+            log.debug('using `%s` key instead of `%s` key for config', transformed_key, key)
+            input_val = envvar_value
+        self.settings[key] = parser_func(input_val)
+        return self.settings[key]

configs/development.ini

@@ -31,7 +31,7 b' asyncore_use_poll = true'
 ; GUNICORN APPLICATION SERVER
 ; ###########################
 
-; run with gunicorn --log-config rhodecode.ini --paste rhodecode.ini
+; run with gunicorn --paste rhodecode.ini
 
 ; Module to use, this setting shouldn't be changed
 #use = egg:gunicorn#main
@@ -86,7 +86,7 b' asyncore_use_poll = true'
 ; serving requests. Workers still alive after the timeout (starting from the
 ; receipt of the restart signal) are force killed.
 ; Examples: 1800 (30min), 3600 (1hr), 7200 (2hr), 43200 (12h)
-#graceful_timeout = 3600
+#graceful_timeout = 21600
 
 # The number of seconds to wait for requests on a Keep-Alive connection.
 # Generally set in the 1-5 seconds range.
@@ -110,6 +110,17 b' asyncore_use_poll = true'
 [app:main]
 ; The %(here)s variable will be replaced with the absolute path of parent directory
 ; of this file
+; Each option in the app:main can be override by an environmental variable
+;
+;To override an option:
+;
+;RC_<KeyName>
+;Everything should be uppercase, . and - should be replaced by _.
+;For example, if you have these configuration settings:
+;rc_cache.repo_object.backend = foo
+;can be overridden by
+;export RC_CACHE_REPO_OBJECT_BACKEND=foo
+
 use = egg:rhodecode-vcsserver
 
 
@@ -133,13 +144,13 b' debugtoolbar.exclude_prefixes ='
 ; #################
 
 ; Pyramid default locales, we need this to be set
-pyramid.default_locale_name = en
+#pyramid.default_locale_name = en
 
 ; default locale used by VCS systems
-locale = en_US.UTF-8
+#locale = en_US.UTF-8
 
 ; path to binaries for vcsserver, it should be set by the installer
-; at installation time, e.g /home/user/vcsserver-1/profile/bin
+; at installation time, e.g /home/user/.rccontrol/vcsserver-1/profile/bin
 ; it can also be a path to nix-build output in case of development
 core.binary_dir = ""
 
@@ -153,21 +164,21 b' core.binary_dir = ""'
 
 ; Default cache dir for caches. Putting this into a ramdisk can boost performance.
 ; eg. /tmpfs/data_ramdisk, however this directory might require large amount of space
-cache_dir = %(here)s/data
+#cache_dir = %(here)s/data
 
 ; ***************************************
 ; `repo_object` cache, default file based
 ; ***************************************
 
 ; `repo_object` cache settings for vcs methods for repositories
-rc_cache.repo_object.backend = dogpile.cache.rc.file_namespace
+#rc_cache.repo_object.backend = dogpile.cache.rc.file_namespace
 
 ; cache auto-expires after N seconds
 ; Examples: 86400 (1Day), 604800 (7Days), 1209600 (14Days), 2592000 (30days), 7776000 (90Days)
-rc_cache.repo_object.expiration_time = 2592000
+#rc_cache.repo_object.expiration_time = 2592000
 
 ; file cache store path. Defaults to `cache_dir =` value or tempdir if both values are not set
-#rc_cache.repo_object.arguments.filename = /tmp/vcsserver_cache.db
+#rc_cache.repo_object.arguments.filename = /tmp/vcsserver_cache_repo_object.db
 
 ; ***********************************************************
 ; `repo_object` cache with redis backend
@@ -202,55 +213,59 b' rc_cache.repo_object.expiration_time = 2'
 #statsd.statsd_prefix =
 #statsd.statsd_ipv6 = false
 
+; configure logging automatically at server startup set to false
+; to use the below custom logging config.
+#logging.autoconfigure = true
+
+; specify your own custom logging config file to configure logging
+#logging.logging_conf_file = /path/to/custom_logging.ini
+
 ; #####################
 ; LOGGING CONFIGURATION
 ; #####################
-[loggers]
-keys = root, vcsserver
+#[loggers]
+#keys = root, vcsserver
 
-[handlers]
-keys = console
+#[handlers]
+#keys = console
 
-[formatters]
-keys = generic
+#[formatters]
+#keys = generic
 
 ; #######
 ; LOGGERS
 ; #######
-[logger_root]
-level = NOTSET
-handlers = console
+#[logger_root]
+#level = NOTSET
+#handlers = console
 
-[logger_vcsserver]
-level = DEBUG
-handlers =
-qualname = vcsserver
-propagate = 1
-
+#[logger_vcsserver]
+#level = INFO
+#handlers =
+#qualname = vcsserver
+#propagate = 1
 
 ; ########
 ; HANDLERS
 ; ########
 
-[handler_console]
-class = StreamHandler
-args = (sys.stderr, )
-level = DEBUG
-formatter = generic
+#[handler_console]
+#class = StreamHandler
+#args = (sys.stderr, )
+#level = INFO
 ; To enable JSON formatted logs replace generic with json
 ; This allows sending properly formatted logs to grafana loki or elasticsearch
 #formatter = json
-
+#formatter = generic
 
 ; ##########
 ; FORMATTERS
 ; ##########
 
-[formatter_generic]
-format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s
-datefmt = %Y-%m-%d %H:%M:%S
+#[formatter_generic]
+#format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s
+#datefmt = %Y-%m-%d %H:%M:%S
 
-[formatter_json]
-format = %(timestamp)s %(levelname)s %(name)s %(message)s %(req_id)s
-class = vcsserver.lib._vendor.jsonlogger.JsonFormatter
-
+#[formatter_json]
+#format = %(timestamp)s %(levelname)s %(name)s %(message)s %(req_id)s
+#class = vcsserver.lib._vendor.jsonlogger.JsonFormatter

configs/gunicorn_config.py

@@ -30,12 +30,12 b' worker_tmp_dir = None'
 tmp_upload_dir = None
 
 # Custom log format
-access_log_format = (
-    '%(t)s %(p)s INFO  [GNCRN] %(h)-15s rqt:%(L)s %(s)s %(b)-6s "%(m)s:%(U)s %(q)s" usr:%(u)s "%(f)s" "%(a)s"')
+#access_log_format = (
+#    '%(t)s %(p)s INFO  [GNCRN] %(h)-15s rqt:%(L)s %(s)s %(b)-6s "%(m)s:%(U)s %(q)s" usr:%(u)s "%(f)s" "%(a)s"')
 
 # loki format for easier parsing in grafana
-#access_log_format = (
-#        'time="%(t)s" pid=%(p)s level="INFO" type="[GNCRN]" ip="%(h)-15s" rqt="%(L)s" response_code="%(s)s" response_bytes="%(b)-6s" uri="%(m)s:%(U)s %(q)s" user=":%(u)s" user_agent="%(a)s"')
+access_log_format = (
+    'time="%(t)s" pid=%(p)s level="INFO" type="[GNCRN]" ip="%(h)-15s" rqt="%(L)s" response_code="%(s)s" response_bytes="%(b)-6s" uri="%(m)s:%(U)s %(q)s" user=":%(u)s" user_agent="%(a)s"')
 
 # self adjust workers based on CPU count
 # workers = get_workers()
@@ -97,9 +97,12 b' def post_fork(server, worker):'
         if conf.has_option(section, 'memory_usage_recovery_threshold'):
             _memory_usage_recovery_threshold = conf.getfloat(section, 'memory_usage_recovery_threshold')
 
-    worker._memory_max_usage = _memory_max_usage
-    worker._memory_usage_check_interval = _memory_usage_check_interval
-    worker._memory_usage_recovery_threshold = _memory_usage_recovery_threshold
+    worker._memory_max_usage = int(os.environ.get('RC_GUNICORN_MEMORY_MAX_USAGE', '')
+                                   or _memory_max_usage)
+    worker._memory_usage_check_interval = int(os.environ.get('RC_GUNICORN_MEMORY_USAGE_CHECK_INTERVAL', '')
+                                              or _memory_usage_check_interval)
+    worker._memory_usage_recovery_threshold = float(os.environ.get('RC_GUNICORN_MEMORY_USAGE_RECOVERY_THRESHOLD', '')
+                                                    or _memory_usage_recovery_threshold)
 
     # register memory last check time, with some random offset so we don't recycle all
     # at once

configs/production.ini

@@ -14,7 +14,7 b' port = 9900'
 ; GUNICORN APPLICATION SERVER
 ; ###########################
 
-; run with gunicorn --log-config rhodecode.ini --paste rhodecode.ini
+; run with gunicorn --paste rhodecode.ini
 
 ; Module to use, this setting shouldn't be changed
 use = egg:gunicorn#main
@@ -69,7 +69,7 b' limit_request_field_size = 0'
 ; serving requests. Workers still alive after the timeout (starting from the
 ; receipt of the restart signal) are force killed.
 ; Examples: 1800 (30min), 3600 (1hr), 7200 (2hr), 43200 (12h)
-graceful_timeout = 3600
+graceful_timeout = 21600
 
 # The number of seconds to wait for requests on a Keep-Alive connection.
 # Generally set in the 1-5 seconds range.
@@ -93,16 +93,27 b' memory_usage_recovery_threshold = 0.8'
 [app:main]
 ; The %(here)s variable will be replaced with the absolute path of parent directory
 ; of this file
+; Each option in the app:main can be override by an environmental variable
+;
+;To override an option:
+;
+;RC_<KeyName>
+;Everything should be uppercase, . and - should be replaced by _.
+;For example, if you have these configuration settings:
+;rc_cache.repo_object.backend = foo
+;can be overridden by
+;export RC_CACHE_REPO_OBJECT_BACKEND=foo
+
 use = egg:rhodecode-vcsserver
 
 ; Pyramid default locales, we need this to be set
-pyramid.default_locale_name = en
+#pyramid.default_locale_name = en
 
 ; default locale used by VCS systems
-locale = en_US.UTF-8
+#locale = en_US.UTF-8
 
 ; path to binaries for vcsserver, it should be set by the installer
-; at installation time, e.g /home/user/vcsserver-1/profile/bin
+; at installation time, e.g /home/user/.rccontrol/vcsserver-1/profile/bin
 ; it can also be a path to nix-build output in case of development
 core.binary_dir = ""
 
@@ -116,21 +127,21 b' core.binary_dir = ""'
 
 ; Default cache dir for caches. Putting this into a ramdisk can boost performance.
 ; eg. /tmpfs/data_ramdisk, however this directory might require large amount of space
-cache_dir = %(here)s/data
+#cache_dir = %(here)s/data
 
 ; ***************************************
 ; `repo_object` cache, default file based
 ; ***************************************
 
 ; `repo_object` cache settings for vcs methods for repositories
-rc_cache.repo_object.backend = dogpile.cache.rc.file_namespace
+#rc_cache.repo_object.backend = dogpile.cache.rc.file_namespace
 
 ; cache auto-expires after N seconds
 ; Examples: 86400 (1Day), 604800 (7Days), 1209600 (14Days), 2592000 (30days), 7776000 (90Days)
-rc_cache.repo_object.expiration_time = 2592000
+#rc_cache.repo_object.expiration_time = 2592000
 
 ; file cache store path. Defaults to `cache_dir =` value or tempdir if both values are not set
-#rc_cache.repo_object.arguments.filename = /tmp/vcsserver_cache.db
+#rc_cache.repo_object.arguments.filename = /tmp/vcsserver_cache_repo_object.db
 
 ; ***********************************************************
 ; `repo_object` cache with redis backend
@@ -165,55 +176,59 b' rc_cache.repo_object.expiration_time = 2'
 #statsd.statsd_prefix =
 #statsd.statsd_ipv6 = false
 
+; configure logging automatically at server startup set to false
+; to use the below custom logging config.
+#logging.autoconfigure = true
+
+; specify your own custom logging config file to configure logging
+#logging.logging_conf_file = /path/to/custom_logging.ini
+
 ; #####################
 ; LOGGING CONFIGURATION
 ; #####################
-[loggers]
-keys = root, vcsserver
+#[loggers]
+#keys = root, vcsserver
 
-[handlers]
-keys = console
+#[handlers]
+#keys = console
 
-[formatters]
-keys = generic
+#[formatters]
+#keys = generic
 
 ; #######
 ; LOGGERS
 ; #######
-[logger_root]
-level = NOTSET
-handlers = console
+#[logger_root]
+#level = NOTSET
+#handlers = console
 
-[logger_vcsserver]
-level = DEBUG
-handlers =
-qualname = vcsserver
-propagate = 1
-
+#[logger_vcsserver]
+#level = INFO
+#handlers =
+#qualname = vcsserver
+#propagate = 1
 
 ; ########
 ; HANDLERS
 ; ########
 
-[handler_console]
-class = StreamHandler
-args = (sys.stderr, )
-level = INFO
-formatter = generic
+#[handler_console]
+#class = StreamHandler
+#args = (sys.stderr, )
+#level = INFO
 ; To enable JSON formatted logs replace generic with json
 ; This allows sending properly formatted logs to grafana loki or elasticsearch
 #formatter = json
-
+#formatter = generic
 
 ; ##########
 ; FORMATTERS
 ; ##########
 
-[formatter_generic]
-format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s
-datefmt = %Y-%m-%d %H:%M:%S
+#[formatter_generic]
+#format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s
+#datefmt = %Y-%m-%d %H:%M:%S
 
-[formatter_json]
-format = %(timestamp)s %(levelname)s %(name)s %(message)s %(req_id)s
-class = vcsserver.lib._vendor.jsonlogger.JsonFormatter
-
+#[formatter_json]
+#format = %(timestamp)s %(levelname)s %(name)s %(message)s %(req_id)s
+#class = vcsserver.lib._vendor.jsonlogger.JsonFormatter

vcsserver/http_main.py

@@ -21,21 +21,22 b' import base64'
 import locale
 import logging
 import uuid
+import time
 import wsgiref.util
 import traceback
 import tempfile
 import psutil
+
 from itertools import chain
 from cStringIO import StringIO
 
 import simplejson as json
 import msgpack
 from pyramid.config import Configurator
-from pyramid.settings import asbool, aslist
 from pyramid.wsgi import wsgiapp
 from pyramid.compat import configparser
 from pyramid.response import Response
-
+from vcsserver.config.settings_maker import SettingsMaker
 from vcsserver.utils import safe_int
 from vcsserver.lib.statsd_client import StatsdClient
 
@@ -51,6 +52,7 b' except locale.Error as e:'
         'LOCALE ERROR: failed to set LC_ALL, fallback to LC_ALL=C, org error: %s', e)
     os.environ['LC_ALL'] = 'C'
 
+
 import vcsserver
 from vcsserver import remote_wsgi, scm_app, settings, hgpatches
 from vcsserver.git_lfs.app import GIT_LFS_CONTENT_TYPE, GIT_LFS_PROTO_PAT
@@ -84,40 +86,6 b' def _is_request_chunked(environ):'
     return stream
 
 
-def _int_setting(settings, name, default):
-    settings[name] = int(settings.get(name, default))
-    return settings[name]
-
-
-def _bool_setting(settings, name, default):
-    input_val = settings.get(name, default)
-    if isinstance(input_val, unicode):
-        input_val = input_val.encode('utf8')
-    settings[name] = asbool(input_val)
-    return settings[name]
-
-
-def _list_setting(settings, name, default):
-    raw_value = settings.get(name, default)
-
-    # Otherwise we assume it uses pyramids space/newline separation.
-    settings[name] = aslist(raw_value)
-    return settings[name]
-
-
-def _string_setting(settings, name, default, lower=True, default_when_empty=False):
-    value = settings.get(name, default)
-
-    if default_when_empty and not value:
-        # use default value when value is empty
-        value = default
-
-    if lower:
-        value = value.lower()
-    settings[name] = value
-    return settings[name]
-
-
 def log_max_fd():
     try:
         maxfd = psutil.Process().rlimit(psutil.RLIMIT_NOFILE)[1]
@@ -241,7 +209,6 b' class HTTPApplication(object):'
     _use_echo_app = False
 
     def __init__(self, settings=None, global_config=None):
-        self._sanitize_settings_and_apply_defaults(settings)
 
         self.config = Configurator(settings=settings)
         # Init our statsd at very start
@@ -285,40 +252,6 b' class HTTPApplication(object):'
         vcsserver.PYRAMID_SETTINGS = settings_merged
         vcsserver.CONFIG = settings_merged
 
-    def _sanitize_settings_and_apply_defaults(self, settings):
-        temp_store = tempfile.gettempdir()
-        default_cache_dir = os.path.join(temp_store, 'rc_cache')
-
-        # save default, cache dir, and use it for all backends later.
-        default_cache_dir = _string_setting(
-            settings,
-            'cache_dir',
-            default_cache_dir, lower=False, default_when_empty=True)
-
-        # ensure we have our dir created
-        if not os.path.isdir(default_cache_dir):
-            os.makedirs(default_cache_dir, mode=0o755)
-
-        # exception store cache
-        _string_setting(
-            settings,
-            'exception_tracker.store_path',
-            temp_store, lower=False, default_when_empty=True)
-
-        # repo_object cache
-        _string_setting(
-            settings,
-            'rc_cache.repo_object.backend',
-            'dogpile.cache.rc.file_namespace', lower=False)
-        _int_setting(
-            settings,
-            'rc_cache.repo_object.expiration_time',
-            30 * 24 * 60 * 60)
-        _string_setting(
-            settings,
-            'rc_cache.repo_object.arguments.filename',
-            os.path.join(default_cache_dir, 'vcsserver_cache_1'), lower=False)
-
     def _configure(self):
         self.config.add_renderer(name='msgpack', factory=self._msgpack_renderer_factory)
 
@@ -708,13 +641,110 b' class ResponseFilter(object):'
         return self._start_response(status, headers, exc_info)
 
 
+def sanitize_settings_and_apply_defaults(global_config, settings):
+    global_settings_maker = SettingsMaker(global_config)
+    settings_maker = SettingsMaker(settings)
+
+    settings_maker.make_setting(
+        'logging.autoconfigure',
+        default=True,
+        parser='bool')
+
+    logging_conf = os.path.join(os.path.dirname(global_config.get('__file__')), 'logging.ini')
+    settings_maker.enable_logging(logging_conf)
+
+    # Default includes, possible to change as a user
+    pyramid_includes = settings_maker.make_setting('pyramid.includes', [], parser='list:newline')
+    log.debug(
+        "Using the following pyramid.includes: %s",
+        pyramid_includes)
+
+    settings_maker.make_setting('__file__', global_config.get('__file__'))
+
+    settings_maker.make_setting(
+        'pyramid.default_locale_name',
+        default='en',
+        parser='string')
+    settings_maker.make_setting(
+        'locale',
+        default='en_US.UTF-8',
+        parser='string')
+
+    settings_maker.make_setting(
+        'core.binary_dir',
+        default='',
+        parser='string')
+
+    temp_store = tempfile.gettempdir()
+    default_cache_dir = os.path.join(temp_store, 'rc_cache')
+    # save default, cache dir, and use it for all backends later.
+    default_cache_dir = settings_maker.make_setting(
+        'cache_dir',
+        default=default_cache_dir, default_when_empty=True,
+        parser='dir:ensured')
+
+    # exception store cache
+    settings_maker.make_setting(
+        'exception_tracker.store_path',
+        default=os.path.join(default_cache_dir, 'exc_store'), default_when_empty=True,
+        parser='dir:ensured'
+    )
+
+    # repo_object cache defaults
+    settings_maker.make_setting(
+        'rc_cache.repo_object.backend',
+        default='dogpile.cache.rc.file_namespace',
+        parser='string')
+    settings_maker.make_setting(
+        'rc_cache.repo_object.expiration_time',
+        default=30 * 24 * 60 * 60,  # 30days
+        parser='int')
+    settings_maker. make_setting(
+        'rc_cache.repo_object.arguments.filename',
+        default=os.path.join(default_cache_dir, 'vcsserver_cache_repo_object.db'),
+        parser='string')
+
+    # statsd
+    settings_maker. make_setting(
+        'statsd.enabled',
+        default=False,
+        parser='bool')
+    settings_maker. make_setting(
+        'statsd.statsd_host',
+        default='statsd-exporter',
+        parser='string')
+    settings_maker. make_setting(
+        'statsd.statsd_port',
+        default=9125,
+        parser='int')
+    settings_maker. make_setting(
+        'statsd.statsd_prefix',
+        default='',
+        parser='string')
+    settings_maker. make_setting(
+        'statsd.statsd_ipv6',
+        default=False,
+        parser='bool')
+
+
 def main(global_config, **settings):
+    start_time = time.time()
+    log.info('Pyramid app config starting')
+
     if MercurialFactory:
         hgpatches.patch_largefiles_capabilities()
         hgpatches.patch_subrepo_type_mapping()
 
+    # Fill in and sanitize the defaults & do ENV expansion
+    sanitize_settings_and_apply_defaults(global_config, settings)
+
     # init and bootstrap StatsdClient
     StatsdClient.setup(settings)
 
-    app = HTTPApplication(settings=settings, global_config=global_config)
-    return app.wsgi_app()
+    pyramid_app = HTTPApplication(settings=settings, global_config=global_config).wsgi_app()
+    total_time = time.time() - start_time
+    log.info('Pyramid app `%s` created and configured in %.2fs',
+             getattr(pyramid_app, 'func_name', 'pyramid_app'), total_time)
+    return pyramid_app
+
+