core: re-implemented the way how configuration can be made

- all controllable via ENV variables that can override .ini file settings
- make logging configurable by extension file
- added more defauts and settings validation via settings_make
- update .ini configs and commented out defaults

configs/logging.ini

@@ -0,0 +1,53 b''
+; #####################
+; LOGGING CONFIGURATION
+; #####################
+; Logging template, used for configure the logging
+; some variables here are replaced by RhodeCode to default values
+
+[loggers]
+keys = root, vcsserver
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic, json
+
+; #######
+; LOGGERS
+; #######
+[logger_root]
+level = NOTSET
+handlers = console
+
+[logger_vcsserver]
+level = $RC_LOGGING_LEVEL
+handlers =
+qualname = vcsserver
+propagate = 1
+
+; ########
+; HANDLERS
+; ########
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr, )
+level = $RC_LOGGING_LEVEL
+; To enable JSON formatted logs replace generic with json
+; This allows sending properly formatted logs to grafana loki or elasticsearch
+#formatter = json
+#formatter = generic
+formatter = $RC_LOGGING_FORMATTER
+
+; ##########
+; FORMATTERS
+; ##########
+
+[formatter_generic]
+format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s
+datefmt = %Y-%m-%d %H:%M:%S
+
+[formatter_json]
+format = %(timestamp)s %(levelname)s %(name)s %(message)s %(req_id)s
+class = vcsserver.lib._vendor.jsonlogger.JsonFormatter

vcsserver/config/settings_maker.py

@@ -0,0 +1,177 b''
+# -*- coding: utf-8 -*-
+
+# Copyright (C) 2010-2020 RhodeCode GmbH
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License, version 3
+# (only), as published by the Free Software Foundation.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# This program is dual-licensed. If you wish to learn more about the
+# RhodeCode Enterprise Edition, including its added features, Support services,
+# and proprietary license terms, please see https://rhodecode.com/licenses/
+
+import os
+import textwrap
+import string
+import functools
+import logging
+import tempfile
+import logging.config
+log = logging.getLogger(__name__)
+
+
+def str2bool(_str):
+    """
+    returns True/False value from given string, it tries to translate the
+    string into boolean
+
+    :param _str: string value to translate into boolean
+    :rtype: boolean
+    :returns: boolean from given string
+    """
+    if _str is None:
+        return False
+    if _str in (True, False):
+        return _str
+    _str = str(_str).strip().lower()
+    return _str in ('t', 'true', 'y', 'yes', 'on', '1')
+
+
+def aslist(obj, sep=None, strip=True):
+    """
+    Returns given string separated by sep as list
+
+    :param obj:
+    :param sep:
+    :param strip:
+    """
+    if isinstance(obj, (basestring,)):
+        lst = obj.split(sep)
+        if strip:
+            lst = [v.strip() for v in lst]
+        return lst
+    elif isinstance(obj, (list, tuple)):
+        return obj
+    elif obj is None:
+        return []
+    else:
+        return [obj]
+
+
+class SettingsMaker(object):
+
+    def __init__(self, app_settings):
+        self.settings = app_settings
+
+    @classmethod
+    def _bool_func(cls, input_val):
+        if isinstance(input_val, unicode):
+            input_val = input_val.encode('utf8')
+        return str2bool(input_val)
+
+    @classmethod
+    def _int_func(cls, input_val):
+        return int(input_val)
+
+    @classmethod
+    def _list_func(cls, input_val, sep=','):
+        return aslist(input_val, sep=sep)
+
+    @classmethod
+    def _string_func(cls, input_val, lower=True):
+        if lower:
+            input_val = input_val.lower()
+        return input_val
+
+    @classmethod
+    def _float_func(cls, input_val):
+        return float(input_val)
+
+    @classmethod
+    def _dir_func(cls, input_val, ensure_dir=False, mode=0o755):
+
+        # ensure we have our dir created
+        if not os.path.isdir(input_val) and ensure_dir:
+            os.makedirs(input_val, mode=mode)
+
+        if not os.path.isdir(input_val):
+            raise Exception('Dir at {} does not exist'.format(input_val))
+        return input_val
+
+    @classmethod
+    def _file_path_func(cls, input_val, ensure_dir=False, mode=0o755):
+        dirname = os.path.dirname(input_val)
+        cls._dir_func(dirname, ensure_dir=ensure_dir)
+        return input_val
+
+    @classmethod
+    def _key_transformator(cls, key):
+        return "{}_{}".format('RC'.upper(), key.upper().replace('.', '_').replace('-', '_'))
+
+    def enable_logging(self, logging_conf=None):
+        """
+        Helper to enable debug on running instance
+        :return:
+        """
+        if not str2bool(self.settings.get('logging.autoconfigure')):
+            log.info('logging configuration based on main .ini file')
+            return
+
+        if logging_conf is None:
+            logging_conf = self.settings.get('logging.logging_conf_file') or ''
+
+        if not os.path.isfile(logging_conf):
+            log.error('Unable to setup logging based on %s, file does not exist...', logging_conf)
+            return
+
+        with open(logging_conf, 'rb') as f:
+            ini_template = textwrap.dedent(f.read())
+            ini_template = string.Template(ini_template).safe_substitute(
+                RC_LOGGING_LEVEL=os.environ.get('RC_LOGGING_LEVEL', '') or 'INFO',
+                RC_LOGGING_FORMATTER=os.environ.get('RC_LOGGING_FORMATTER', '') or 'generic'
+            )
+
+        with tempfile.NamedTemporaryFile(prefix='rc_logging_', suffix='.ini', delete=False) as f:
+            log.info('Saved Temporary LOGGING config at %s', f.name)
+            f.write(ini_template)
+
+        logging.config.fileConfig(f.name)
+        os.remove(f.name)
+
+    def make_setting(self, key, default, lower=False, default_when_empty=False, parser=None):
+
+        input_val = self.settings.get(key, default)
+
+        if default_when_empty and not input_val:
+            # use default value when value is set in the config but it is empty
+            input_val = default
+
+        parser_func = {
+            'bool': self._bool_func,
+            'int': self._int_func,
+            'list': self._list_func,
+            'list:newline': functools.partial(self._list_func, sep='/n'),
+            'string': functools.partial(self._string_func, lower=lower),
+            'dir': self._dir_func,
+            'dir:ensured': functools.partial(self._dir_func, ensure_dir=True),
+            'file': self._file_path_func,
+            'file:ensured': functools.partial(self._file_path_func, ensure_dir=True),
+            None: lambda i: i
+        }[parser]
+
+        # now maybe we have this KEY in env, search and use the value with higher priority.
+        transformed_key = self._key_transformator(key)
+        envvar_value = os.environ.get(transformed_key)
+        if envvar_value:
+            log.debug('using `%s` key instead of `%s` key for config', transformed_key, key)
+            input_val = envvar_value
+        self.settings[key] = parser_func(input_val)
+        return self.settings[key]

configs/development.ini

@@ -1,256 +1,271 b''
 ## -*- coding: utf-8 -*-
 
 ; #################################
 ; RHODECODE VCSSERVER CONFIGURATION
 ; #################################
 
 [server:main]
 ; COMMON HOST/IP CONFIG
 host = 0.0.0.0
 port = 9900
 
 ; ##################################################
 ; WAITRESS WSGI SERVER - Recommended for Development
 ; ##################################################
 
 ; use server type
 use = egg:waitress#main
 
 ; number of worker threads
 threads = 5
 
 ; MAX BODY SIZE 100GB
 max_request_body_size = 107374182400
 
 ; Use poll instead of select, fixes file descriptors limits problems.
 ; May not work on old windows systems.
 asyncore_use_poll = true
 
 
 ; ###########################
 ; GUNICORN APPLICATION SERVER
 ; ###########################
 
-; run with gunicorn --log-config rhodecode.ini --paste rhodecode.ini
+; run with gunicorn --paste rhodecode.ini
 
 ; Module to use, this setting shouldn't be changed
 #use = egg:gunicorn#main
 
 ; Sets the number of process workers. More workers means more concurrent connections
 ; RhodeCode can handle at the same time. Each additional worker also it increases
 ; memory usage as each has it's own set of caches.
 ; Recommended value is (2 * NUMBER_OF_CPUS + 1), eg 2CPU = 5 workers, but no more
 ; than 8-10 unless for really big deployments .e.g 700-1000 users.
 ; `instance_id = *` must be set in the [app:main] section below (which is the default)
 ; when using more than 1 worker.
 #workers = 2
 
 ; Gunicorn access log level
 #loglevel = info
 
 ; Process name visible in process list
 #proc_name = rhodecode_vcsserver
 
 ; Type of worker class, one of `sync`, `gevent`
 ; currently `sync` is the only option allowed.
 #worker_class = sync
 
 ; The maximum number of simultaneous clients. Valid only for gevent
 #worker_connections = 10
 
 ; Max number of requests that worker will handle before being gracefully restarted.
 ; Prevents memory leaks, jitter adds variability so not all workers are restarted at once.
 #max_requests = 1000
 #max_requests_jitter = 30
 
 ; Amount of time a worker can spend with handling a request before it
 ; gets killed and restarted. By default set to 21600 (6hrs)
 ; Examples: 1800 (30min), 3600 (1hr), 7200 (2hr), 43200 (12h)
 #timeout = 21600
 
 ; The maximum size of HTTP request line in bytes.
 ; 0 for unlimited
 #limit_request_line = 0
 
 ; Limit the number of HTTP headers fields in a request.
 ; By default this value is 100 and can't be larger than 32768.
 #limit_request_fields = 32768
 
 ; Limit the allowed size of an HTTP request header field.
 ; Value is a positive number or 0.
 ; Setting it to 0 will allow unlimited header field sizes.
 #limit_request_field_size = 0
 
 ; Timeout for graceful workers restart.
 ; After receiving a restart signal, workers have this much time to finish
 ; serving requests. Workers still alive after the timeout (starting from the
 ; receipt of the restart signal) are force killed.
 ; Examples: 1800 (30min), 3600 (1hr), 7200 (2hr), 43200 (12h)
-#graceful_timeout = 3600
+#graceful_timeout = 21600
 
 # The number of seconds to wait for requests on a Keep-Alive connection.
 # Generally set in the 1-5 seconds range.
 #keepalive = 2
 
 ; Maximum memory usage that each worker can use before it will receive a
 ; graceful restart signal 0 = memory monitoring is disabled
 ; Examples: 268435456 (256MB), 536870912 (512MB)
 ; 1073741824 (1GB), 2147483648 (2GB), 4294967296 (4GB)
 #memory_max_usage = 0
 
 ; How often in seconds to check for memory usage for each gunicorn worker
 #memory_usage_check_interval = 60
 
 ; Threshold value for which we don't recycle worker if GarbageCollection
 ; frees up enough resources. Before each restart we try to run GC on worker
 ; in case we get enough free memory after that, restart will not happen.
 #memory_usage_recovery_threshold = 0.8
 
 
 [app:main]
 ; The %(here)s variable will be replaced with the absolute path of parent directory
 ; of this file
+; Each option in the app:main can be override by an environmental variable
+;
+;To override an option:
+;
+;RC_<KeyName>
+;Everything should be uppercase, . and - should be replaced by _.
+;For example, if you have these configuration settings:
+;rc_cache.repo_object.backend = foo
+;can be overridden by
+;export RC_CACHE_REPO_OBJECT_BACKEND=foo
+
 use = egg:rhodecode-vcsserver
 
 
 ; #############
 ; DEBUG OPTIONS
 ; #############
 
 # During development the we want to have the debug toolbar enabled
 pyramid.includes =
     pyramid_debugtoolbar
 
 debugtoolbar.hosts = 0.0.0.0/0
 debugtoolbar.exclude_prefixes =
     /css
     /fonts
     /images
     /js
 
 ; #################
 ; END DEBUG OPTIONS
 ; #################
 
 ; Pyramid default locales, we need this to be set
-pyramid.default_locale_name = en
+#pyramid.default_locale_name = en
 
 ; default locale used by VCS systems
-locale = en_US.UTF-8
+#locale = en_US.UTF-8
 
 ; path to binaries for vcsserver, it should be set by the installer
-; at installation time, e.g /home/user/vcsserver-1/profile/bin
+; at installation time, e.g /home/user/.rccontrol/vcsserver-1/profile/bin
 ; it can also be a path to nix-build output in case of development
 core.binary_dir = ""
 
 ; Custom exception store path, defaults to TMPDIR
 ; This is used to store exception from RhodeCode in shared directory
 #exception_tracker.store_path =
 
 ; #############
 ; DOGPILE CACHE
 ; #############
 
 ; Default cache dir for caches. Putting this into a ramdisk can boost performance.
 ; eg. /tmpfs/data_ramdisk, however this directory might require large amount of space
-cache_dir = %(here)s/data
+#cache_dir = %(here)s/data
 
 ; ***************************************
 ; `repo_object` cache, default file based
 ; ***************************************
 
 ; `repo_object` cache settings for vcs methods for repositories
-rc_cache.repo_object.backend = dogpile.cache.rc.file_namespace
+#rc_cache.repo_object.backend = dogpile.cache.rc.file_namespace
 
 ; cache auto-expires after N seconds
 ; Examples: 86400 (1Day), 604800 (7Days), 1209600 (14Days), 2592000 (30days), 7776000 (90Days)
-rc_cache.repo_object.expiration_time = 2592000
+#rc_cache.repo_object.expiration_time = 2592000
 
 ; file cache store path. Defaults to `cache_dir =` value or tempdir if both values are not set
-#rc_cache.repo_object.arguments.filename = /tmp/vcsserver_cache.db
+#rc_cache.repo_object.arguments.filename = /tmp/vcsserver_cache_repo_object.db
 
 ; ***********************************************************
 ; `repo_object` cache with redis backend
 ; recommended for larger instance, and for better performance
 ; ***********************************************************
 
 ; `repo_object` cache settings for vcs methods for repositories
 #rc_cache.repo_object.backend = dogpile.cache.rc.redis_msgpack
 
 ; cache auto-expires after N seconds
 ; Examples: 86400 (1Day), 604800 (7Days), 1209600 (14Days), 2592000 (30days), 7776000 (90Days)
 #rc_cache.repo_object.expiration_time = 2592000
 
 ; redis_expiration_time needs to be greater then expiration_time
 #rc_cache.repo_object.arguments.redis_expiration_time = 3592000
 
 #rc_cache.repo_object.arguments.host = localhost
 #rc_cache.repo_object.arguments.port = 6379
 #rc_cache.repo_object.arguments.db = 5
 #rc_cache.repo_object.arguments.socket_timeout = 30
 ; more Redis options: https://dogpilecache.sqlalchemy.org/en/latest/api.html#redis-backends
 #rc_cache.repo_object.arguments.distributed_lock = true
 
 ; auto-renew lock to prevent stale locks, slower but safer. Use only if problems happen
 #rc_cache.repo_object.arguments.lock_auto_renewal = true
 
 ; Statsd client config, this is used to send metrics to statsd
 ; We recommend setting statsd_exported and scrape them using Promethues
 #statsd.enabled = false
 #statsd.statsd_host = 0.0.0.0
 #statsd.statsd_port = 8125
 #statsd.statsd_prefix =
 #statsd.statsd_ipv6 = false
 
+; configure logging automatically at server startup set to false
+; to use the below custom logging config.
+#logging.autoconfigure = true
+
+; specify your own custom logging config file to configure logging
+#logging.logging_conf_file = /path/to/custom_logging.ini
+
 ; #####################
 ; LOGGING CONFIGURATION
 ; #####################
-[loggers]
-keys = root, vcsserver
+#[loggers]
+#keys = root, vcsserver
 
-[handlers]
-keys = console
+#[handlers]
+#keys = console
 
-[formatters]
-keys = generic
+#[formatters]
+#keys = generic
 
 ; #######
 ; LOGGERS
 ; #######
-[logger_root]
-level = NOTSET
-handlers = console
+#[logger_root]
+#level = NOTSET
+#handlers = console
 
-[logger_vcsserver]
-level = DEBUG
-handlers =
-qualname = vcsserver
-propagate = 1
-
+#[logger_vcsserver]
+#level = INFO
+#handlers =
+#qualname = vcsserver
+#propagate = 1
 
 ; ########
 ; HANDLERS
 ; ########
 
-[handler_console]
-class = StreamHandler
-args = (sys.stderr, )
-level = DEBUG
-formatter = generic
+#[handler_console]
+#class = StreamHandler
+#args = (sys.stderr, )
+#level = INFO
 ; To enable JSON formatted logs replace generic with json
 ; This allows sending properly formatted logs to grafana loki or elasticsearch
 #formatter = json
-
+#formatter = generic
 
 ; ##########
 ; FORMATTERS
 ; ##########
 
-[formatter_generic]
-format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s
-datefmt = %Y-%m-%d %H:%M:%S
+#[formatter_generic]
+#format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s
+#datefmt = %Y-%m-%d %H:%M:%S
 
-[formatter_json]
-format = %(timestamp)s %(levelname)s %(name)s %(message)s %(req_id)s
-class = vcsserver.lib._vendor.jsonlogger.JsonFormatter
-
+#[formatter_json]
+#format = %(timestamp)s %(levelname)s %(name)s %(message)s %(req_id)s
+#class = vcsserver.lib._vendor.jsonlogger.JsonFormatter

configs/gunicorn_config.py

@@ -1,269 +1,272 b''
 """
 Gunicorn config extension and hooks. This config file adds some extra settings and memory management.
 Gunicorn configuration should be managed by .ini files entries of RhodeCode or VCSServer
 """
 
 import gc
 import os
 import sys
 import math
 import time
 import threading
 import traceback
 import random
 from gunicorn.glogging import Logger
 
 
 def get_workers():
     import multiprocessing
     return multiprocessing.cpu_count() * 2 + 1
 
 # GLOBAL
 errorlog = '-'
 accesslog = '-'
 
 
 # SERVER MECHANICS
 # None == system temp dir
 # worker_tmp_dir is recommended to be set to some tmpfs
 worker_tmp_dir = None
 tmp_upload_dir = None
 
 # Custom log format
-access_log_format = (
-    '%(t)s %(p)s INFO  [GNCRN] %(h)-15s rqt:%(L)s %(s)s %(b)-6s "%(m)s:%(U)s %(q)s" usr:%(u)s "%(f)s" "%(a)s"')
+#access_log_format = (
+#    '%(t)s %(p)s INFO  [GNCRN] %(h)-15s rqt:%(L)s %(s)s %(b)-6s "%(m)s:%(U)s %(q)s" usr:%(u)s "%(f)s" "%(a)s"')
 
 # loki format for easier parsing in grafana
-#access_log_format = (
-#        'time="%(t)s" pid=%(p)s level="INFO" type="[GNCRN]" ip="%(h)-15s" rqt="%(L)s" response_code="%(s)s" response_bytes="%(b)-6s" uri="%(m)s:%(U)s %(q)s" user=":%(u)s" user_agent="%(a)s"')
+access_log_format = (
+    'time="%(t)s" pid=%(p)s level="INFO" type="[GNCRN]" ip="%(h)-15s" rqt="%(L)s" response_code="%(s)s" response_bytes="%(b)-6s" uri="%(m)s:%(U)s %(q)s" user=":%(u)s" user_agent="%(a)s"')
 
 # self adjust workers based on CPU count
 # workers = get_workers()
 
 
 def _get_process_rss(pid=None):
     try:
         import psutil
         if pid:
             proc = psutil.Process(pid)
         else:
             proc = psutil.Process()
         return proc.memory_info().rss
     except Exception:
         return None
 
 
 def _get_config(ini_path):
 
     try:
         import configparser
     except ImportError:
         import ConfigParser as configparser
     try:
         config = configparser.RawConfigParser()
         config.read(ini_path)
         return config
     except Exception:
         return None
 
 
 def _time_with_offset(memory_usage_check_interval):
     return time.time() - random.randint(0, memory_usage_check_interval/2.0)
 
 
 def pre_fork(server, worker):
     pass
 
 
 def post_fork(server, worker):
 
     # memory spec defaults
     _memory_max_usage = 0
     _memory_usage_check_interval = 60
     _memory_usage_recovery_threshold = 0.8
 
     ini_path = os.path.abspath(server.cfg.paste)
     conf = _get_config(ini_path)
 
     section = 'server:main'
     if conf and conf.has_section(section):
 
         if conf.has_option(section, 'memory_max_usage'):
             _memory_max_usage = conf.getint(section, 'memory_max_usage')
 
         if conf.has_option(section, 'memory_usage_check_interval'):
             _memory_usage_check_interval = conf.getint(section, 'memory_usage_check_interval')
 
         if conf.has_option(section, 'memory_usage_recovery_threshold'):
             _memory_usage_recovery_threshold = conf.getfloat(section, 'memory_usage_recovery_threshold')
 
-    worker._memory_max_usage = _memory_max_usage
-    worker._memory_usage_check_interval = _memory_usage_check_interval
-    worker._memory_usage_recovery_threshold = _memory_usage_recovery_threshold
+    worker._memory_max_usage = int(os.environ.get('RC_GUNICORN_MEMORY_MAX_USAGE', '')
+                                   or _memory_max_usage)
+    worker._memory_usage_check_interval = int(os.environ.get('RC_GUNICORN_MEMORY_USAGE_CHECK_INTERVAL', '')
+                                              or _memory_usage_check_interval)
+    worker._memory_usage_recovery_threshold = float(os.environ.get('RC_GUNICORN_MEMORY_USAGE_RECOVERY_THRESHOLD', '')
+                                                    or _memory_usage_recovery_threshold)
 
     # register memory last check time, with some random offset so we don't recycle all
     # at once
     worker._last_memory_check_time = _time_with_offset(_memory_usage_check_interval)
 
     if _memory_max_usage:
         server.log.info("[%-10s] WORKER spawned with max memory set at %s", worker.pid,
                         _format_data_size(_memory_max_usage))
     else:
         server.log.info("[%-10s] WORKER spawned", worker.pid)
 
 
 def pre_exec(server):
     server.log.info("Forked child, re-executing.")
 
 
 def on_starting(server):
     server_lbl = '{} {}'.format(server.proc_name, server.address)
     server.log.info("Server %s is starting.", server_lbl)
 
 
 def when_ready(server):
     server.log.info("Server %s is ready. Spawning workers", server)
 
 
 def on_reload(server):
     pass
 
 
 def _format_data_size(size, unit="B", precision=1, binary=True):
     """Format a number using SI units (kilo, mega, etc.).
 
     ``size``: The number as a float or int.
 
     ``unit``: The unit name in plural form. Examples: "bytes", "B".
 
     ``precision``: How many digits to the right of the decimal point. Default
     is 1.  0 suppresses the decimal point.
 
     ``binary``: If false, use base-10 decimal prefixes (kilo = K = 1000).
     If true, use base-2 binary prefixes (kibi = Ki = 1024).
 
     ``full_name``: If false (default), use the prefix abbreviation ("k" or
     "Ki").  If true, use the full prefix ("kilo" or "kibi"). If false,
     use abbreviation ("k" or "Ki").
 
     """
 
     if not binary:
         base = 1000
         multiples = ('', 'k', 'M', 'G', 'T', 'P', 'E', 'Z', 'Y')
     else:
         base = 1024
         multiples = ('', 'Ki', 'Mi', 'Gi', 'Ti', 'Pi', 'Ei', 'Zi', 'Yi')
 
     sign = ""
     if size > 0:
         m = int(math.log(size, base))
     elif size < 0:
         sign = "-"
         size = -size
         m = int(math.log(size, base))
     else:
         m = 0
     if m > 8:
         m = 8
 
     if m == 0:
         precision = '%.0f'
     else:
         precision = '%%.%df' % precision
 
     size = precision % (size / math.pow(base, m))
 
     return '%s%s %s%s' % (sign, size.strip(), multiples[m], unit)
 
 
 def _check_memory_usage(worker):
     memory_max_usage = worker._memory_max_usage
     if not memory_max_usage:
         return
 
     memory_usage_check_interval = worker._memory_usage_check_interval
     memory_usage_recovery_threshold = memory_max_usage * worker._memory_usage_recovery_threshold
 
     elapsed = time.time() - worker._last_memory_check_time
     if elapsed > memory_usage_check_interval:
         mem_usage = _get_process_rss()
         if mem_usage and mem_usage > memory_max_usage:
             worker.log.info(
                 "memory usage %s > %s, forcing gc",
                 _format_data_size(mem_usage), _format_data_size(memory_max_usage))
             # Try to clean it up by forcing a full collection.
             gc.collect()
             mem_usage = _get_process_rss()
             if mem_usage > memory_usage_recovery_threshold:
                 # Didn't clean up enough, we'll have to terminate.
                 worker.log.warning(
                     "memory usage %s > %s after gc, quitting",
                     _format_data_size(mem_usage), _format_data_size(memory_max_usage))
                 # This will cause worker to auto-restart itself
                 worker.alive = False
         worker._last_memory_check_time = time.time()
 
 
 def worker_int(worker):
     worker.log.info("[%-10s] worker received INT or QUIT signal", worker.pid)
 
     # get traceback info, on worker crash
     id2name = dict([(th.ident, th.name) for th in threading.enumerate()])
     code = []
     for thread_id, stack in sys._current_frames().items():
         code.append(
             "\n# Thread: %s(%d)" % (id2name.get(thread_id, ""), thread_id))
         for fname, lineno, name, line in traceback.extract_stack(stack):
             code.append('File: "%s", line %d, in %s' % (fname, lineno, name))
             if line:
                 code.append("  %s" % (line.strip()))
     worker.log.debug("\n".join(code))
 
 
 def worker_abort(worker):
     worker.log.info("[%-10s] worker received SIGABRT signal", worker.pid)
 
 
 def worker_exit(server, worker):
     worker.log.info("[%-10s] worker exit", worker.pid)
 
 
 def child_exit(server, worker):
     worker.log.info("[%-10s] worker child exit", worker.pid)
 
 
 def pre_request(worker, req):
     worker.start_time = time.time()
     worker.log.debug(
         "GNCRN PRE  WORKER [cnt:%s]: %s %s", worker.nr, req.method, req.path)
 
 
 def post_request(worker, req, environ, resp):
     total_time = time.time() - worker.start_time
     # Gunicorn sometimes has problems with reading the status_code
     status_code = getattr(resp, 'status_code', '')
     worker.log.debug(
         "GNCRN POST WORKER [cnt:%s]: %s %s resp: %s, Load Time: %.4fs",
         worker.nr, req.method, req.path, status_code, total_time)
     _check_memory_usage(worker)
 
 
 class RhodeCodeLogger(Logger):
     """
     Custom Logger that allows some customization that gunicorn doesn't allow
     """
 
     datefmt = r"%Y-%m-%d %H:%M:%S"
 
     def __init__(self, cfg):
         Logger.__init__(self, cfg)
 
     def now(self):
         """ return date in RhodeCode Log format """
         now = time.time()
         msecs = int((now - long(now)) * 1000)
         return time.strftime(self.datefmt, time.localtime(now)) + '.{0:03d}'.format(msecs)
 
 
 logger_class = RhodeCodeLogger

configs/production.ini

@@ -1,219 +1,234 b''
 ## -*- coding: utf-8 -*-
 
 ; #################################
 ; RHODECODE VCSSERVER CONFIGURATION
 ; #################################
 
 [server:main]
 ; COMMON HOST/IP CONFIG
 host = 127.0.0.1
 port = 9900
 
 
 ; ###########################
 ; GUNICORN APPLICATION SERVER
 ; ###########################
 
-; run with gunicorn --log-config rhodecode.ini --paste rhodecode.ini
+; run with gunicorn --paste rhodecode.ini
 
 ; Module to use, this setting shouldn't be changed
 use = egg:gunicorn#main
 
 ; Sets the number of process workers. More workers means more concurrent connections
 ; RhodeCode can handle at the same time. Each additional worker also it increases
 ; memory usage as each has it's own set of caches.
 ; Recommended value is (2 * NUMBER_OF_CPUS + 1), eg 2CPU = 5 workers, but no more
 ; than 8-10 unless for really big deployments .e.g 700-1000 users.
 ; `instance_id = *` must be set in the [app:main] section below (which is the default)
 ; when using more than 1 worker.
 workers = 2
 
 ; Gunicorn access log level
 loglevel = info
 
 ; Process name visible in process list
 proc_name = rhodecode_vcsserver
 
 ; Type of worker class, one of `sync`, `gevent`
 ; currently `sync` is the only option allowed.
 worker_class = sync
 
 ; The maximum number of simultaneous clients. Valid only for gevent
 worker_connections = 10
 
 ; Max number of requests that worker will handle before being gracefully restarted.
 ; Prevents memory leaks, jitter adds variability so not all workers are restarted at once.
 max_requests = 1000
 max_requests_jitter = 30
 
 ; Amount of time a worker can spend with handling a request before it
 ; gets killed and restarted. By default set to 21600 (6hrs)
 ; Examples: 1800 (30min), 3600 (1hr), 7200 (2hr), 43200 (12h)
 timeout = 21600
 
 ; The maximum size of HTTP request line in bytes.
 ; 0 for unlimited
 limit_request_line = 0
 
 ; Limit the number of HTTP headers fields in a request.
 ; By default this value is 100 and can't be larger than 32768.
 limit_request_fields = 32768
 
 ; Limit the allowed size of an HTTP request header field.
 ; Value is a positive number or 0.
 ; Setting it to 0 will allow unlimited header field sizes.
 limit_request_field_size = 0
 
 ; Timeout for graceful workers restart.
 ; After receiving a restart signal, workers have this much time to finish
 ; serving requests. Workers still alive after the timeout (starting from the
 ; receipt of the restart signal) are force killed.
 ; Examples: 1800 (30min), 3600 (1hr), 7200 (2hr), 43200 (12h)
-graceful_timeout = 3600
+graceful_timeout = 21600
 
 # The number of seconds to wait for requests on a Keep-Alive connection.
 # Generally set in the 1-5 seconds range.
 keepalive = 2
 
 ; Maximum memory usage that each worker can use before it will receive a
 ; graceful restart signal 0 = memory monitoring is disabled
 ; Examples: 268435456 (256MB), 536870912 (512MB)
 ; 1073741824 (1GB), 2147483648 (2GB), 4294967296 (4GB)
 memory_max_usage = 0
 
 ; How often in seconds to check for memory usage for each gunicorn worker
 memory_usage_check_interval = 60
 
 ; Threshold value for which we don't recycle worker if GarbageCollection
 ; frees up enough resources. Before each restart we try to run GC on worker
 ; in case we get enough free memory after that, restart will not happen.
 memory_usage_recovery_threshold = 0.8
 
 
 [app:main]
 ; The %(here)s variable will be replaced with the absolute path of parent directory
 ; of this file
+; Each option in the app:main can be override by an environmental variable
+;
+;To override an option:
+;
+;RC_<KeyName>
+;Everything should be uppercase, . and - should be replaced by _.
+;For example, if you have these configuration settings:
+;rc_cache.repo_object.backend = foo
+;can be overridden by
+;export RC_CACHE_REPO_OBJECT_BACKEND=foo
+
 use = egg:rhodecode-vcsserver
 
 ; Pyramid default locales, we need this to be set
-pyramid.default_locale_name = en
+#pyramid.default_locale_name = en
 
 ; default locale used by VCS systems
-locale = en_US.UTF-8
+#locale = en_US.UTF-8
 
 ; path to binaries for vcsserver, it should be set by the installer
-; at installation time, e.g /home/user/vcsserver-1/profile/bin
+; at installation time, e.g /home/user/.rccontrol/vcsserver-1/profile/bin
 ; it can also be a path to nix-build output in case of development
 core.binary_dir = ""
 
 ; Custom exception store path, defaults to TMPDIR
 ; This is used to store exception from RhodeCode in shared directory
 #exception_tracker.store_path =
 
 ; #############
 ; DOGPILE CACHE
 ; #############
 
 ; Default cache dir for caches. Putting this into a ramdisk can boost performance.
 ; eg. /tmpfs/data_ramdisk, however this directory might require large amount of space
-cache_dir = %(here)s/data
+#cache_dir = %(here)s/data
 
 ; ***************************************
 ; `repo_object` cache, default file based
 ; ***************************************
 
 ; `repo_object` cache settings for vcs methods for repositories
-rc_cache.repo_object.backend = dogpile.cache.rc.file_namespace
+#rc_cache.repo_object.backend = dogpile.cache.rc.file_namespace
 
 ; cache auto-expires after N seconds
 ; Examples: 86400 (1Day), 604800 (7Days), 1209600 (14Days), 2592000 (30days), 7776000 (90Days)
-rc_cache.repo_object.expiration_time = 2592000
+#rc_cache.repo_object.expiration_time = 2592000
 
 ; file cache store path. Defaults to `cache_dir =` value or tempdir if both values are not set
-#rc_cache.repo_object.arguments.filename = /tmp/vcsserver_cache.db
+#rc_cache.repo_object.arguments.filename = /tmp/vcsserver_cache_repo_object.db
 
 ; ***********************************************************
 ; `repo_object` cache with redis backend
 ; recommended for larger instance, and for better performance
 ; ***********************************************************
 
 ; `repo_object` cache settings for vcs methods for repositories
 #rc_cache.repo_object.backend = dogpile.cache.rc.redis_msgpack
 
 ; cache auto-expires after N seconds
 ; Examples: 86400 (1Day), 604800 (7Days), 1209600 (14Days), 2592000 (30days), 7776000 (90Days)
 #rc_cache.repo_object.expiration_time = 2592000
 
 ; redis_expiration_time needs to be greater then expiration_time
 #rc_cache.repo_object.arguments.redis_expiration_time = 3592000
 
 #rc_cache.repo_object.arguments.host = localhost
 #rc_cache.repo_object.arguments.port = 6379
 #rc_cache.repo_object.arguments.db = 5
 #rc_cache.repo_object.arguments.socket_timeout = 30
 ; more Redis options: https://dogpilecache.sqlalchemy.org/en/latest/api.html#redis-backends
 #rc_cache.repo_object.arguments.distributed_lock = true
 
 ; auto-renew lock to prevent stale locks, slower but safer. Use only if problems happen
 #rc_cache.repo_object.arguments.lock_auto_renewal = true
 
 ; Statsd client config, this is used to send metrics to statsd
 ; We recommend setting statsd_exported and scrape them using Promethues
 #statsd.enabled = false
 #statsd.statsd_host = 0.0.0.0
 #statsd.statsd_port = 8125
 #statsd.statsd_prefix =
 #statsd.statsd_ipv6 = false
 
+; configure logging automatically at server startup set to false
+; to use the below custom logging config.
+#logging.autoconfigure = true
+
+; specify your own custom logging config file to configure logging
+#logging.logging_conf_file = /path/to/custom_logging.ini
+
 ; #####################
 ; LOGGING CONFIGURATION
 ; #####################
-[loggers]
-keys = root, vcsserver
+#[loggers]
+#keys = root, vcsserver
 
-[handlers]
-keys = console
+#[handlers]
+#keys = console
 
-[formatters]
-keys = generic
+#[formatters]
+#keys = generic
 
 ; #######
 ; LOGGERS
 ; #######
-[logger_root]
-level = NOTSET
-handlers = console
+#[logger_root]
+#level = NOTSET
+#handlers = console
 
-[logger_vcsserver]
-level = DEBUG
-handlers =
-qualname = vcsserver
-propagate = 1
-
+#[logger_vcsserver]
+#level = INFO
+#handlers =
+#qualname = vcsserver
+#propagate = 1
 
 ; ########
 ; HANDLERS
 ; ########
 
-[handler_console]
-class = StreamHandler
-args = (sys.stderr, )
-level = INFO
-formatter = generic
+#[handler_console]
+#class = StreamHandler
+#args = (sys.stderr, )
+#level = INFO
 ; To enable JSON formatted logs replace generic with json
 ; This allows sending properly formatted logs to grafana loki or elasticsearch
 #formatter = json
-
+#formatter = generic
 
 ; ##########
 ; FORMATTERS
 ; ##########
 
-[formatter_generic]
-format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s
-datefmt = %Y-%m-%d %H:%M:%S
+#[formatter_generic]
+#format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s
+#datefmt = %Y-%m-%d %H:%M:%S
 
-[formatter_json]
-format = %(timestamp)s %(levelname)s %(name)s %(message)s %(req_id)s
-class = vcsserver.lib._vendor.jsonlogger.JsonFormatter
-
+#[formatter_json]
+#format = %(timestamp)s %(levelname)s %(name)s %(message)s %(req_id)s
+#class = vcsserver.lib._vendor.jsonlogger.JsonFormatter

vcsserver/http_main.py

@@ -1,720 +1,750 b''
 # RhodeCode VCSServer provides access to different vcs backends via network.
 # Copyright (C) 2014-2020 RhodeCode GmbH
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation; either version 3 of the License, or
 # (at your option) any later version.
 #
 # This program is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
 
 import os
 import sys
 import base64
 import locale
 import logging
 import uuid
+import time
 import wsgiref.util
 import traceback
 import tempfile
 import psutil
+
 from itertools import chain
 from cStringIO import StringIO
 
 import simplejson as json
 import msgpack
 from pyramid.config import Configurator
-from pyramid.settings import asbool, aslist
 from pyramid.wsgi import wsgiapp
 from pyramid.compat import configparser
 from pyramid.response import Response
-
+from vcsserver.config.settings_maker import SettingsMaker
 from vcsserver.utils import safe_int
 from vcsserver.lib.statsd_client import StatsdClient
 
 log = logging.getLogger(__name__)
 
 # due to Mercurial/glibc2.27 problems we need to detect if locale settings are
 # causing problems and "fix" it in case they do and fallback to LC_ALL = C
 
 try:
     locale.setlocale(locale.LC_ALL, '')
 except locale.Error as e:
     log.error(
         'LOCALE ERROR: failed to set LC_ALL, fallback to LC_ALL=C, org error: %s', e)
     os.environ['LC_ALL'] = 'C'
 
+
 import vcsserver
 from vcsserver import remote_wsgi, scm_app, settings, hgpatches
 from vcsserver.git_lfs.app import GIT_LFS_CONTENT_TYPE, GIT_LFS_PROTO_PAT
 from vcsserver.echo_stub import remote_wsgi as remote_wsgi_stub
 from vcsserver.echo_stub.echo_app import EchoApp
 from vcsserver.exceptions import HTTPRepoLocked, HTTPRepoBranchProtected
 from vcsserver.lib.exc_tracking import store_exception
 from vcsserver.server import VcsServer
 
 try:
     from vcsserver.git import GitFactory, GitRemote
 except ImportError:
     GitFactory = None
     GitRemote = None
 
 try:
     from vcsserver.hg import MercurialFactory, HgRemote
 except ImportError:
     MercurialFactory = None
     HgRemote = None
 
 try:
     from vcsserver.svn import SubversionFactory, SvnRemote
 except ImportError:
     SubversionFactory = None
     SvnRemote = None
 
 
 def _is_request_chunked(environ):
     stream = environ.get('HTTP_TRANSFER_ENCODING', '') == 'chunked'
     return stream
 
 
-def _int_setting(settings, name, default):
-    settings[name] = int(settings.get(name, default))
-    return settings[name]
-
-
-def _bool_setting(settings, name, default):
-    input_val = settings.get(name, default)
-    if isinstance(input_val, unicode):
-        input_val = input_val.encode('utf8')
-    settings[name] = asbool(input_val)
-    return settings[name]
-
-
-def _list_setting(settings, name, default):
-    raw_value = settings.get(name, default)
-
-    # Otherwise we assume it uses pyramids space/newline separation.
-    settings[name] = aslist(raw_value)
-    return settings[name]
-
-
-def _string_setting(settings, name, default, lower=True, default_when_empty=False):
-    value = settings.get(name, default)
-
-    if default_when_empty and not value:
-        # use default value when value is empty
-        value = default
-
-    if lower:
-        value = value.lower()
-    settings[name] = value
-    return settings[name]
-
-
 def log_max_fd():
     try:
         maxfd = psutil.Process().rlimit(psutil.RLIMIT_NOFILE)[1]
         log.info('Max file descriptors value: %s', maxfd)
     except Exception:
         pass
 
 
 class VCS(object):
     def __init__(self, locale_conf=None, cache_config=None):
         self.locale = locale_conf
         self.cache_config = cache_config
         self._configure_locale()
 
         log_max_fd()
 
         if GitFactory and GitRemote:
             git_factory = GitFactory()
             self._git_remote = GitRemote(git_factory)
         else:
             log.info("Git client import failed")
 
         if MercurialFactory and HgRemote:
             hg_factory = MercurialFactory()
             self._hg_remote = HgRemote(hg_factory)
         else:
             log.info("Mercurial client import failed")
 
         if SubversionFactory and SvnRemote:
             svn_factory = SubversionFactory()
 
             # hg factory is used for svn url validation
             hg_factory = MercurialFactory()
             self._svn_remote = SvnRemote(svn_factory, hg_factory=hg_factory)
         else:
             log.warning("Subversion client import failed")
 
         self._vcsserver = VcsServer()
 
     def _configure_locale(self):
         if self.locale:
             log.info('Settings locale: `LC_ALL` to %s', self.locale)
         else:
             log.info(
                 'Configuring locale subsystem based on environment variables')
         try:
             # If self.locale is the empty string, then the locale
             # module will use the environment variables. See the
             # documentation of the package `locale`.
             locale.setlocale(locale.LC_ALL, self.locale)
 
             language_code, encoding = locale.getlocale()
             log.info(
                 'Locale set to language code "%s" with encoding "%s".',
                 language_code, encoding)
         except locale.Error:
             log.exception(
                 'Cannot set locale, not configuring the locale system')
 
 
 class WsgiProxy(object):
     def __init__(self, wsgi):
         self.wsgi = wsgi
 
     def __call__(self, environ, start_response):
         input_data = environ['wsgi.input'].read()
         input_data = msgpack.unpackb(input_data)
 
         error = None
         try:
             data, status, headers = self.wsgi.handle(
                 input_data['environment'], input_data['input_data'],
                 *input_data['args'], **input_data['kwargs'])
         except Exception as e:
             data, status, headers = [], None, None
             error = {
                 'message': str(e),
                 '_vcs_kind': getattr(e, '_vcs_kind', None)
             }
 
         start_response(200, {})
         return self._iterator(error, status, headers, data)
 
     def _iterator(self, error, status, headers, data):
         initial_data = [
             error,
             status,
             headers,
         ]
 
         for d in chain(initial_data, data):
             yield msgpack.packb(d)
 
 
 def not_found(request):
     return {'status': '404 NOT FOUND'}
 
 
 class VCSViewPredicate(object):
     def __init__(self, val, config):
         self.remotes = val
 
     def text(self):
         return 'vcs view method = %s' % (self.remotes.keys(),)
 
     phash = text
 
     def __call__(self, context, request):
         """
         View predicate that returns true if given backend is supported by
         defined remotes.
         """
         backend = request.matchdict.get('backend')
         return backend in self.remotes
 
 
 class HTTPApplication(object):
     ALLOWED_EXCEPTIONS = ('KeyError', 'URLError')
 
     remote_wsgi = remote_wsgi
     _use_echo_app = False
 
     def __init__(self, settings=None, global_config=None):
-        self._sanitize_settings_and_apply_defaults(settings)
 
         self.config = Configurator(settings=settings)
         # Init our statsd at very start
         self.config.registry.statsd = StatsdClient.statsd
 
         self.global_config = global_config
         self.config.include('vcsserver.lib.rc_cache')
 
         settings_locale = settings.get('locale', '') or 'en_US.UTF-8'
         vcs = VCS(locale_conf=settings_locale, cache_config=settings)
         self._remotes = {
             'hg': vcs._hg_remote,
             'git': vcs._git_remote,
             'svn': vcs._svn_remote,
             'server': vcs._vcsserver,
         }
         if settings.get('dev.use_echo_app', 'false').lower() == 'true':
             self._use_echo_app = True
             log.warning("Using EchoApp for VCS operations.")
             self.remote_wsgi = remote_wsgi_stub
 
         self._configure_settings(global_config, settings)
 
         self._configure()
 
     def _configure_settings(self, global_config, app_settings):
         """
         Configure the settings module.
         """
         settings_merged = global_config.copy()
         settings_merged.update(app_settings)
 
         git_path = app_settings.get('git_path', None)
         if git_path:
             settings.GIT_EXECUTABLE = git_path
         binary_dir = app_settings.get('core.binary_dir', None)
         if binary_dir:
             settings.BINARY_DIR = binary_dir
 
         # Store the settings to make them available to other modules.
         vcsserver.PYRAMID_SETTINGS = settings_merged
         vcsserver.CONFIG = settings_merged
 
-    def _sanitize_settings_and_apply_defaults(self, settings):
-        temp_store = tempfile.gettempdir()
-        default_cache_dir = os.path.join(temp_store, 'rc_cache')
-
-        # save default, cache dir, and use it for all backends later.
-        default_cache_dir = _string_setting(
-            settings,
-            'cache_dir',
-            default_cache_dir, lower=False, default_when_empty=True)
-
-        # ensure we have our dir created
-        if not os.path.isdir(default_cache_dir):
-            os.makedirs(default_cache_dir, mode=0o755)
-
-        # exception store cache
-        _string_setting(
-            settings,
-            'exception_tracker.store_path',
-            temp_store, lower=False, default_when_empty=True)
-
-        # repo_object cache
-        _string_setting(
-            settings,
-            'rc_cache.repo_object.backend',
-            'dogpile.cache.rc.file_namespace', lower=False)
-        _int_setting(
-            settings,
-            'rc_cache.repo_object.expiration_time',
-            30 * 24 * 60 * 60)
-        _string_setting(
-            settings,
-            'rc_cache.repo_object.arguments.filename',
-            os.path.join(default_cache_dir, 'vcsserver_cache_1'), lower=False)
-
     def _configure(self):
         self.config.add_renderer(name='msgpack', factory=self._msgpack_renderer_factory)
 
         self.config.add_route('service', '/_service')
         self.config.add_route('status', '/status')
         self.config.add_route('hg_proxy', '/proxy/hg')
         self.config.add_route('git_proxy', '/proxy/git')
 
         # rpc methods
         self.config.add_route('vcs', '/{backend}')
 
         # streaming rpc remote methods
         self.config.add_route('vcs_stream', '/{backend}/stream')
 
         # vcs operations clone/push as streaming
         self.config.add_route('stream_git', '/stream/git/*repo_name')
         self.config.add_route('stream_hg', '/stream/hg/*repo_name')
 
         self.config.add_view(self.status_view, route_name='status', renderer='json')
         self.config.add_view(self.service_view, route_name='service', renderer='msgpack')
 
         self.config.add_view(self.hg_proxy(), route_name='hg_proxy')
         self.config.add_view(self.git_proxy(), route_name='git_proxy')
         self.config.add_view(self.vcs_view, route_name='vcs', renderer='msgpack',
                              vcs_view=self._remotes)
         self.config.add_view(self.vcs_stream_view, route_name='vcs_stream',
                              vcs_view=self._remotes)
 
         self.config.add_view(self.hg_stream(), route_name='stream_hg')
         self.config.add_view(self.git_stream(), route_name='stream_git')
 
         self.config.add_view_predicate('vcs_view', VCSViewPredicate)
 
         self.config.add_notfound_view(not_found, renderer='json')
 
         self.config.add_view(self.handle_vcs_exception, context=Exception)
 
         self.config.add_tween(
             'vcsserver.tweens.request_wrapper.RequestWrapperTween',
         )
         self.config.add_request_method(
             'vcsserver.lib.request_counter.get_request_counter',
             'request_count')
 
     def wsgi_app(self):
         return self.config.make_wsgi_app()
 
     def _vcs_view_params(self, request):
         remote = self._remotes[request.matchdict['backend']]
         payload = msgpack.unpackb(request.body, use_list=True)
         method = payload.get('method')
         params = payload['params']
         wire = params.get('wire')
         args = params.get('args')
         kwargs = params.get('kwargs')
         context_uid = None
 
         if wire:
             try:
                 wire['context'] = context_uid = uuid.UUID(wire['context'])
             except KeyError:
                 pass
             args.insert(0, wire)
         repo_state_uid = wire.get('repo_state_uid') if wire else None
 
         # NOTE(marcink): trading complexity for slight performance
         if log.isEnabledFor(logging.DEBUG):
             no_args_methods = [
 
             ]
             if method in no_args_methods:
                 call_args = ''
             else:
                 call_args = args[1:]
 
             log.debug('Method requested:`%s` with args:%s kwargs:%s context_uid: %s, repo_state_uid:%s',
                       method, call_args, kwargs, context_uid, repo_state_uid)
 
             statsd = request.registry.statsd
             if statsd:
                 statsd.incr(
                     'vcsserver_method_total', tags=[
                         "method:{}".format(method),
                     ])
         return payload, remote, method, args, kwargs
 
     def vcs_view(self, request):
 
         payload, remote, method, args, kwargs = self._vcs_view_params(request)
         payload_id = payload.get('id')
 
         try:
             resp = getattr(remote, method)(*args, **kwargs)
         except Exception as e:
             exc_info = list(sys.exc_info())
             exc_type, exc_value, exc_traceback = exc_info
 
             org_exc = getattr(e, '_org_exc', None)
             org_exc_name = None
             org_exc_tb = ''
             if org_exc:
                 org_exc_name = org_exc.__class__.__name__
                 org_exc_tb = getattr(e, '_org_exc_tb', '')
                 # replace our "faked" exception with our org
                 exc_info[0] = org_exc.__class__
                 exc_info[1] = org_exc
 
             should_store_exc = True
             if org_exc:
                 def get_exc_fqn(_exc_obj):
                     module_name = getattr(org_exc.__class__, '__module__', 'UNKNOWN')
                     return module_name + '.' + org_exc_name
 
                 exc_fqn = get_exc_fqn(org_exc)
 
                 if exc_fqn in ['mercurial.error.RepoLookupError',
                                'vcsserver.exceptions.RefNotFoundException']:
                     should_store_exc = False
 
             if should_store_exc:
                 store_exception(id(exc_info), exc_info, request_path=request.path)
 
             tb_info = ''.join(
                 traceback.format_exception(exc_type, exc_value, exc_traceback))
 
             type_ = e.__class__.__name__
             if type_ not in self.ALLOWED_EXCEPTIONS:
                 type_ = None
 
             resp = {
                 'id': payload_id,
                 'error': {
                     'message': e.message,
                     'traceback': tb_info,
                     'org_exc': org_exc_name,
                     'org_exc_tb': org_exc_tb,
                     'type': type_
                 }
             }
 
             try:
                 resp['error']['_vcs_kind'] = getattr(e, '_vcs_kind', None)
             except AttributeError:
                 pass
         else:
             resp = {
                 'id': payload_id,
                 'result': resp
             }
 
         return resp
 
     def vcs_stream_view(self, request):
         payload, remote, method, args, kwargs = self._vcs_view_params(request)
         # this method has a stream: marker we remove it here
         method = method.split('stream:')[-1]
         chunk_size = safe_int(payload.get('chunk_size')) or 4096
 
         try:
             resp = getattr(remote, method)(*args, **kwargs)
         except Exception as e:
             raise
 
         def get_chunked_data(method_resp):
             stream = StringIO(method_resp)
             while 1:
                 chunk = stream.read(chunk_size)
                 if not chunk:
                     break
                 yield chunk
 
         response = Response(app_iter=get_chunked_data(resp))
         response.content_type = 'application/octet-stream'
 
         return response
 
     def status_view(self, request):
         import vcsserver
         return {'status': 'OK', 'vcsserver_version': vcsserver.__version__,
                 'pid': os.getpid()}
 
     def service_view(self, request):
         import vcsserver
 
         payload = msgpack.unpackb(request.body, use_list=True)
         server_config, app_config = {}, {}
 
         try:
             path = self.global_config['__file__']
             config = configparser.RawConfigParser()
 
             config.read(path)
 
             if config.has_section('server:main'):
                 server_config = dict(config.items('server:main'))
             if config.has_section('app:main'):
                 app_config = dict(config.items('app:main'))
 
         except Exception:
             log.exception('Failed to read .ini file for display')
 
         environ = os.environ.items()
 
         resp = {
             'id': payload.get('id'),
             'result': dict(
                 version=vcsserver.__version__,
                 config=server_config,
                 app_config=app_config,
                 environ=environ,
                 payload=payload,
             )
         }
         return resp
 
     def _msgpack_renderer_factory(self, info):
         def _render(value, system):
             request = system.get('request')
             if request is not None:
                 response = request.response
                 ct = response.content_type
                 if ct == response.default_content_type:
                     response.content_type = 'application/x-msgpack'
             return msgpack.packb(value)
         return _render
 
     def set_env_from_config(self, environ, config):
         dict_conf = {}
         try:
             for elem in config:
                 if elem[0] == 'rhodecode':
                     dict_conf = json.loads(elem[2])
                     break
         except Exception:
             log.exception('Failed to fetch SCM CONFIG')
             return
 
         username = dict_conf.get('username')
         if username:
             environ['REMOTE_USER'] = username
             # mercurial specific, some extension api rely on this
             environ['HGUSER'] = username
 
         ip = dict_conf.get('ip')
         if ip:
             environ['REMOTE_HOST'] = ip
 
         if _is_request_chunked(environ):
             # set the compatibility flag for webob
             environ['wsgi.input_terminated'] = True
 
     def hg_proxy(self):
         @wsgiapp
         def _hg_proxy(environ, start_response):
             app = WsgiProxy(self.remote_wsgi.HgRemoteWsgi())
             return app(environ, start_response)
         return _hg_proxy
 
     def git_proxy(self):
         @wsgiapp
         def _git_proxy(environ, start_response):
             app = WsgiProxy(self.remote_wsgi.GitRemoteWsgi())
             return app(environ, start_response)
         return _git_proxy
 
     def hg_stream(self):
         if self._use_echo_app:
             @wsgiapp
             def _hg_stream(environ, start_response):
                 app = EchoApp('fake_path', 'fake_name', None)
                 return app(environ, start_response)
             return _hg_stream
         else:
             @wsgiapp
             def _hg_stream(environ, start_response):
                 log.debug('http-app: handling hg stream')
                 repo_path = environ['HTTP_X_RC_REPO_PATH']
                 repo_name = environ['HTTP_X_RC_REPO_NAME']
                 packed_config = base64.b64decode(
                     environ['HTTP_X_RC_REPO_CONFIG'])
                 config = msgpack.unpackb(packed_config)
                 app = scm_app.create_hg_wsgi_app(
                     repo_path, repo_name, config)
 
                 # Consistent path information for hgweb
                 environ['PATH_INFO'] = environ['HTTP_X_RC_PATH_INFO']
                 environ['REPO_NAME'] = repo_name
                 self.set_env_from_config(environ, config)
 
                 log.debug('http-app: starting app handler '
                           'with %s and process request', app)
                 return app(environ, ResponseFilter(start_response))
             return _hg_stream
 
     def git_stream(self):
         if self._use_echo_app:
             @wsgiapp
             def _git_stream(environ, start_response):
                 app = EchoApp('fake_path', 'fake_name', None)
                 return app(environ, start_response)
             return _git_stream
         else:
             @wsgiapp
             def _git_stream(environ, start_response):
                 log.debug('http-app: handling git stream')
                 repo_path = environ['HTTP_X_RC_REPO_PATH']
                 repo_name = environ['HTTP_X_RC_REPO_NAME']
                 packed_config = base64.b64decode(
                     environ['HTTP_X_RC_REPO_CONFIG'])
                 config = msgpack.unpackb(packed_config)
 
                 environ['PATH_INFO'] = environ['HTTP_X_RC_PATH_INFO']
                 self.set_env_from_config(environ, config)
 
                 content_type = environ.get('CONTENT_TYPE', '')
 
                 path = environ['PATH_INFO']
                 is_lfs_request = GIT_LFS_CONTENT_TYPE in content_type
                 log.debug(
                     'LFS: Detecting if request `%s` is LFS server path based '
                     'on content type:`%s`, is_lfs:%s',
                     path, content_type, is_lfs_request)
 
                 if not is_lfs_request:
                     # fallback detection by path
                     if GIT_LFS_PROTO_PAT.match(path):
                         is_lfs_request = True
                     log.debug(
                         'LFS: fallback detection by path of: `%s`, is_lfs:%s',
                         path, is_lfs_request)
 
                 if is_lfs_request:
                     app = scm_app.create_git_lfs_wsgi_app(
                         repo_path, repo_name, config)
                 else:
                     app = scm_app.create_git_wsgi_app(
                         repo_path, repo_name, config)
 
                 log.debug('http-app: starting app handler '
                           'with %s and process request', app)
 
                 return app(environ, start_response)
 
             return _git_stream
 
     def handle_vcs_exception(self, exception, request):
         _vcs_kind = getattr(exception, '_vcs_kind', '')
         if _vcs_kind == 'repo_locked':
             # Get custom repo-locked status code if present.
             status_code = request.headers.get('X-RC-Locked-Status-Code')
             return HTTPRepoLocked(
                 title=exception.message, status_code=status_code)
 
         elif _vcs_kind == 'repo_branch_protected':
             # Get custom repo-branch-protected status code if present.
             return HTTPRepoBranchProtected(title=exception.message)
 
         exc_info = request.exc_info
         store_exception(id(exc_info), exc_info)
 
         traceback_info = 'unavailable'
         if request.exc_info:
             exc_type, exc_value, exc_tb = request.exc_info
             traceback_info = ''.join(traceback.format_exception(exc_type, exc_value, exc_tb))
 
         log.error(
             'error occurred handling this request for path: %s, \n tb: %s',
             request.path, traceback_info)
 
         statsd = request.registry.statsd
         if statsd:
             exc_type = "{}.{}".format(exception.__class__.__module__, exception.__class__.__name__)
             statsd.incr('vcsserver_exception_total',
                         tags=["type:{}".format(exc_type)])
         raise exception
 
 
 class ResponseFilter(object):
 
     def __init__(self, start_response):
         self._start_response = start_response
 
     def __call__(self, status, response_headers, exc_info=None):
         headers = tuple(
             (h, v) for h, v in response_headers
             if not wsgiref.util.is_hop_by_hop(h))
         return self._start_response(status, headers, exc_info)
 
 
+def sanitize_settings_and_apply_defaults(global_config, settings):
+    global_settings_maker = SettingsMaker(global_config)
+    settings_maker = SettingsMaker(settings)
+
+    settings_maker.make_setting(
+        'logging.autoconfigure',
+        default=True,
+        parser='bool')
+
+    logging_conf = os.path.join(os.path.dirname(global_config.get('__file__')), 'logging.ini')
+    settings_maker.enable_logging(logging_conf)
+
+    # Default includes, possible to change as a user
+    pyramid_includes = settings_maker.make_setting('pyramid.includes', [], parser='list:newline')
+    log.debug(
+        "Using the following pyramid.includes: %s",
+        pyramid_includes)
+
+    settings_maker.make_setting('__file__', global_config.get('__file__'))
+
+    settings_maker.make_setting(
+        'pyramid.default_locale_name',
+        default='en',
+        parser='string')
+    settings_maker.make_setting(
+        'locale',
+        default='en_US.UTF-8',
+        parser='string')
+
+    settings_maker.make_setting(
+        'core.binary_dir',
+        default='',
+        parser='string')
+
+    temp_store = tempfile.gettempdir()
+    default_cache_dir = os.path.join(temp_store, 'rc_cache')
+    # save default, cache dir, and use it for all backends later.
+    default_cache_dir = settings_maker.make_setting(
+        'cache_dir',
+        default=default_cache_dir, default_when_empty=True,
+        parser='dir:ensured')
+
+    # exception store cache
+    settings_maker.make_setting(
+        'exception_tracker.store_path',
+        default=os.path.join(default_cache_dir, 'exc_store'), default_when_empty=True,
+        parser='dir:ensured'
+    )
+
+    # repo_object cache defaults
+    settings_maker.make_setting(
+        'rc_cache.repo_object.backend',
+        default='dogpile.cache.rc.file_namespace',
+        parser='string')
+    settings_maker.make_setting(
+        'rc_cache.repo_object.expiration_time',
+        default=30 * 24 * 60 * 60,  # 30days
+        parser='int')
+    settings_maker. make_setting(
+        'rc_cache.repo_object.arguments.filename',
+        default=os.path.join(default_cache_dir, 'vcsserver_cache_repo_object.db'),
+        parser='string')
+
+    # statsd
+    settings_maker. make_setting(
+        'statsd.enabled',
+        default=False,
+        parser='bool')
+    settings_maker. make_setting(
+        'statsd.statsd_host',
+        default='statsd-exporter',
+        parser='string')
+    settings_maker. make_setting(
+        'statsd.statsd_port',
+        default=9125,
+        parser='int')
+    settings_maker. make_setting(
+        'statsd.statsd_prefix',
+        default='',
+        parser='string')
+    settings_maker. make_setting(
+        'statsd.statsd_ipv6',
+        default=False,
+        parser='bool')
+
+
 def main(global_config, **settings):
+    start_time = time.time()
+    log.info('Pyramid app config starting')
+
     if MercurialFactory:
         hgpatches.patch_largefiles_capabilities()
         hgpatches.patch_subrepo_type_mapping()
 
+    # Fill in and sanitize the defaults & do ENV expansion
+    sanitize_settings_and_apply_defaults(global_config, settings)
+
     # init and bootstrap StatsdClient
     StatsdClient.setup(settings)
 
-    app = HTTPApplication(settings=settings, global_config=global_config)
-    return app.wsgi_app()
+    pyramid_app = HTTPApplication(settings=settings, global_config=global_config).wsgi_app()
+    total_time = time.time() - start_time
+    log.info('Pyramid app `%s` created and configured in %.2fs',
+             getattr(pyramid_app, 'func_name', 'pyramid_app'), total_time)
+    return pyramid_app
+
+