env: made configuration completly overridable by env variables

configs/development.ini

@@ -215,6 +215,10 b' core.binary_dir = ""'
 
 ; configure logging automatically at server startup set to false
 ; to use the below custom logging config.
+; RC_LOGGING_FORMATTER
+; RC_LOGGING_LEVEL
+; env variables can control the settings for logging in case of autoconfigure
+
 #logging.autoconfigure = true
 
 ; specify your own custom logging config file to configure logging
@@ -223,6 +227,7 b' core.binary_dir = ""'
 ; #####################
 ; LOGGING CONFIGURATION
 ; #####################
+
 #[loggers]
 #keys = root, vcsserver
 

configs/production.ini

@@ -178,6 +178,10 b' core.binary_dir = ""'
 
 ; configure logging automatically at server startup set to false
 ; to use the below custom logging config.
+; RC_LOGGING_FORMATTER
+; RC_LOGGING_LEVEL
+; env variables can control the settings for logging in case of autoconfigure
+
 #logging.autoconfigure = true
 
 ; specify your own custom logging config file to configure logging
@@ -186,6 +190,7 b' core.binary_dir = ""'
 ; #####################
 ; LOGGING CONFIGURATION
 ; #####################
+
 #[loggers]
 #keys = root, vcsserver
 

vcsserver/config/settings_maker.py

@@ -27,6 +27,11 b' import tempfile'
 import logging.config
 log = logging.getLogger(__name__)
 
+# skip keys, that are set here, so we don't double process those
+set_keys = {
+    '__file__': ''
+}
+
 
 def str2bool(_str):
     """
@@ -119,6 +124,27 b' class SettingsMaker(object):'
     def _key_transformator(cls, key):
         return "{}_{}".format('RC'.upper(), key.upper().replace('.', '_').replace('-', '_'))
 
+    def maybe_env_key(self, key):
+        # now maybe we have this KEY in env, search and use the value with higher priority.
+        transformed_key = self._key_transformator(key)
+        envvar_value = os.environ.get(transformed_key)
+        if envvar_value:
+            log.debug('using `%s` key instead of `%s` key for config', transformed_key, key)
+
+        return envvar_value
+
+    def env_expand(self):
+        replaced = {}
+        for k, v in self.settings.items():
+            if k not in set_keys:
+                envvar_value = self.maybe_env_key(k)
+                if envvar_value:
+                    replaced[k] = envvar_value
+                    set_keys[k] = envvar_value
+
+        # replace ALL keys updated
+        self.settings.update(replaced)
+
     def enable_logging(self, logging_conf=None, level='INFO', formatter='generic'):
         """
         Helper to enable debug on running instance
@@ -133,7 +159,8 b' class SettingsMaker(object):'
             logging_conf = self.settings.get('logging.logging_conf_file') or ''
 
         if not os.path.isfile(logging_conf):
-            log.error('Unable to setup logging based on %s, file does not exist...', logging_conf)
+            log.error('Unable to setup logging based on %s, '
+                      'file does not exist.... specify path using logging.logging_conf_file= config setting. ', logging_conf)
             return
 
         with open(logging_conf, 'rb') as f:
@@ -143,14 +170,6 b' class SettingsMaker(object):'
                 RC_LOGGING_FORMATTER=os.environ.get('RC_LOGGING_FORMATTER', '') or formatter
             )
 
-
-        with open(logging_conf, 'rb') as f:
-            ini_template = textwrap.dedent(f.read())
-            ini_template = string.Template(ini_template).safe_substitute(
-                RC_LOGGING_LEVEL=os.environ.get('RC_LOGGING_LEVEL', '') or 'INFO',
-                RC_LOGGING_FORMATTER=os.environ.get('RC_LOGGING_FORMATTER', '') or 'generic'
-            )
-
         with tempfile.NamedTemporaryFile(prefix='rc_logging_', suffix='.ini', delete=False) as f:
             log.info('Saved Temporary LOGGING config at %s', f.name)
             f.write(ini_template)
@@ -159,7 +178,6 b' class SettingsMaker(object):'
         os.remove(f.name)
 
     def make_setting(self, key, default, lower=False, default_when_empty=False, parser=None):
-
         input_val = self.settings.get(key, default)
 
         if default_when_empty and not input_val:
@@ -180,11 +198,10 b' class SettingsMaker(object):'
             None: lambda i: i
         }[parser]
 
-        # now maybe we have this KEY in env, search and use the value with higher priority.
-        transformed_key = self._key_transformator(key)
-        envvar_value = os.environ.get(transformed_key)
+        envvar_value = self.maybe_env_key(key)
         if envvar_value:
-            log.debug('using `%s` key instead of `%s` key for config', transformed_key, key)
             input_val = envvar_value
+            set_keys[key] = input_val
+
         self.settings[key] = parser_func(input_val)
         return self.settings[key]

vcsserver/http_main.py

@@ -645,35 +645,21 b' def sanitize_settings_and_apply_defaults'
     global_settings_maker = SettingsMaker(global_config)
     settings_maker = SettingsMaker(settings)
 
-    settings_maker.make_setting(
-        'logging.autoconfigure',
-        default=True,
-        parser='bool')
+    settings_maker.make_setting('logging.autoconfigure', True, parser='bool')
 
     logging_conf = os.path.join(os.path.dirname(global_config.get('__file__')), 'logging.ini')
     settings_maker.enable_logging(logging_conf)
 
     # Default includes, possible to change as a user
     pyramid_includes = settings_maker.make_setting('pyramid.includes', [], parser='list:newline')
-    log.debug(
-        "Using the following pyramid.includes: %s",
-        pyramid_includes)
+    log.debug("Using the following pyramid.includes: %s", pyramid_includes)
 
     settings_maker.make_setting('__file__', global_config.get('__file__'))
 
-    settings_maker.make_setting(
-        'pyramid.default_locale_name',
-        default='en',
-        parser='string')
-    settings_maker.make_setting(
-        'locale',
-        default='en_US.UTF-8',
-        parser='string')
+    settings_maker.make_setting('pyramid.default_locale_name', 'en')
+    settings_maker.make_setting('locale', 'en_US.UTF-8')
 
-    settings_maker.make_setting(
-        'core.binary_dir',
-        default='',
-        parser='string')
+    settings_maker.make_setting('core.binary_dir', '')
 
     temp_store = tempfile.gettempdir()
     default_cache_dir = os.path.join(temp_store, 'rc_cache')
@@ -699,32 +685,19 b' def sanitize_settings_and_apply_defaults'
         'rc_cache.repo_object.expiration_time',
         default=30 * 24 * 60 * 60,  # 30days
         parser='int')
-    settings_maker. make_setting(
+    settings_maker.make_setting(
         'rc_cache.repo_object.arguments.filename',
         default=os.path.join(default_cache_dir, 'vcsserver_cache_repo_object.db'),
         parser='string')
 
     # statsd
-    settings_maker. make_setting(
-        'statsd.enabled',
-        default=False,
-        parser='bool')
-    settings_maker. make_setting(
-        'statsd.statsd_host',
-        default='statsd-exporter',
-        parser='string')
-    settings_maker. make_setting(
-        'statsd.statsd_port',
-        default=9125,
-        parser='int')
-    settings_maker. make_setting(
-        'statsd.statsd_prefix',
-        default='',
-        parser='string')
-    settings_maker. make_setting(
-        'statsd.statsd_ipv6',
-        default=False,
-        parser='bool')
+    settings_maker.make_setting('statsd.enabled', False, parser='bool')
+    settings_maker.make_setting('statsd.statsd_host', 'statsd-exporter', parser='string')
+    settings_maker.make_setting('statsd.statsd_port', 9125, parser='int')
+    settings_maker.make_setting('statsd.statsd_prefix', '')
+    settings_maker.make_setting('statsd.statsd_ipv6', False, parser='bool')
+
+    settings_maker.env_expand()
 
 
 def main(global_config, **settings):