configs: moved most of configuration back to .ini files instead of gunicorn file

configs/development.ini

@@ -1,91 +1,175 b''
 ################################################################################
 # RhodeCode VCSServer with HTTP Backend - configuration                        #
 ################################################################################
 
 
 [server:main]
 ## COMMON ##
 host = 0.0.0.0
 port = 9900
 
+###########################################################
+## WAITRESS WSGI SERVER - Recommended for Development  ####
+###########################################################
+
 use = egg:waitress#main
+## number of worker threads
+threads = 5
+## MAX BODY SIZE 100GB
+max_request_body_size = 107374182400
+## Use poll instead of select, fixes file descriptors limits problems.
+## May not work on old windows systems.
+asyncore_use_poll = true
+
+
+##########################
+## GUNICORN WSGI SERVER ##
+##########################
+## run with gunicorn --log-config vcsserver.ini --paste vcsserver.ini
+
+#use = egg:gunicorn#main
+## Sets the number of process workers. More workers means more concurrent connections
+## RhodeCode can handle at the same time. Each additional worker also it increases
+## memory usage as each has it's own set of caches.
+## Recommended value is (2 * NUMBER_OF_CPUS + 1), eg 2CPU = 5 workers, but no more
+## than 8-10 unless for really big deployments .e.g 700-1000 users.
+## `instance_id = *` must be set in the [app:main] section below (which is the default)
+## when using more than 1 worker.
+#workers = 2
+
+## Gunicorn access log level
+#loglevel = info
+
+## process name visible in process list
+#proc_name = rhodecode_vcsserver
+
+## type of worker class, currently `sync` is the only option allowed.
+#worker_class = sync
+
+## The maximum number of simultaneous clients. Valid only for Gevent
+#worker_connections = 10
+
+## max number of requests that worker will handle before being gracefully
+## restarted, could prevent memory leaks
+#max_requests = 1000
+#max_requests_jitter = 30
+
+## amount of time a worker can spend with handling a request before it
+## gets killed and restarted. Set to 6hrs
+#timeout = 21600
+
+## The maximum size of HTTP request line in bytes.
+## 0 for unlimited
+#limit_request_line = 0
+
+## Limit the number of HTTP headers fields in a request.
+## By default this value is 100 and can't be larger than 32768.
+#limit_request_fields = 32768
+
+## Limit the allowed size of an HTTP request header field.
+## Value is a positive number or 0.
+## Setting it to 0 will allow unlimited header field sizes.
+#limit_request_field_size = 0
+
+## Timeout for graceful workers restart.
+## After receiving a restart signal, workers have this much time to finish
+## serving requests. Workers still alive after the timeout (starting from the
+## receipt of the restart signal) are force killed.
+#graceful_timeout = 3600
+
+# The number of seconds to wait for requests on a Keep-Alive connection.
+# Generally set in the 1-5 seconds range.
+#keepalive = 2
+
+## Maximum memory usage that each worker can use before it will receive a
+## graceful restart signal, e.g 10MB = 10485760 (10 * 1024 * 1024)
+# 0 = memory monitoring is disabled
+#memory_max_usage = 0
+
+## How often in seconds to check for memory usage for each gunicorn worker
+#memory_usage_check_interval = 60
+
+## Threshold value for which we don't recycle worker if GarbageCollection
+## frees up enough resources. Before each restart we try to run GC on worker
+## in case we get enough free memory after that, restart will not happen.
+#memory_usage_recovery_threshold = 0.8
 
 
 [app:main]
 use = egg:rhodecode-vcsserver
 
 pyramid.default_locale_name = en
 pyramid.includes =
 
 ## default locale used by VCS systems
 locale = en_US.UTF-8
 
 
 ## path to binaries for vcsserver, it should be set by the installer
 ## at installation time, e.g /home/user/vcsserver-1/profile/bin
 core.binary_dir = ""
 
 ## Custom exception store path, defaults to TMPDIR
 ## This is used to store exception from RhodeCode in shared directory
 #exception_tracker.store_path =
 
 ## Default cache dir for caches.  Putting this into a ramdisk
 ## can boost performance, eg. /tmpfs/data_ramdisk, however this directory might require
 ## large amount of space
 cache_dir = %(here)s/rcdev/data
 
 ## cache region for storing repo_objects cache
 rc_cache.repo_object.backend = dogpile.cache.rc.file_namespace
 ## cache auto-expires after N seconds (2592000 == 30 days)
 rc_cache.repo_object.expiration_time = 2592000
 
 ## cache file store path, if empty set automatically to tmp dir location
 #rc_cache.repo_object.arguments.filename = /tmp/vcsserver_cache.db
 
 ## max size of LRU, old values will be discarded if the size of cache reaches max_size
 rc_cache.repo_object.max_size = 100
 
 
 ################################
 ### LOGGING CONFIGURATION   ####
 ################################
 [loggers]
 keys = root, vcsserver
 
 [handlers]
 keys = console
 
 [formatters]
 keys = generic
 
 #############
 ## LOGGERS ##
 #############
 [logger_root]
 level = NOTSET
 handlers = console
 
 [logger_vcsserver]
 level = DEBUG
 handlers =
 qualname = vcsserver
 propagate = 1
 
 
 ##############
 ## HANDLERS ##
 ##############
 
 [handler_console]
 class = StreamHandler
 args = (sys.stderr,)
 level = DEBUG
 formatter = generic
 
 ################
 ## FORMATTERS ##
 ################
 
 [formatter_generic]
 format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s
 datefmt = %Y-%m-%d %H:%M:%S

configs/gunicorn_config.py

@@ -1,259 +1,270 b''
 """
 gunicorn config extension and hooks. Sets additional configuration that is
 available post the .ini config.
 
 - workers = ${cpu_number}
 - threads = 1
 - proc_name = ${gunicorn_proc_name}
 - worker_class = sync
 - worker_connections = 10
 - max_requests = 1000
 - max_requests_jitter = 30
 - timeout = 21600
 
 """
 
+import gc
+import os
+import sys
 import math
-import gc
-import sys
 import time
 import threading
 import traceback
 import random
 from gunicorn.glogging import Logger
 
 
 def get_workers():
     import multiprocessing
     return multiprocessing.cpu_count() * 2 + 1
 
 # GLOBAL
 errorlog = '-'
 accesslog = '-'
-loglevel = 'info'
 
-# SECURITY
-
-# The maximum size of HTTP request line in bytes.
-# 0 for unlimited
-limit_request_line = 0
-
-# Limit the number of HTTP headers fields in a request.
-# By default this value is 100 and can't be larger than 32768.
-limit_request_fields = 32768
-
-# Limit the allowed size of an HTTP request header field.
-# Value is a positive number or 0.
-# Setting it to 0 will allow unlimited header field sizes.
-limit_request_field_size = 0
-
-# Timeout for graceful workers restart.
-# After receiving a restart signal, workers have this much time to finish
-# serving requests. Workers still alive after the timeout (starting from the
-# receipt of the restart signal) are force killed.
-graceful_timeout = 60 * 60
-
-# The number of seconds to wait for requests on a Keep-Alive connection.
-# Generally set in the 1-5 seconds range.
-keepalive = 2
 
 # SERVER MECHANICS
 # None == system temp dir
 # worker_tmp_dir is recommended to be set to some tmpfs
 worker_tmp_dir = None
 tmp_upload_dir = None
 
 # Custom log format
 access_log_format = (
     '%(t)s %(p)s INFO  [GNCRN] %(h)-15s rqt:%(L)s %(s)s %(b)-6s "%(m)s:%(U)s %(q)s" usr:%(u)s "%(f)s" "%(a)s"')
 
 # self adjust workers based on CPU count
 # workers = get_workers()
 
-# n * 1024 * 0124 == n MBs, 0 = memory monitoring is disabled
-MAX_MEMORY_USAGE = 0 * 1024 * 1024
-
-# How often in seconds to check for memory usage
-MEMORY_USAGE_CHECK_INTERVAL = 30
-
-# If a gc brings us back below this threshold, we can avoid termination.
-MEMORY_USAGE_RECOVERY_THRESHOLD = MAX_MEMORY_USAGE * 0.8
-
 
 def _get_process_rss(pid=None):
     try:
         import psutil
         if pid:
             proc = psutil.Process(pid)
         else:
             proc = psutil.Process()
         return proc.memory_info().rss
     except Exception:
         return None
 
 
-def _time_with_offset():
-    return time.time() - random.randint(0, MEMORY_USAGE_CHECK_INTERVAL/2.0)
+def _get_config(ini_path):
+
+    try:
+        import configparser
+    except ImportError:
+        import ConfigParser as configparser
+    try:
+        config = configparser.ConfigParser()
+        config.read(ini_path)
+        return config
+    except Exception:
+        return None
+
+
+def _time_with_offset(memory_usage_check_interval):
+    return time.time() - random.randint(0, memory_usage_check_interval/2.0)
 
 
 def pre_fork(server, worker):
     pass
 
 
 def post_fork(server, worker):
-    server.log.info("<%s> WORKER spawned", worker.pid)
+
+    # memory spec defaults
+    _memory_max_usage = 0
+    _memory_usage_check_interval = 60
+    _memory_usage_recovery_threshold = 0.8
+
+    ini_path = os.path.abspath(server.cfg.paste)
+    conf = _get_config(ini_path)
+    if conf and 'server:main' in conf:
+        section = conf['server:main']
+
+        if section.get('memory_max_usage'):
+            _memory_max_usage = int(section.get('memory_max_usage'))
+        if section.get('memory_usage_check_interval'):
+            _memory_usage_check_interval = int(section.get('memory_usage_check_interval'))
+        if section.get('memory_usage_recovery_threshold'):
+            _memory_usage_recovery_threshold = float(section.get('memory_usage_recovery_threshold'))
+
+    worker._memory_max_usage = _memory_max_usage
+    worker._memory_usage_check_interval = _memory_usage_check_interval
+    worker._memory_usage_recovery_threshold = _memory_usage_recovery_threshold
+
     # register memory last check time, with some random offset so we don't recycle all
     # at once
-    worker._last_memory_check_time = _time_with_offset()
+    worker._last_memory_check_time = _time_with_offset(_memory_usage_check_interval)
+
+    if _memory_max_usage:
+        server.log.info("[%-10s] WORKER spawned with max memory set at %s", worker.pid,
+                        _format_data_size(_memory_max_usage))
+    else:
+        server.log.info("[%-10s] WORKER spawned", worker.pid)
 
 
 def pre_exec(server):
     server.log.info("Forked child, re-executing.")
 
 
 def on_starting(server):
     server_lbl = '{} {}'.format(server.proc_name, server.address)
     server.log.info("Server %s is starting.", server_lbl)
 
 
 def when_ready(server):
     server.log.info("Server %s is ready. Spawning workers", server)
 
 
 def on_reload(server):
     pass
 
 
 def _format_data_size(size, unit="B", precision=1, binary=True):
     """Format a number using SI units (kilo, mega, etc.).
 
     ``size``: The number as a float or int.
 
     ``unit``: The unit name in plural form. Examples: "bytes", "B".
 
     ``precision``: How many digits to the right of the decimal point. Default
     is 1.  0 suppresses the decimal point.
 
     ``binary``: If false, use base-10 decimal prefixes (kilo = K = 1000).
     If true, use base-2 binary prefixes (kibi = Ki = 1024).
 
     ``full_name``: If false (default), use the prefix abbreviation ("k" or
     "Ki").  If true, use the full prefix ("kilo" or "kibi"). If false,
     use abbreviation ("k" or "Ki").
 
     """
 
     if not binary:
         base = 1000
         multiples = ('', 'k', 'M', 'G', 'T', 'P', 'E', 'Z', 'Y')
     else:
         base = 1024
         multiples = ('', 'Ki', 'Mi', 'Gi', 'Ti', 'Pi', 'Ei', 'Zi', 'Yi')
 
     sign = ""
     if size > 0:
         m = int(math.log(size, base))
     elif size < 0:
         sign = "-"
         size = -size
         m = int(math.log(size, base))
     else:
         m = 0
     if m > 8:
         m = 8
 
     if m == 0:
         precision = '%.0f'
     else:
         precision = '%%.%df' % precision
 
     size = precision % (size / math.pow(base, m))
 
     return '%s%s %s%s' % (sign, size.strip(), multiples[m], unit)
 
 
 def _check_memory_usage(worker):
+    memory_max_usage = worker._memory_max_usage
+    if not memory_max_usage:
+        return
 
-    if not MAX_MEMORY_USAGE:
-        return
+    memory_usage_check_interval = worker._memory_usage_check_interval
+    memory_usage_recovery_threshold = memory_max_usage * worker._memory_usage_recovery_threshold
 
     elapsed = time.time() - worker._last_memory_check_time
-    if elapsed > MEMORY_USAGE_CHECK_INTERVAL:
+    if elapsed > memory_usage_check_interval:
         mem_usage = _get_process_rss()
-        if mem_usage and mem_usage > MAX_MEMORY_USAGE:
+        if mem_usage and mem_usage > memory_max_usage:
             worker.log.info(
                 "memory usage %s > %s, forcing gc",
-                _format_data_size(mem_usage), _format_data_size(MAX_MEMORY_USAGE))
+                _format_data_size(mem_usage), _format_data_size(memory_max_usage))
             # Try to clean it up by forcing a full collection.
             gc.collect()
             mem_usage = _get_process_rss()
-            if mem_usage > MEMORY_USAGE_RECOVERY_THRESHOLD:
+            if mem_usage > memory_usage_recovery_threshold:
                 # Didn't clean up enough, we'll have to terminate.
                 worker.log.warning(
                     "memory usage %s > %s after gc, quitting",
-                    _format_data_size(mem_usage), _format_data_size(MAX_MEMORY_USAGE))
+                    _format_data_size(mem_usage), _format_data_size(memory_max_usage))
                 # This will cause worker to auto-restart itself
                 worker.alive = False
         worker._last_memory_check_time = time.time()
 
 
 def worker_int(worker):
-    worker.log.info("[<%-10s>] worker received INT or QUIT signal", worker.pid)
+    worker.log.info("[%-10s] worker received INT or QUIT signal", worker.pid)
 
     # get traceback info, on worker crash
     id2name = dict([(th.ident, th.name) for th in threading.enumerate()])
     code = []
     for thread_id, stack in sys._current_frames().items():
         code.append(
             "\n# Thread: %s(%d)" % (id2name.get(thread_id, ""), thread_id))
         for fname, lineno, name, line in traceback.extract_stack(stack):
             code.append('File: "%s", line %d, in %s' % (fname, lineno, name))
             if line:
                 code.append("  %s" % (line.strip()))
     worker.log.debug("\n".join(code))
 
 
 def worker_abort(worker):
-    worker.log.info("[<%-10s>] worker received SIGABRT signal", worker.pid)
+    worker.log.info("[%-10s] worker received SIGABRT signal", worker.pid)
 
 
 def worker_exit(server, worker):
-    worker.log.info("[<%-10s>] worker exit", worker.pid)
+    worker.log.info("[%-10s] worker exit", worker.pid)
 
 
 def child_exit(server, worker):
-    worker.log.info("[<%-10s>] worker child exit", worker.pid)
+    worker.log.info("[%-10s] worker child exit", worker.pid)
 
 
 def pre_request(worker, req):
     worker.start_time = time.time()
     worker.log.debug(
         "GNCRN PRE  WORKER [cnt:%s]: %s %s", worker.nr, req.method, req.path)
 
 
 def post_request(worker, req, environ, resp):
     total_time = time.time() - worker.start_time
     worker.log.debug(
         "GNCRN POST WORKER [cnt:%s]: %s %s resp: %s, Load Time: %.4fs",
         worker.nr, req.method, req.path, resp.status_code, total_time)
     _check_memory_usage(worker)
 
 
 class RhodeCodeLogger(Logger):
     """
     Custom Logger that allows some customization that gunicorn doesn't allow
     """
 
     datefmt = r"%Y-%m-%d %H:%M:%S"
 
     def __init__(self, cfg):
         Logger.__init__(self, cfg)
 
     def now(self):
         """ return date in RhodeCode Log format """
         now = time.time()
         msecs = int((now - long(now)) * 1000)
         return time.strftime(self.datefmt, time.localtime(now)) + '.{0:03d}'.format(msecs)
 
 
 logger_class = RhodeCodeLogger

configs/production.ini

@@ -1,112 +1,175 b''
 ################################################################################
 # RhodeCode VCSServer with HTTP Backend - configuration                        #
 ################################################################################
 
 
 [server:main]
 ## COMMON ##
 host = 127.0.0.1
 port = 9900
 
+###########################################################
+## WAITRESS WSGI SERVER - Recommended for Development  ####
+###########################################################
+
+#use = egg:waitress#main
+## number of worker threads
+#threads = 5
+## MAX BODY SIZE 100GB
+#max_request_body_size = 107374182400
+## Use poll instead of select, fixes file descriptors limits problems.
+## May not work on old windows systems.
+#asyncore_use_poll = true
+
 
 ##########################
 ## GUNICORN WSGI SERVER ##
 ##########################
 ## run with gunicorn --log-config vcsserver.ini --paste vcsserver.ini
+
 use = egg:gunicorn#main
-## Sets the number of process workers. Recommended
-## value is (2 * NUMBER_OF_CPUS + 1), eg 2CPU = 5 workers
+## Sets the number of process workers. More workers means more concurrent connections
+## RhodeCode can handle at the same time. Each additional worker also it increases
+## memory usage as each has it's own set of caches.
+## Recommended value is (2 * NUMBER_OF_CPUS + 1), eg 2CPU = 5 workers, but no more
+## than 8-10 unless for really big deployments .e.g 700-1000 users.
+## `instance_id = *` must be set in the [app:main] section below (which is the default)
+## when using more than 1 worker.
 workers = 2
-## process name
+
+## Gunicorn access log level
+loglevel = info
+
+## process name visible in process list
 proc_name = rhodecode_vcsserver
+
 ## type of worker class, currently `sync` is the only option allowed.
 worker_class = sync
+
 ## The maximum number of simultaneous clients. Valid only for Gevent
-#worker_connections = 10
+worker_connections = 10
+
 ## max number of requests that worker will handle before being gracefully
 ## restarted, could prevent memory leaks
 max_requests = 1000
 max_requests_jitter = 30
+
 ## amount of time a worker can spend with handling a request before it
 ## gets killed and restarted. Set to 6hrs
 timeout = 21600
 
+## The maximum size of HTTP request line in bytes.
+## 0 for unlimited
+limit_request_line = 0
+
+## Limit the number of HTTP headers fields in a request.
+## By default this value is 100 and can't be larger than 32768.
+limit_request_fields = 32768
+
+## Limit the allowed size of an HTTP request header field.
+## Value is a positive number or 0.
+## Setting it to 0 will allow unlimited header field sizes.
+limit_request_field_size = 0
+
+## Timeout for graceful workers restart.
+## After receiving a restart signal, workers have this much time to finish
+## serving requests. Workers still alive after the timeout (starting from the
+## receipt of the restart signal) are force killed.
+graceful_timeout = 3600
+
+# The number of seconds to wait for requests on a Keep-Alive connection.
+# Generally set in the 1-5 seconds range.
+keepalive = 2
+
+## Maximum memory usage that each worker can use before it will receive a
+## graceful restart signal, e.g 10MB = 10485760 (10 * 1024 * 1024)
+# 0 = memory monitoring is disabled
+memory_max_usage = 0
+
+## How often in seconds to check for memory usage for each gunicorn worker
+memory_usage_check_interval = 60
+
+## Threshold value for which we don't recycle worker if GarbageCollection
+## frees up enough resources. Before each restart we try to run GC on worker
+## in case we get enough free memory after that, restart will not happen.
+memory_usage_recovery_threshold = 0.8
+
 
 [app:main]
 use = egg:rhodecode-vcsserver
 
 pyramid.default_locale_name = en
 pyramid.includes =
 
 ## default locale used by VCS systems
 locale = en_US.UTF-8
 
 
 ## path to binaries for vcsserver, it should be set by the installer
 ## at installation time, e.g /home/user/vcsserver-1/profile/bin
 core.binary_dir = ""
 
 ## Custom exception store path, defaults to TMPDIR
 ## This is used to store exception from RhodeCode in shared directory
 #exception_tracker.store_path =
 
 ## Default cache dir for caches.  Putting this into a ramdisk
 ## can boost performance, eg. /tmpfs/data_ramdisk, however this directory might require
 ## large amount of space
 cache_dir = %(here)s/rcdev/data
 
 ## cache region for storing repo_objects cache
 rc_cache.repo_object.backend = dogpile.cache.rc.file_namespace
 ## cache auto-expires after N seconds (2592000 == 30 days)
 rc_cache.repo_object.expiration_time = 2592000
 
 ## cache file store path, if empty set automatically to tmp dir location
 #rc_cache.repo_object.arguments.filename = /tmp/vcsserver_cache.db
 
 ## max size of LRU, old values will be discarded if the size of cache reaches max_size
 rc_cache.repo_object.max_size = 100
 
 
 ################################
 ### LOGGING CONFIGURATION   ####
 ################################
 [loggers]
 keys = root, vcsserver
 
 [handlers]
 keys = console
 
 [formatters]
 keys = generic
 
 #############
 ## LOGGERS ##
 #############
 [logger_root]
 level = NOTSET
 handlers = console
 
 [logger_vcsserver]
 level = DEBUG
 handlers =
 qualname = vcsserver
 propagate = 1
 
 
 ##############
 ## HANDLERS ##
 ##############
 
 [handler_console]
 class = StreamHandler
 args = (sys.stderr,)
 level = DEBUG
 formatter = generic
 
 ################
 ## FORMATTERS ##
 ################
 
 [formatter_generic]
 format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s
 datefmt = %Y-%m-%d %H:%M:%S