configs: moved gunicorn configs to python files

configs/development.ini

@@ -27,9 +27,10 b' debug = true'
 #smtp_use_ssl = true
 
 [server:main]
-; COMMON HOST/IP CONFIG
+; COMMON HOST/IP CONFIG, This applies mostly to develop setup,
+; Host port for gunicorn are controlled by gunicorn_conf.py
 host = 127.0.0.1
-port = 5000
+port = 10020
 
 ; ##################################################
 ; WAITRESS WSGI SERVER - Recommended for Development
@@ -53,85 +54,11 b' asyncore_use_poll = true'
 ; GUNICORN APPLICATION SERVER
 ; ###########################
 
-; run with gunicorn --log-config rhodecode.ini --paste rhodecode.ini
+; run with gunicorn --paste rhodecode.ini --config gunicorn_conf.py
 
 ; Module to use, this setting shouldn't be changed
 #use = egg:gunicorn#main
 
-; Sets the number of process workers. More workers means more concurrent connections
-; RhodeCode can handle at the same time. Each additional worker also it increases
-; memory usage as each has it's own set of caches.
-; Recommended value is (2 * NUMBER_OF_CPUS + 1), eg 2CPU = 5 workers, but no more
-; than 8-10 unless for really big deployments .e.g 700-1000 users.
-; `instance_id = *` must be set in the [app:main] section below (which is the default)
-; when using more than 1 worker.
-#workers = 2
-
-; Gunicorn access log level
-#loglevel = info
-
-; Process name visible in process list
-#proc_name = rhodecode
-
-; Type of worker class, one of `sync`, `gevent`
-; Recommended type is `gevent`
-#worker_class = gevent
-
-; The maximum number of simultaneous clients per worker. Valid only for gevent
-#worker_connections = 10
-
-; The maximum number of pending connections worker will queue to handle
-#backlog = 64
-
-; Max number of requests that worker will handle before being gracefully restarted.
-; Prevents memory leaks, jitter adds variability so not all workers are restarted at once.
-#max_requests = 1000
-#max_requests_jitter = 30
-
-; Amount of time a worker can spend with handling a request before it
-; gets killed and restarted. By default set to 21600 (6hrs)
-; Examples: 1800 (30min), 3600 (1hr), 7200 (2hr), 43200 (12h)
-#timeout = 21600
-
-; The maximum size of HTTP request line in bytes.
-; 0 for unlimited
-#limit_request_line = 0
-
-; Limit the number of HTTP headers fields in a request.
-; By default this value is 100 and can't be larger than 32768.
-#limit_request_fields = 32768
-
-; Limit the allowed size of an HTTP request header field.
-; Value is a positive number or 0.
-; Setting it to 0 will allow unlimited header field sizes.
-#limit_request_field_size = 0
-
-; Timeout for graceful workers restart.
-; After receiving a restart signal, workers have this much time to finish
-; serving requests. Workers still alive after the timeout (starting from the
-; receipt of the restart signal) are force killed.
-; Examples: 1800 (30min), 3600 (1hr), 7200 (2hr), 43200 (12h)
-#graceful_timeout = 3600
-
-# The number of seconds to wait for requests on a Keep-Alive connection.
-# Generally set in the 1-5 seconds range.
-#keepalive = 2
-
-; Maximum memory usage that each worker can use before it will receive a
-; graceful restart signal 0 = memory monitoring is disabled
-; Examples: 268435456 (256MB), 536870912 (512MB)
-; 1073741824 (1GB), 2147483648 (2GB), 4294967296 (4GB)
-#memory_max_usage = 0
-
-; How often in seconds to check for memory usage for each gunicorn worker
-#memory_usage_check_interval = 60
-
-; Threshold value for which we don't recycle worker if GarbageCollection
-; frees up enough resources. Before each restart we try to run GC on worker
-; in case we get enough free memory after that, restart will not happen.
-#memory_usage_recovery_threshold = 0.8
-
-
 ; Prefix middleware for RhodeCode.
 ; recommended when using proxy setup.
 ; allows to set RhodeCode under a prefix in server.
@@ -625,6 +552,9 b' sqlalchemy.db1.pool_recycle = 3600'
 
 ; the number of connections to keep open inside the connection pool.
 ; 0 indicates no limit
+; the general calculus with gevent is:
+; if your system allows 500 concurrent greenlets (max_connections) that all do database access,
+; then increase pool size + max overflow so that they add up to 500.
 #sqlalchemy.db1.pool_size = 5
 
 ; The number of connections to allow in connection pool "overflow", that is

configs/gunicorn_config.py

@@ -12,6 +12,7 b' import threading'
 import traceback
 import random
 import socket
+import dataclasses
 from gunicorn.glogging import Logger
 
 
@@ -19,8 +20,14 b' def get_workers():'
     import multiprocessing
     return multiprocessing.cpu_count() * 2 + 1
 
-# GLOBAL
+
+bind = "127.0.0.1:10020"
+
+
+# Error logging output for gunicorn (-) is stdout
 errorlog = '-'
+
+# Access logging output for gunicorn (-) is stdout
 accesslog = '-'
 
 
@@ -30,6 +37,7 b" accesslog = '-'"
 worker_tmp_dir = None
 tmp_upload_dir = None
 
+# use re-use port logic
 #reuse_port = True
 
 # Custom log format
@@ -40,9 +48,91 b' tmp_upload_dir = None'
 access_log_format = (
     'time="%(t)s" pid=%(p)s level="INFO" type="[GNCRN]" ip="%(h)-15s" rqt="%(L)s" response_code="%(s)s" response_bytes="%(b)-6s" uri="%(m)s:%(U)s %(q)s" user=":%(u)s" user_agent="%(a)s"')
 
-# self adjust workers based on CPU count
+
+# Sets the number of process workers. More workers means more concurrent connections
+# RhodeCode can handle at the same time. Each additional worker also it increases
+# memory usage as each has it's own set of caches.
+# Recommended value is (2 * NUMBER_OF_CPUS + 1), eg 2CPU = 5 workers, but no more
+# than 8-10 unless for huge deployments .e.g 700-1000 users.
+# `instance_id = *` must be set in the [app:main] section below (which is the default)
+# when using more than 1 worker.
+workers = 4
+
+# self adjust workers based on CPU count, to use maximum of CPU and not overquota the resources
 # workers = get_workers()
 
+# Gunicorn access log level
+loglevel = 'info'
+
+# Process name visible in process list
+proc_name = 'rhodecode_enterprise'
+
+# Type of worker class, one of `sync`, `gevent`
+# currently `sync` is the only option allowed.
+worker_class = 'gevent'
+
+# The maximum number of simultaneous clients. Valid only for gevent
+worker_connections = 10
+
+# Max number of requests that worker will handle before being gracefully restarted.
+# Prevents memory leaks, jitter adds variability so not all workers are restarted at once.
+max_requests = 2000
+max_requests_jitter = 30
+
+# The maximum number of pending connections.
+# Exceeding this number results in the client getting an error when attempting to connect.
+backlog = 64
+
+# Amount of time a worker can spend with handling a request before it
+# gets killed and restarted. By default set to 21600 (6hrs)
+# Examples: 1800 (30min), 3600 (1hr), 7200 (2hr), 43200 (12h)
+timeout = 21600
+
+# The maximum size of HTTP request line in bytes.
+# 0 for unlimited
+limit_request_line = 0
+
+# Limit the number of HTTP headers fields in a request.
+# By default this value is 100 and can't be larger than 32768.
+limit_request_fields = 32768
+
+# Limit the allowed size of an HTTP request header field.
+# Value is a positive number or 0.
+# Setting it to 0 will allow unlimited header field sizes.
+limit_request_field_size = 0
+
+# Timeout for graceful workers restart.
+# After receiving a restart signal, workers have this much time to finish
+# serving requests. Workers still alive after the timeout (starting from the
+# receipt of the restart signal) are force killed.
+# Examples: 1800 (30min), 3600 (1hr), 7200 (2hr), 43200 (12h)
+graceful_timeout = 21600
+
+# The number of seconds to wait for requests on a Keep-Alive connection.
+# Generally set in the 1-5 seconds range.
+keepalive = 2
+
+# Maximum memory usage that each worker can use before it will receive a
+# graceful restart signal 0 = memory monitoring is disabled
+# Examples: 268435456 (256MB), 536870912 (512MB)
+# 1073741824 (1GB), 2147483648 (2GB), 4294967296 (4GB)
+memory_max_usage = 0
+
+# How often in seconds to check for memory usage for each gunicorn worker
+memory_usage_check_interval = 60
+
+# Threshold value for which we don't recycle worker if GarbageCollection
+# frees up enough resources. Before each restart we try to run GC on worker
+# in case we get enough free memory after that, restart will not happen.
+memory_usage_recovery_threshold = 0.8
+
+
[e-mail]
+class MemoryCheckConfig:
+    max_usage: int
+    check_interval: int
+    recovery_threshold: float
+
 
 def _get_process_rss(pid=None):
     try:
@@ -67,22 +157,14 b' def _get_config(ini_path):'
         return None
 
 
-def _time_with_offset(memory_usage_check_interval):
-    return time.time() - random.randint(0, memory_usage_check_interval/2.0)
-
-
-def pre_fork(server, worker):
-    pass
-
+def get_memory_usage_params(config=None):
+    # memory spec defaults
+    _memory_max_usage = memory_max_usage
+    _memory_usage_check_interval = memory_usage_check_interval
+    _memory_usage_recovery_threshold = memory_usage_recovery_threshold
 
-def post_fork(server, worker):
-
-    # memory spec defaults
-    _memory_max_usage = 0
-    _memory_usage_check_interval = 60
-    _memory_usage_recovery_threshold = 0.8
-
-    ini_path = os.path.abspath(server.cfg.paste)
+    if config:
+        ini_path = os.path.abspath(config)
     conf = _get_config(ini_path)
 
     section = 'server:main'
@@ -97,6 +179,31 b' def post_fork(server, worker):'
         if conf.has_option(section, 'memory_usage_recovery_threshold'):
             _memory_usage_recovery_threshold = conf.getfloat(section, 'memory_usage_recovery_threshold')
 
+    _memory_max_usage = int(os.environ.get('RC_GUNICORN_MEMORY_MAX_USAGE', '')
+                            or _memory_max_usage)
+    _memory_usage_check_interval = int(os.environ.get('RC_GUNICORN_MEMORY_USAGE_CHECK_INTERVAL', '')
+                                       or _memory_usage_check_interval)
+    _memory_usage_recovery_threshold = float(os.environ.get('RC_GUNICORN_MEMORY_USAGE_RECOVERY_THRESHOLD', '')
+                                             or _memory_usage_recovery_threshold)
+
+    return MemoryCheckConfig(_memory_max_usage, _memory_usage_check_interval, _memory_usage_recovery_threshold)
+
+
+def _time_with_offset(check_interval):
+    return time.time() - random.randint(0, check_interval/2.0)
+
+
+def pre_fork(server, worker):
+    pass
+
+
+def post_fork(server, worker):
+
+    memory_conf = get_memory_usage_params()
+    _memory_max_usage = memory_conf.max_usage
+    _memory_usage_check_interval = memory_conf.check_interval
+    _memory_usage_recovery_threshold = memory_conf.recovery_threshold
+
     worker._memory_max_usage = int(os.environ.get('RC_GUNICORN_MEMORY_MAX_USAGE', '')
                                    or _memory_max_usage)
     worker._memory_usage_check_interval = int(os.environ.get('RC_GUNICORN_MEMORY_USAGE_CHECK_INTERVAL', '')
@@ -109,10 +216,10 b' def post_fork(server, worker):'
     worker._last_memory_check_time = _time_with_offset(_memory_usage_check_interval)
 
     if _memory_max_usage:
-        server.log.info("[%-10s] WORKER spawned with max memory set at %s", worker.pid,
+        server.log.info("pid=[%-10s] WORKER spawned with max memory set at %s", worker.pid,
                         _format_data_size(_memory_max_usage))
     else:
-        server.log.info("[%-10s] WORKER spawned", worker.pid)
+        server.log.info("pid=[%-10s] WORKER spawned", worker.pid)
 
 
 def pre_exec(server):
@@ -181,42 +288,45 b' def _format_data_size(size, unit="B", pr'
 
 
 def _check_memory_usage(worker):
-    memory_max_usage = worker._memory_max_usage
-    if not memory_max_usage:
+    _memory_max_usage = worker._memory_max_usage
+    if not _memory_max_usage:
         return
 
-    memory_usage_check_interval = worker._memory_usage_check_interval
-    memory_usage_recovery_threshold = memory_max_usage * worker._memory_usage_recovery_threshold
+    _memory_usage_check_interval = worker._memory_usage_check_interval
+    _memory_usage_recovery_threshold = memory_max_usage * worker._memory_usage_recovery_threshold
 
     elapsed = time.time() - worker._last_memory_check_time
-    if elapsed > memory_usage_check_interval:
+    if elapsed > _memory_usage_check_interval:
         mem_usage = _get_process_rss()
-        if mem_usage and mem_usage > memory_max_usage:
+        if mem_usage and mem_usage > _memory_max_usage:
             worker.log.info(
                 "memory usage %s > %s, forcing gc",
-                _format_data_size(mem_usage), _format_data_size(memory_max_usage))
+                _format_data_size(mem_usage), _format_data_size(_memory_max_usage))
             # Try to clean it up by forcing a full collection.
             gc.collect()
             mem_usage = _get_process_rss()
-            if mem_usage > memory_usage_recovery_threshold:
+            if mem_usage > _memory_usage_recovery_threshold:
                 # Didn't clean up enough, we'll have to terminate.
                 worker.log.warning(
                     "memory usage %s > %s after gc, quitting",
-                    _format_data_size(mem_usage), _format_data_size(memory_max_usage))
+                    _format_data_size(mem_usage), _format_data_size(_memory_max_usage))
                 # This will cause worker to auto-restart itself
                 worker.alive = False
         worker._last_memory_check_time = time.time()
 
 
 def worker_int(worker):
-    worker.log.info("[%-10s] worker received INT or QUIT signal", worker.pid)
+    worker.log.info("pid=[%-10s] worker received INT or QUIT signal", worker.pid)
 
     # get traceback info, on worker crash
+    def get_thread_id(t_id):
     id2name = dict([(th.ident, th.name) for th in threading.enumerate()])
+        return id2name.get(t_id, "unknown_thread_id")
+
     code = []
     for thread_id, stack in sys._current_frames().items():
         code.append(
-            "\n# Thread: %s(%d)" % (id2name.get(thread_id, ""), thread_id))
+            "\n# Thread: %s(%d)" % (get_thread_id(thread_id), thread_id))
         for fname, lineno, name, line in traceback.extract_stack(stack):
             code.append('File: "%s", line %d, in %s' % (fname, lineno, name))
             if line:
@@ -225,15 +335,15 b' def worker_int(worker):'
 
 
 def worker_abort(worker):
-    worker.log.info("[%-10s] worker received SIGABRT signal", worker.pid)
+    worker.log.info("pid=[%-10s] worker received SIGABRT signal", worker.pid)
 
 
 def worker_exit(server, worker):
-    worker.log.info("[%-10s] worker exit", worker.pid)
+    worker.log.info("pid=[%-10s] worker exit", worker.pid)
 
 
 def child_exit(server, worker):
-    worker.log.info("[%-10s] worker child exit", worker.pid)
+    worker.log.info("pid=[%-10s] worker child exit", worker.pid)
 
 
 def pre_request(worker, req):
@@ -306,7 +416,9 b' def get_ip_addr(environ):'
     proxy_key = 'HTTP_X_REAL_IP'
     proxy_key2 = 'HTTP_X_FORWARDED_FOR'
     def_key = 'REMOTE_ADDR'
-    _filters = lambda x: _filter_port(_filter_proxy(x))
+
+    def _filters(x):
+        return _filter_port(_filter_proxy(x))
 
     ip = environ.get(proxy_key)
     if ip:
@@ -390,4 +502,5 b' class RhodeCodeLogger(Logger):'
 
         return atoms
 
+
 logger_class = RhodeCodeLogger

configs/production.ini

@@ -27,94 +27,21 b' debug = false'
 #smtp_use_ssl = true
 
 [server:main]
-; COMMON HOST/IP CONFIG
+; COMMON HOST/IP CONFIG, This applies mostly to develop setup,
+; Host port for gunicorn are controlled by gunicorn_conf.py
 host = 127.0.0.1
-port = 5000
+port = 10020
 
 
 ; ###########################
 ; GUNICORN APPLICATION SERVER
 ; ###########################
 
-; run with gunicorn --paste rhodecode.ini
+; run with gunicorn --paste rhodecode.ini --config gunicorn_conf.py
 
 ; Module to use, this setting shouldn't be changed
 use = egg:gunicorn#main
 
-; Sets the number of process workers. More workers means more concurrent connections
-; RhodeCode can handle at the same time. Each additional worker also it increases
-; memory usage as each has it's own set of caches.
-; Recommended value is (2 * NUMBER_OF_CPUS + 1), eg 2CPU = 5 workers, but no more
-; than 8-10 unless for really big deployments .e.g 700-1000 users.
-; `instance_id = *` must be set in the [app:main] section below (which is the default)
-; when using more than 1 worker.
-workers = 2
-
-; Gunicorn access log level
-loglevel = info
-
-; Process name visible in process list
-proc_name = rhodecode
-
-; Type of worker class, one of `sync`, `gevent`
-; Recommended type is `gevent`
-worker_class = gevent
-
-; The maximum number of simultaneous clients per worker. Valid only for gevent
-worker_connections = 10
-
-; The maximum number of pending connections worker will queue to handle
-backlog = 64
-
-; Max number of requests that worker will handle before being gracefully restarted.
-; Prevents memory leaks, jitter adds variability so not all workers are restarted at once.
-max_requests = 1000
-max_requests_jitter = 30
-
-; Amount of time a worker can spend with handling a request before it
-; gets killed and restarted. By default set to 21600 (6hrs)
-; Examples: 1800 (30min), 3600 (1hr), 7200 (2hr), 43200 (12h)
-timeout = 21600
-
-; The maximum size of HTTP request line in bytes.
-; 0 for unlimited
-limit_request_line = 0
-
-; Limit the number of HTTP headers fields in a request.
-; By default this value is 100 and can't be larger than 32768.
-limit_request_fields = 32768
-
-; Limit the allowed size of an HTTP request header field.
-; Value is a positive number or 0.
-; Setting it to 0 will allow unlimited header field sizes.
-limit_request_field_size = 0
-
-; Timeout for graceful workers restart.
-; After receiving a restart signal, workers have this much time to finish
-; serving requests. Workers still alive after the timeout (starting from the
-; receipt of the restart signal) are force killed.
-; Examples: 1800 (30min), 3600 (1hr), 7200 (2hr), 43200 (12h)
-graceful_timeout = 3600
-
-# The number of seconds to wait for requests on a Keep-Alive connection.
-# Generally set in the 1-5 seconds range.
-keepalive = 2
-
-; Maximum memory usage that each worker can use before it will receive a
-; graceful restart signal 0 = memory monitoring is disabled
-; Examples: 268435456 (256MB), 536870912 (512MB)
-; 1073741824 (1GB), 2147483648 (2GB), 4294967296 (4GB)
-memory_max_usage = 0
-
-; How often in seconds to check for memory usage for each gunicorn worker
-memory_usage_check_interval = 60
-
-; Threshold value for which we don't recycle worker if GarbageCollection
-; frees up enough resources. Before each restart we try to run GC on worker
-; in case we get enough free memory after that, restart will not happen.
-memory_usage_recovery_threshold = 0.8
-
-
 ; Prefix middleware for RhodeCode.
 ; recommended when using proxy setup.
 ; allows to set RhodeCode under a prefix in server.