configs: added small fixes into gunicorn configs

Makefile

@@ -1,172 +1,172 b''
 # required for pushd to work..
 SHELL = /bin/bash
 
 
 # set by: PATH_TO_OUTDATED_PACKAGES=/some/path/outdated_packages.py
 OUTDATED_PACKAGES = ${PATH_TO_OUTDATED_PACKAGES}
 
 NODE_PATH=./node_modules
 WEBPACK=./node_binaries/webpack
 GRUNT=./node_binaries/grunt
 
 .PHONY: clean
 ## Cleanup compiled and cache py files
 clean:
 	make test-clean
 	find . -type f \( -iname '*.c' -o -iname '*.pyc' -o -iname '*.so' -o -iname '*.orig' \) -exec rm '{}' ';'
 	find . -type d -name "build" -prune -exec rm -rf '{}' ';'
 
 .PHONY: test
 ## run test-clean and tests
 test:
 	make test-clean
 	make test-only
 
 
 .PHONY: test-clean
 ## run test-clean and tests
 test-clean:
 	rm -rf coverage.xml htmlcov junit.xml pylint.log result
 	find . -type d -name "__pycache__" -prune -exec rm -rf '{}' ';'
 	find . -type f \( -iname '.coverage.*' \) -exec rm '{}' ';'
 
 
 .PHONY: test-only
 ## Run tests only without cleanup
 test-only:
 	PYTHONHASHSEED=random \
 	py.test -x -vv -r xw -p no:sugar \
 	--cov-report=term-missing --cov-report=html \
 	--cov=rhodecode rhodecode
 
 
 .PHONY: test-only-mysql
 ## run tests against mysql
 test-only-mysql:
 	PYTHONHASHSEED=random \
 	py.test -x -vv -r xw -p no:sugar \
 	--cov-report=term-missing --cov-report=html \
     --ini-config-override='{"app:main": {"sqlalchemy.db1.url": "mysql://root:qweqwe@localhost/rhodecode_test?charset=utf8"}}' \
     --cov=rhodecode rhodecode
 
 
 .PHONY: test-only-postgres
 ## run tests against postgres
 test-only-postgres:
 	PYTHONHASHSEED=random \
 	py.test -x -vv -r xw -p no:sugar \
 	--cov-report=term-missing --cov-report=html \
     --ini-config-override='{"app:main": {"sqlalchemy.db1.url": "postgresql://postgres:qweqwe@localhost/rhodecode_test"}}' \
     --cov=rhodecode rhodecode
 
 .PHONY: docs
 ## build docs
 docs:
 	(cd docs; nix-build default.nix -o result; make clean html)
 
 
 .PHONY: docs-clean
 ## Cleanup docs
 docs-clean:
 	(cd docs; make clean)
 
 
 .PHONY: docs-cleanup
 ## Cleanup docs
 docs-cleanup:
 	(cd docs; make cleanup)
 
 
 .PHONY: web-build
 ## Build JS packages static/js
 # https://hub.docker.com/r/huli/grunt
 web-build:
 	NODE_PATH=$(NODE_PATH) $(GRUNT)
 
 	# check required files
 	STATIC_CHECK="/robots.txt /502.html \
 				  /js/scripts.min.js /js/rhodecode-components.js \
 				  /css/style.css /css/style-polymer.css /css/style-ipython.css"
 
 	for file in $STATIC_CHECK;
 	do
 		if [ ! -f rhodecode/public/$file ]; then
 		  echo "Missing $file expected after web-build"
 		  exit 1
 		fi
 	done
 
 .PHONY: pip-packages
 ## Show outdated packages
 pip-packages:
 	python ${OUTDATED_PACKAGES}
 
 
 .PHONY: build
 ## Build sdist/egg
 build:
 	python -m build
 
 
 .PHONY: dev-env
 ## make dev-env based on the requirements files and install develop of packages
 dev-env:
 	pip install build virtualenv
 	pushd ../rhodecode-vcsserver/ && make dev-env && popd
 	pip wheel --wheel-dir=/home/rhodecode/.cache/pip/wheels -r requirements.txt -r requirements_rc_tools.txt -r requirements_test.txt -r requirements_debug.txt
 	pip install --no-index --find-links=/home/rhodecode/.cache/pip/wheels -r requirements.txt -r requirements_rc_tools.txt -r requirements_test.txt -r requirements_debug.txt
 	pip install -e .
 
 
 .PHONY: dev-srv
 ## run develop server instance, docker exec -it $(docker ps -q --filter 'name=dev-enterprise-ce') /bin/bash
 dev-srv:
 	pserve --reload .dev/dev.ini
 
 .PHONY: dev-srv-g
 ## run gunicorn multi process workers
 dev-srv-g:
-	gunicorn --workers=4 --paste .dev/dev_g.ini --bind=0.0.0.0:10020 --worker-class=gevent --threads=1 --config=configs/gunicorn_config.py --timeout=120
+	gunicorn --paste .dev/dev.ini --bind=0.0.0.0:10020 --config=.dev/gunicorn_config.py
 
 
 # Default command on calling make
 .DEFAULT_GOAL := show-help
 
 .PHONY: show-help
 show-help:
 	@echo "$$(tput bold)Available rules:$$(tput sgr0)"
 	@echo
 	@sed -n -e "/^## / { \
 		h; \
 		s/.*//; \
 		:doc" \
 		-e "H; \
 		n; \
 		s/^## //; \
 		t doc" \
 		-e "s/:.*//; \
 		G; \
 		s/\\n## /---/; \
 		s/\\n/ /g; \
 		p; \
 	}" ${MAKEFILE_LIST} \
 	| LC_ALL='C' sort --ignore-case \
 	| awk -F '---' \
 		-v ncol=$$(tput cols) \
 		-v indent=19 \
 		-v col_on="$$(tput setaf 6)" \
 		-v col_off="$$(tput sgr0)" \
 	'{ \
 		printf "%s%*s%s ", col_on, -indent, $$1, col_off; \
 		n = split($$2, words, " "); \
 		line_length = ncol - indent; \
 		for (i = 1; i <= n; i++) { \
 			line_length -= length(words[i]) + 1; \
 			if (line_length <= 0) { \
 				line_length = ncol - indent - length(words[i]) - 1; \
 				printf "\n%*s ", -indent, " "; \
 			} \
 			printf "%s ", words[i]; \
 		} \
 		printf "\n"; \
 	}'

configs/gunicorn_config.py

@@ -1,506 +1,510 b''
 """
 Gunicorn config extension and hooks. This config file adds some extra settings and memory management.
 Gunicorn configuration should be managed by .ini files entries of RhodeCode or VCSServer
 """
 
 import gc
 import os
 import sys
 import math
 import time
 import threading
 import traceback
 import random
 import socket
 import dataclasses
 from gunicorn.glogging import Logger
 
 
 def get_workers():
     import multiprocessing
     return multiprocessing.cpu_count() * 2 + 1
 
 
 bind = "127.0.0.1:10020"
 
 
 # Error logging output for gunicorn (-) is stdout
 errorlog = '-'
 
 # Access logging output for gunicorn (-) is stdout
 accesslog = '-'
 
 
 # SERVER MECHANICS
 # None == system temp dir
 # worker_tmp_dir is recommended to be set to some tmpfs
 worker_tmp_dir = None
 tmp_upload_dir = None
 
 # use re-use port logic
 #reuse_port = True
 
 # Custom log format
 #access_log_format = (
 #    '%(t)s %(p)s INFO  [GNCRN] %(h)-15s rqt:%(L)s %(s)s %(b)-6s "%(m)s:%(U)s %(q)s" usr:%(u)s "%(f)s" "%(a)s"')
 
 # loki format for easier parsing in grafana
 access_log_format = (
     'time="%(t)s" pid=%(p)s level="INFO" type="[GNCRN]" ip="%(h)-15s" rqt="%(L)s" response_code="%(s)s" response_bytes="%(b)-6s" uri="%(m)s:%(U)s %(q)s" user=":%(u)s" user_agent="%(a)s"')
 
 
 # Sets the number of process workers. More workers means more concurrent connections
 # RhodeCode can handle at the same time. Each additional worker also it increases
 # memory usage as each has it's own set of caches.
 # Recommended value is (2 * NUMBER_OF_CPUS + 1), eg 2CPU = 5 workers, but no more
 # than 8-10 unless for huge deployments .e.g 700-1000 users.
 # `instance_id = *` must be set in the [app:main] section below (which is the default)
 # when using more than 1 worker.
 workers = 4
 
 # self adjust workers based on CPU count, to use maximum of CPU and not overquota the resources
 # workers = get_workers()
 
 # Gunicorn access log level
 loglevel = 'info'
 
 # Process name visible in process list
 proc_name = 'rhodecode_enterprise'
 
 # Type of worker class, one of `sync`, `gevent`
 # currently `sync` is the only option allowed.
 worker_class = 'gevent'
 
 # The maximum number of simultaneous clients. Valid only for gevent
 worker_connections = 10
 
 # Max number of requests that worker will handle before being gracefully restarted.
 # Prevents memory leaks, jitter adds variability so not all workers are restarted at once.
 max_requests = 2000
 max_requests_jitter = 30
 
 # The maximum number of pending connections.
 # Exceeding this number results in the client getting an error when attempting to connect.
 backlog = 64
 
 # Amount of time a worker can spend with handling a request before it
 # gets killed and restarted. By default set to 21600 (6hrs)
 # Examples: 1800 (30min), 3600 (1hr), 7200 (2hr), 43200 (12h)
 timeout = 21600
 
 # The maximum size of HTTP request line in bytes.
 # 0 for unlimited
 limit_request_line = 0
 
 # Limit the number of HTTP headers fields in a request.
 # By default this value is 100 and can't be larger than 32768.
 limit_request_fields = 32768
 
 # Limit the allowed size of an HTTP request header field.
 # Value is a positive number or 0.
 # Setting it to 0 will allow unlimited header field sizes.
 limit_request_field_size = 0
 
 # Timeout for graceful workers restart.
 # After receiving a restart signal, workers have this much time to finish
 # serving requests. Workers still alive after the timeout (starting from the
 # receipt of the restart signal) are force killed.
 # Examples: 1800 (30min), 3600 (1hr), 7200 (2hr), 43200 (12h)
 graceful_timeout = 21600
 
 # The number of seconds to wait for requests on a Keep-Alive connection.
 # Generally set in the 1-5 seconds range.
 keepalive = 2
 
 # Maximum memory usage that each worker can use before it will receive a
 # graceful restart signal 0 = memory monitoring is disabled
 # Examples: 268435456 (256MB), 536870912 (512MB)
 # 1073741824 (1GB), 2147483648 (2GB), 4294967296 (4GB)
+# Dynamic formula 1024 * 1024 * 256 == 256MBs
 memory_max_usage = 0
 
 # How often in seconds to check for memory usage for each gunicorn worker
 memory_usage_check_interval = 60
 
 # Threshold value for which we don't recycle worker if GarbageCollection
-# frees up enough resources. Before each restart we try to run GC on worker
+# frees up enough resources. Before each restart, we try to run GC on worker
 # in case we get enough free memory after that, restart will not happen.
 memory_usage_recovery_threshold = 0.8
 
 
 @dataclasses.dataclass
 class MemoryCheckConfig:
     max_usage: int
     check_interval: int
     recovery_threshold: float
 
 
 def _get_process_rss(pid=None):
     try:
         import psutil
         if pid:
             proc = psutil.Process(pid)
         else:
             proc = psutil.Process()
         return proc.memory_info().rss
     except Exception:
         return None
 
 
 def _get_config(ini_path):
     import configparser
 
     try:
         config = configparser.RawConfigParser()
         config.read(ini_path)
         return config
     except Exception:
         return None
 
 
 def get_memory_usage_params(config=None):
     # memory spec defaults
     _memory_max_usage = memory_max_usage
     _memory_usage_check_interval = memory_usage_check_interval
     _memory_usage_recovery_threshold = memory_usage_recovery_threshold
 
     if config:
         ini_path = os.path.abspath(config)
         conf = _get_config(ini_path)
 
         section = 'server:main'
         if conf and conf.has_section(section):
 
             if conf.has_option(section, 'memory_max_usage'):
                 _memory_max_usage = conf.getint(section, 'memory_max_usage')
 
             if conf.has_option(section, 'memory_usage_check_interval'):
                 _memory_usage_check_interval = conf.getint(section, 'memory_usage_check_interval')
 
             if conf.has_option(section, 'memory_usage_recovery_threshold'):
                 _memory_usage_recovery_threshold = conf.getfloat(section, 'memory_usage_recovery_threshold')
 
     _memory_max_usage = int(os.environ.get('RC_GUNICORN_MEMORY_MAX_USAGE', '')
                             or _memory_max_usage)
     _memory_usage_check_interval = int(os.environ.get('RC_GUNICORN_MEMORY_USAGE_CHECK_INTERVAL', '')
                                        or _memory_usage_check_interval)
     _memory_usage_recovery_threshold = float(os.environ.get('RC_GUNICORN_MEMORY_USAGE_RECOVERY_THRESHOLD', '')
                                              or _memory_usage_recovery_threshold)
 
     return MemoryCheckConfig(_memory_max_usage, _memory_usage_check_interval, _memory_usage_recovery_threshold)
 
 
 def _time_with_offset(check_interval):
     return time.time() - random.randint(0, check_interval/2.0)
 
 
 def pre_fork(server, worker):
     pass
 
 
 def post_fork(server, worker):
 
     memory_conf = get_memory_usage_params()
     _memory_max_usage = memory_conf.max_usage
     _memory_usage_check_interval = memory_conf.check_interval
     _memory_usage_recovery_threshold = memory_conf.recovery_threshold
 
     worker._memory_max_usage = int(os.environ.get('RC_GUNICORN_MEMORY_MAX_USAGE', '')
                                    or _memory_max_usage)
     worker._memory_usage_check_interval = int(os.environ.get('RC_GUNICORN_MEMORY_USAGE_CHECK_INTERVAL', '')
                                               or _memory_usage_check_interval)
     worker._memory_usage_recovery_threshold = float(os.environ.get('RC_GUNICORN_MEMORY_USAGE_RECOVERY_THRESHOLD', '')
                                                     or _memory_usage_recovery_threshold)
 
     # register memory last check time, with some random offset so we don't recycle all
     # at once
     worker._last_memory_check_time = _time_with_offset(_memory_usage_check_interval)
 
     if _memory_max_usage:
         server.log.info("pid=[%-10s] WORKER spawned with max memory set at %s", worker.pid,
                         _format_data_size(_memory_max_usage))
     else:
         server.log.info("pid=[%-10s] WORKER spawned", worker.pid)
 
 
 def pre_exec(server):
     server.log.info("Forked child, re-executing.")
 
 
 def on_starting(server):
     server_lbl = '{} {}'.format(server.proc_name, server.address)
     server.log.info("Server %s is starting.", server_lbl)
+    server.log.info('Config:')
+    server.log.info(f"\n{server.cfg}")
+    server.log.info(get_memory_usage_params())
 
 
 def when_ready(server):
     server.log.info("Server %s is ready. Spawning workers", server)
 
 
 def on_reload(server):
     pass
 
 
 def _format_data_size(size, unit="B", precision=1, binary=True):
     """Format a number using SI units (kilo, mega, etc.).
 
     ``size``: The number as a float or int.
 
     ``unit``: The unit name in plural form. Examples: "bytes", "B".
 
     ``precision``: How many digits to the right of the decimal point. Default
     is 1.  0 suppresses the decimal point.
 
     ``binary``: If false, use base-10 decimal prefixes (kilo = K = 1000).
     If true, use base-2 binary prefixes (kibi = Ki = 1024).
 
     ``full_name``: If false (default), use the prefix abbreviation ("k" or
     "Ki").  If true, use the full prefix ("kilo" or "kibi"). If false,
     use abbreviation ("k" or "Ki").
 
     """
 
     if not binary:
         base = 1000
         multiples = ('', 'k', 'M', 'G', 'T', 'P', 'E', 'Z', 'Y')
     else:
         base = 1024
         multiples = ('', 'Ki', 'Mi', 'Gi', 'Ti', 'Pi', 'Ei', 'Zi', 'Yi')
 
     sign = ""
     if size > 0:
         m = int(math.log(size, base))
     elif size < 0:
         sign = "-"
         size = -size
         m = int(math.log(size, base))
     else:
         m = 0
     if m > 8:
         m = 8
 
     if m == 0:
         precision = '%.0f'
     else:
         precision = '%%.%df' % precision
 
     size = precision % (size / math.pow(base, m))
 
     return '%s%s %s%s' % (sign, size.strip(), multiples[m], unit)
 
 
 def _check_memory_usage(worker):
     _memory_max_usage = worker._memory_max_usage
     if not _memory_max_usage:
         return
 
     _memory_usage_check_interval = worker._memory_usage_check_interval
     _memory_usage_recovery_threshold = memory_max_usage * worker._memory_usage_recovery_threshold
 
     elapsed = time.time() - worker._last_memory_check_time
     if elapsed > _memory_usage_check_interval:
         mem_usage = _get_process_rss()
         if mem_usage and mem_usage > _memory_max_usage:
             worker.log.info(
                 "memory usage %s > %s, forcing gc",
                 _format_data_size(mem_usage), _format_data_size(_memory_max_usage))
             # Try to clean it up by forcing a full collection.
             gc.collect()
             mem_usage = _get_process_rss()
             if mem_usage > _memory_usage_recovery_threshold:
                 # Didn't clean up enough, we'll have to terminate.
                 worker.log.warning(
                     "memory usage %s > %s after gc, quitting",
                     _format_data_size(mem_usage), _format_data_size(_memory_max_usage))
                 # This will cause worker to auto-restart itself
                 worker.alive = False
         worker._last_memory_check_time = time.time()
 
 
 def worker_int(worker):
     worker.log.info("pid=[%-10s] worker received INT or QUIT signal", worker.pid)
 
     # get traceback info, on worker crash
     def get_thread_id(t_id):
         id2name = dict([(th.ident, th.name) for th in threading.enumerate()])
         return id2name.get(t_id, "unknown_thread_id")
 
     code = []
-    for thread_id, stack in sys._current_frames().items():
+    for thread_id, stack in sys._current_frames().items():  # noqa
         code.append(
             "\n# Thread: %s(%d)" % (get_thread_id(thread_id), thread_id))
         for fname, lineno, name, line in traceback.extract_stack(stack):
             code.append('File: "%s", line %d, in %s' % (fname, lineno, name))
             if line:
                 code.append("  %s" % (line.strip()))
     worker.log.debug("\n".join(code))
 
 
 def worker_abort(worker):
     worker.log.info("pid=[%-10s] worker received SIGABRT signal", worker.pid)
 
 
 def worker_exit(server, worker):
     worker.log.info("pid=[%-10s] worker exit", worker.pid)
 
 
 def child_exit(server, worker):
     worker.log.info("pid=[%-10s] worker child exit", worker.pid)
 
 
 def pre_request(worker, req):
     worker.start_time = time.time()
     worker.log.debug(
         "GNCRN PRE  WORKER [cnt:%s]: %s %s", worker.nr, req.method, req.path)
 
 
 def post_request(worker, req, environ, resp):
     total_time = time.time() - worker.start_time
     # Gunicorn sometimes has problems with reading the status_code
     status_code = getattr(resp, 'status_code', '')
     worker.log.debug(
         "GNCRN POST WORKER [cnt:%s]: %s %s resp: %s, Load Time: %.4fs",
         worker.nr, req.method, req.path, status_code, total_time)
     _check_memory_usage(worker)
 
 
 def _filter_proxy(ip):
     """
     Passed in IP addresses in HEADERS can be in a special format of multiple
     ips. Those comma separated IPs are passed from various proxies in the
     chain of request processing. The left-most being the original client.
     We only care about the first IP which came from the org. client.
 
     :param ip: ip string from headers
     """
     if ',' in ip:
         _ips = ip.split(',')
         _first_ip = _ips[0].strip()
         return _first_ip
     return ip
 
 
 def _filter_port(ip):
     """
     Removes a port from ip, there are 4 main cases to handle here.
     - ipv4 eg. 127.0.0.1
     - ipv6 eg. ::1
     - ipv4+port eg. 127.0.0.1:8080
     - ipv6+port eg. [::1]:8080
 
     :param ip:
     """
     def is_ipv6(ip_addr):
         if hasattr(socket, 'inet_pton'):
             try:
                 socket.inet_pton(socket.AF_INET6, ip_addr)
             except socket.error:
                 return False
         else:
             return False
         return True
 
     if ':' not in ip:  # must be ipv4 pure ip
         return ip
 
     if '[' in ip and ']' in ip:  # ipv6 with port
         return ip.split(']')[0][1:].lower()
 
     # must be ipv6 or ipv4 with port
     if is_ipv6(ip):
         return ip
     else:
         ip, _port = ip.split(':')[:2]  # means ipv4+port
         return ip
 
 
 def get_ip_addr(environ):
     proxy_key = 'HTTP_X_REAL_IP'
     proxy_key2 = 'HTTP_X_FORWARDED_FOR'
     def_key = 'REMOTE_ADDR'
 
     def _filters(x):
         return _filter_port(_filter_proxy(x))
 
     ip = environ.get(proxy_key)
     if ip:
         return _filters(ip)
 
     ip = environ.get(proxy_key2)
     if ip:
         return _filters(ip)
 
     ip = environ.get(def_key, '0.0.0.0')
     return _filters(ip)
 
 
 class RhodeCodeLogger(Logger):
     """
     Custom Logger that allows some customization that gunicorn doesn't allow
     """
 
     datefmt = r"%Y-%m-%d %H:%M:%S"
 
     def __init__(self, cfg):
         Logger.__init__(self, cfg)
 
     def now(self):
         """ return date in RhodeCode Log format """
         now = time.time()
         msecs = int((now - int(now)) * 1000)
         return time.strftime(self.datefmt, time.localtime(now)) + '.{0:03d}'.format(msecs)
 
     def atoms(self, resp, req, environ, request_time):
         """ Gets atoms for log formatting.
         """
         status = resp.status
         if isinstance(status, str):
             status = status.split(None, 1)[0]
         atoms = {
             'h': get_ip_addr(environ),
             'l': '-',
             'u': self._get_user(environ) or '-',
             't': self.now(),
             'r': "%s %s %s" % (environ['REQUEST_METHOD'],
                                environ['RAW_URI'],
                                environ["SERVER_PROTOCOL"]),
             's': status,
             'm': environ.get('REQUEST_METHOD'),
             'U': environ.get('PATH_INFO'),
             'q': environ.get('QUERY_STRING'),
             'H': environ.get('SERVER_PROTOCOL'),
             'b': getattr(resp, 'sent', None) is not None and str(resp.sent) or '-',
             'B': getattr(resp, 'sent', None),
             'f': environ.get('HTTP_REFERER', '-'),
             'a': environ.get('HTTP_USER_AGENT', '-'),
             'T': request_time.seconds,
             'D': (request_time.seconds * 1000000) + request_time.microseconds,
             'M': (request_time.seconds * 1000) + int(request_time.microseconds/1000),
             'L': "%d.%06d" % (request_time.seconds, request_time.microseconds),
             'p': "<%s>" % os.getpid()
         }
 
         # add request headers
         if hasattr(req, 'headers'):
             req_headers = req.headers
         else:
             req_headers = req
 
         if hasattr(req_headers, "items"):
             req_headers = req_headers.items()
 
         atoms.update({"{%s}i" % k.lower(): v for k, v in req_headers})
 
         resp_headers = resp.headers
         if hasattr(resp_headers, "items"):
             resp_headers = resp_headers.items()
 
         # add response headers
         atoms.update({"{%s}o" % k.lower(): v for k, v in resp_headers})
 
         # add environ variables
         environ_variables = environ.items()
         atoms.update({"{%s}e" % k.lower(): v for k, v in environ_variables})
 
         return atoms
 
 
 logger_class = RhodeCodeLogger

configs/production.ini

@@ -1,814 +1,813 b''
-
 
 ; #########################################
 ; RHODECODE COMMUNITY EDITION CONFIGURATION
 ; #########################################
 
 [DEFAULT]
 ; Debug flag sets all loggers to debug, and enables request tracking
 debug = false
 
 ; ########################################################################
 ; EMAIL CONFIGURATION
 ; These settings will be used by the RhodeCode mailing system
 ; ########################################################################
 
 ; prefix all emails subjects with given prefix, helps filtering out emails
 #email_prefix = [RhodeCode]
 
 ; email FROM address all mails will be sent
 #app_email_from = rhodecode-noreply@localhost
 
 #smtp_server = mail.server.com
 #smtp_username =
 #smtp_password =
 #smtp_port =
 #smtp_use_tls = false
 #smtp_use_ssl = true
 
 [server:main]
 ; COMMON HOST/IP CONFIG, This applies mostly to develop setup,
 ; Host port for gunicorn are controlled by gunicorn_conf.py
 host = 127.0.0.1
 port = 10020
 
 
 ; ###########################
 ; GUNICORN APPLICATION SERVER
 ; ###########################
 
 ; run with gunicorn --paste rhodecode.ini --config gunicorn_conf.py
 
 ; Module to use, this setting shouldn't be changed
 use = egg:gunicorn#main
 
 ; Prefix middleware for RhodeCode.
 ; recommended when using proxy setup.
 ; allows to set RhodeCode under a prefix in server.
 ; eg https://server.com/custom_prefix. Enable `filter-with =` option below as well.
 ; And set your prefix like: `prefix = /custom_prefix`
 ; be sure to also set beaker.session.cookie_path = /custom_prefix if you need
 ; to make your cookies only work on prefix url
 [filter:proxy-prefix]
 use = egg:PasteDeploy#prefix
 prefix = /
 
 [app:main]
 ; The %(here)s variable will be replaced with the absolute path of parent directory
 ; of this file
 ; Each option in the app:main can be override by an environmental variable
 ;
 ;To override an option:
 ;
 ;RC_<KeyName>
 ;Everything should be uppercase, . and - should be replaced by _.
 ;For example, if you have these configuration settings:
 ;rc_cache.repo_object.backend = foo
 ;can be overridden by
 ;export RC_CACHE_REPO_OBJECT_BACKEND=foo
 
 use = egg:rhodecode-enterprise-ce
 
 ; enable proxy prefix middleware, defined above
 #filter-with = proxy-prefix
 
 ; encryption key used to encrypt social plugin tokens,
 ; remote_urls with credentials etc, if not set it defaults to
 ; `beaker.session.secret`
 #rhodecode.encrypted_values.secret =
 
 ; decryption strict mode (enabled by default). It controls if decryption raises
 ; `SignatureVerificationError` in case of wrong key, or damaged encryption data.
 #rhodecode.encrypted_values.strict = false
 
 ; Pick algorithm for encryption. Either fernet (more secure) or aes (default)
 ; fernet is safer, and we strongly recommend switching to it.
 ; Due to backward compatibility aes is used as default.
 #rhodecode.encrypted_values.algorithm = fernet
 
 ; Return gzipped responses from RhodeCode (static files/application)
 gzip_responses = false
 
 ; Auto-generate javascript routes file on startup
 generate_js_files = false
 
 ; System global default language.
 ; All available languages: en (default), be, de, es, fr, it, ja, pl, pt, ru, zh
 lang = en
 
 ; Perform a full repository scan and import on each server start.
 ; Settings this to true could lead to very long startup time.
 startup.import_repos = false
 
 ; URL at which the application is running. This is used for Bootstrapping
 ; requests in context when no web request is available. Used in ishell, or
 ; SSH calls. Set this for events to receive proper url for SSH calls.
 app.base_url = http://rhodecode.local
 
 ; Unique application ID. Should be a random unique string for security.
 app_instance_uuid = rc-production
 
 ; Cut off limit for large diffs (size in bytes). If overall diff size on
 ; commit, or pull request exceeds this limit this diff will be displayed
 ; partially. E.g 512000 == 512Kb
 cut_off_limit_diff = 512000
 
 ; Cut off limit for large files inside diffs (size in bytes). Each individual
 ; file inside diff which exceeds this limit will be displayed partially.
 ;  E.g 128000 == 128Kb
 cut_off_limit_file = 128000
 
 ; Use cached version of vcs repositories everywhere. Recommended to be `true`
 vcs_full_cache = true
 
 ; Force https in RhodeCode, fixes https redirects, assumes it's always https.
 ; Normally this is controlled by proper flags sent from http server such as Nginx or Apache
 force_https = false
 
 ; use Strict-Transport-Security headers
 use_htsts = false
 
 ; Set to true if your repos are exposed using the dumb protocol
 git_update_server_info = false
 
 ; RSS/ATOM feed options
 rss_cut_off_limit = 256000
 rss_items_per_page = 10
 rss_include_diff = false
 
 ; gist URL alias, used to create nicer urls for gist. This should be an
 ; url that does rewrites to _admin/gists/{gistid}.
 ; example: http://gist.rhodecode.org/{gistid}. Empty means use the internal
 ; RhodeCode url, ie. http[s]://rhodecode.server/_admin/gists/{gistid}
 gist_alias_url =
 
 ; List of views (using glob pattern syntax) that AUTH TOKENS could be
 ; used for access.
 ; Adding ?auth_token=TOKEN_HASH to the url authenticates this request as if it
 ; came from the the logged in user who own this authentication token.
 ; Additionally @TOKEN syntax can be used to bound the view to specific
 ; authentication token. Such view would be only accessible when used together
 ; with this authentication token
 ; list of all views can be found under `/_admin/permissions/auth_token_access`
 ; The list should be "," separated and on a single line.
 ; Most common views to enable:
 
 #    RepoCommitsView:repo_commit_download
 #    RepoCommitsView:repo_commit_patch
 #    RepoCommitsView:repo_commit_raw
 #    RepoCommitsView:repo_commit_raw@TOKEN
 #    RepoFilesView:repo_files_diff
 #    RepoFilesView:repo_archivefile
 #    RepoFilesView:repo_file_raw
 #    GistView:*
 api_access_controllers_whitelist =
 
 ; Default encoding used to convert from and to unicode
 ; can be also a comma separated list of encoding in case of mixed encodings
 default_encoding = UTF-8
 
 ; instance-id prefix
 ; a prefix key for this instance used for cache invalidation when running
 ; multiple instances of RhodeCode, make sure it's globally unique for
 ; all running RhodeCode instances. Leave empty if you don't use it
 instance_id =
 
 ; Fallback authentication plugin. Set this to a plugin ID to force the usage
 ; of an authentication plugin also if it is disabled by it's settings.
 ; This could be useful if you are unable to log in to the system due to broken
 ; authentication settings. Then you can enable e.g. the internal RhodeCode auth
 ; module to log in again and fix the settings.
 ; Available builtin plugin IDs (hash is part of the ID):
 ; egg:rhodecode-enterprise-ce#rhodecode
 ; egg:rhodecode-enterprise-ce#pam
 ; egg:rhodecode-enterprise-ce#ldap
 ; egg:rhodecode-enterprise-ce#jasig_cas
 ; egg:rhodecode-enterprise-ce#headers
 ; egg:rhodecode-enterprise-ce#crowd
 
 #rhodecode.auth_plugin_fallback = egg:rhodecode-enterprise-ce#rhodecode
 
 ; Flag to control loading of legacy plugins in py:/path format
 auth_plugin.import_legacy_plugins = true
 
 ; alternative return HTTP header for failed authentication. Default HTTP
 ; response is 401 HTTPUnauthorized. Currently HG clients have troubles with
 ; handling that causing a series of failed authentication calls.
 ; Set this variable to 403 to return HTTPForbidden, or any other HTTP code
 ; This will be served instead of default 401 on bad authentication
 auth_ret_code =
 
 ; use special detection method when serving auth_ret_code, instead of serving
 ; ret_code directly, use 401 initially (Which triggers credentials prompt)
 ; and then serve auth_ret_code to clients
 auth_ret_code_detection = false
 
 ; locking return code. When repository is locked return this HTTP code. 2XX
 ; codes don't break the transactions while 4XX codes do
 lock_ret_code = 423
 
 ; allows to change the repository location in settings page
 allow_repo_location_change = true
 
 ; allows to setup custom hooks in settings page
 allow_custom_hooks_settings = true
 
 ; Generated license token required for EE edition license.
 ; New generated token value can be found in Admin > settings > license page.
 license_token =
 
 ; This flag hides sensitive information on the license page such as token, and license data
 license.hide_license_info = false
 
 ; supervisor connection uri, for managing supervisor and logs.
 supervisor.uri =
 
 ; supervisord group name/id we only want this RC instance to handle
 supervisor.group_id = prod
 
 ; Display extended labs settings
 labs_settings_active = true
 
 ; Custom exception store path, defaults to TMPDIR
 ; This is used to store exception from RhodeCode in shared directory
 #exception_tracker.store_path =
 
 ; Send email with exception details when it happens
 #exception_tracker.send_email = false
 
 ; Comma separated list of recipients for exception emails,
 ; e.g [e-mail],[e-mail]
 ; Can be left empty, then emails will be sent to ALL super-admins
 #exception_tracker.send_email_recipients =
 
 ; optional prefix to Add to email Subject
 #exception_tracker.email_prefix = [RHODECODE ERROR]
 
 ; File store configuration. This is used to store and serve uploaded files
 file_store.enabled = true
 
 ; Storage backend, available options are: local
 file_store.backend = local
 
 ; path to store the uploaded binaries
 file_store.storage_path = %(here)s/data/file_store
 
 ; Uncomment and set this path to control settings for archive download cache.
 ; Generated repo archives will be cached at this location
 ; and served from the cache during subsequent requests for the same archive of
 ; the repository. This path is important to be shared across filesystems and with
 ; RhodeCode and vcsserver
 
 ; Default is $cache_dir/archive_cache if not set
 archive_cache.store_dir = %(here)s/data/archive_cache
 
 ; The limit in GB sets how much data we cache before recycling last used, defaults to 10 gb
-archive_cache.cache_size_gb = 10
+archive_cache.cache_size_gb = 40
 
 ; By default cache uses sharding technique, this specifies how many shards are there
-archive_cache.cache_shards = 10
+archive_cache.cache_shards = 4
 
 ; #############
 ; CELERY CONFIG
 ; #############
 
 ; manually run celery: /path/to/celery worker --task-events --beat --app rhodecode.lib.celerylib.loader --scheduler rhodecode.lib.celerylib.scheduler.RcScheduler --loglevel DEBUG --ini /path/to/rhodecode.ini
 
 use_celery = false
 
 ; path to store schedule database
 #celerybeat-schedule.path =
 
 ; connection url to the message broker (default redis)
 celery.broker_url = redis://localhost:6379/8
 
 ; rabbitmq example
 #celery.broker_url = amqp://rabbitmq:qweqwe@localhost:5672/rabbitmqhost
 
 ; maximum tasks to execute before worker restart
 celery.max_tasks_per_child = 20
 
 ; tasks will never be sent to the queue, but executed locally instead.
 celery.task_always_eager = false
 
 ; #############
 ; DOGPILE CACHE
 ; #############
 
 ; Default cache dir for caches. Putting this into a ramdisk can boost performance.
 ; eg. /tmpfs/data_ramdisk, however this directory might require large amount of space
 cache_dir = %(here)s/data
 
 ; *********************************************
 ; `sql_cache_short` cache for heavy SQL queries
 ; Only supported backend is `memory_lru`
 ; *********************************************
 rc_cache.sql_cache_short.backend = dogpile.cache.rc.memory_lru
 rc_cache.sql_cache_short.expiration_time = 30
 
 
 ; *****************************************************
 ; `cache_repo_longterm` cache for repo object instances
 ; Only supported backend is `memory_lru`
 ; *****************************************************
 rc_cache.cache_repo_longterm.backend = dogpile.cache.rc.memory_lru
 ; by default we use 30 Days, cache is still invalidated on push
 rc_cache.cache_repo_longterm.expiration_time = 2592000
 ; max items in LRU cache, set to smaller number to save memory, and expire last used caches
 rc_cache.cache_repo_longterm.max_size = 10000
 
 
 ; *********************************************
 ; `cache_general` cache for general purpose use
 ; for simplicity use rc.file_namespace backend,
 ; for performance and scale use rc.redis
 ; *********************************************
 rc_cache.cache_general.backend = dogpile.cache.rc.file_namespace
 rc_cache.cache_general.expiration_time = 43200
 ; file cache store path. Defaults to `cache_dir =` value or tempdir if both values are not set
 #rc_cache.cache_general.arguments.filename = /tmp/cache_general_db
 
 ; alternative `cache_general` redis backend with distributed lock
 #rc_cache.cache_general.backend = dogpile.cache.rc.redis
 #rc_cache.cache_general.expiration_time = 300
 
 ; redis_expiration_time needs to be greater then expiration_time
 #rc_cache.cache_general.arguments.redis_expiration_time = 7200
 
 #rc_cache.cache_general.arguments.host = localhost
 #rc_cache.cache_general.arguments.port = 6379
 #rc_cache.cache_general.arguments.db = 0
 #rc_cache.cache_general.arguments.socket_timeout = 30
 ; more Redis options: https://dogpilecache.sqlalchemy.org/en/latest/api.html#redis-backends
 #rc_cache.cache_general.arguments.distributed_lock = true
 
 ; auto-renew lock to prevent stale locks, slower but safer. Use only if problems happen
 #rc_cache.cache_general.arguments.lock_auto_renewal = true
 
 ; *************************************************
 ; `cache_perms` cache for permission tree, auth TTL
 ; for simplicity use rc.file_namespace backend,
 ; for performance and scale use rc.redis
 ; *************************************************
 rc_cache.cache_perms.backend = dogpile.cache.rc.file_namespace
 rc_cache.cache_perms.expiration_time = 3600
 ; file cache store path. Defaults to `cache_dir =` value or tempdir if both values are not set
 #rc_cache.cache_perms.arguments.filename = /tmp/cache_perms_db
 
 ; alternative `cache_perms` redis backend with distributed lock
 #rc_cache.cache_perms.backend = dogpile.cache.rc.redis
 #rc_cache.cache_perms.expiration_time = 300
 
 ; redis_expiration_time needs to be greater then expiration_time
 #rc_cache.cache_perms.arguments.redis_expiration_time = 7200
 
 #rc_cache.cache_perms.arguments.host = localhost
 #rc_cache.cache_perms.arguments.port = 6379
 #rc_cache.cache_perms.arguments.db = 0
 #rc_cache.cache_perms.arguments.socket_timeout = 30
 ; more Redis options: https://dogpilecache.sqlalchemy.org/en/latest/api.html#redis-backends
 #rc_cache.cache_perms.arguments.distributed_lock = true
 
 ; auto-renew lock to prevent stale locks, slower but safer. Use only if problems happen
 #rc_cache.cache_perms.arguments.lock_auto_renewal = true
 
 ; ***************************************************
 ; `cache_repo` cache for file tree, Readme, RSS FEEDS
 ; for simplicity use rc.file_namespace backend,
 ; for performance and scale use rc.redis
 ; ***************************************************
 rc_cache.cache_repo.backend = dogpile.cache.rc.file_namespace
 rc_cache.cache_repo.expiration_time = 2592000
 ; file cache store path. Defaults to `cache_dir =` value or tempdir if both values are not set
 #rc_cache.cache_repo.arguments.filename = /tmp/cache_repo_db
 
 ; alternative `cache_repo` redis backend with distributed lock
 #rc_cache.cache_repo.backend = dogpile.cache.rc.redis
 #rc_cache.cache_repo.expiration_time = 2592000
 
 ; redis_expiration_time needs to be greater then expiration_time
 #rc_cache.cache_repo.arguments.redis_expiration_time = 2678400
 
 #rc_cache.cache_repo.arguments.host = localhost
 #rc_cache.cache_repo.arguments.port = 6379
 #rc_cache.cache_repo.arguments.db = 1
 #rc_cache.cache_repo.arguments.socket_timeout = 30
 ; more Redis options: https://dogpilecache.sqlalchemy.org/en/latest/api.html#redis-backends
 #rc_cache.cache_repo.arguments.distributed_lock = true
 
 ; auto-renew lock to prevent stale locks, slower but safer. Use only if problems happen
 #rc_cache.cache_repo.arguments.lock_auto_renewal = true
 
 ; ##############
 ; BEAKER SESSION
 ; ##############
 
 ; beaker.session.type is type of storage options for the logged users sessions. Current allowed
 ; types are file, ext:redis, ext:database, ext:memcached, and memory (default if not specified).
 ; Fastest ones are Redis and ext:database
 beaker.session.type = file
 beaker.session.data_dir = %(here)s/data/sessions
 
 ; Redis based sessions
 #beaker.session.type = ext:redis
 #beaker.session.url = redis://127.0.0.1:6379/2
 
 ; DB based session, fast, and allows easy management over logged in users
 #beaker.session.type = ext:database
 #beaker.session.table_name = db_session
 #beaker.session.sa.url = postgresql://postgres:secret@localhost/rhodecode
 #beaker.session.sa.url = mysql://root:secret@127.0.0.1/rhodecode
 #beaker.session.sa.pool_recycle = 3600
 #beaker.session.sa.echo = false
 
 beaker.session.key = rhodecode
 beaker.session.secret = production-rc-uytcxaz
 beaker.session.lock_dir = %(here)s/data/sessions/lock
 
 ; Secure encrypted cookie. Requires AES and AES python libraries
 ; you must disable beaker.session.secret to use this
 #beaker.session.encrypt_key = key_for_encryption
 #beaker.session.validate_key = validation_key
 
 ; Sets session as invalid (also logging out user) if it haven not been
 ; accessed for given amount of time in seconds
 beaker.session.timeout = 2592000
 beaker.session.httponly = true
 
 ; Path to use for the cookie. Set to prefix if you use prefix middleware
 #beaker.session.cookie_path = /custom_prefix
 
 ; Set https secure cookie
 beaker.session.secure = false
 
 ; default cookie expiration time in seconds, set to `true` to set expire
 ; at browser close
 #beaker.session.cookie_expires = 3600
 
 ; #############################
 ; SEARCH INDEXING CONFIGURATION
 ; #############################
 
 ; Full text search indexer is available in rhodecode-tools under
 ; `rhodecode-tools index` command
 
 ; WHOOSH Backend, doesn't require additional services to run
 ; it works good with few dozen repos
 search.module = rhodecode.lib.index.whoosh
 search.location = %(here)s/data/index
 
 ; ####################
 ; CHANNELSTREAM CONFIG
 ; ####################
 
 ; channelstream enables persistent connections and live notification
 ; in the system. It's also used by the chat system
 
 channelstream.enabled = false
 
 ; server address for channelstream server on the backend
 channelstream.server = 127.0.0.1:9800
 
 ; location of the channelstream server from outside world
 ; use ws:// for http or wss:// for https. This address needs to be handled
 ; by external HTTP server such as Nginx or Apache
 ; see Nginx/Apache configuration examples in our docs
 channelstream.ws_url = ws://rhodecode.yourserver.com/_channelstream
 channelstream.secret = secret
 channelstream.history.location = %(here)s/channelstream_history
 
 ; Internal application path that Javascript uses to connect into.
 ; If you use proxy-prefix the prefix should be added before /_channelstream
 channelstream.proxy_path = /_channelstream
 
 
 ; ##############################
 ; MAIN RHODECODE DATABASE CONFIG
 ; ##############################
 
 #sqlalchemy.db1.url = sqlite:///%(here)s/rhodecode.db?timeout=30
 #sqlalchemy.db1.url = postgresql://postgres:qweqwe@localhost/rhodecode
 #sqlalchemy.db1.url = mysql://root:qweqwe@localhost/rhodecode?charset=utf8
 ; pymysql is an alternative driver for MySQL, use in case of problems with default one
 #sqlalchemy.db1.url = mysql+pymysql://root:qweqwe@localhost/rhodecode
 
 sqlalchemy.db1.url = postgresql://postgres:qweqwe@localhost/rhodecode
 
 ; see sqlalchemy docs for other advanced settings
 ; print the sql statements to output
 sqlalchemy.db1.echo = false
 
 ; recycle the connections after this amount of seconds
 sqlalchemy.db1.pool_recycle = 3600
 
 ; the number of connections to keep open inside the connection pool.
 ; 0 indicates no limit
 ; the general calculus with gevent is:
 ; if your system allows 500 concurrent greenlets (max_connections) that all do database access,
 ; then increase pool size + max overflow so that they add up to 500.
 #sqlalchemy.db1.pool_size = 5
 
 ; The number of connections to allow in connection pool "overflow", that is
 ; connections that can be opened above and beyond the pool_size setting,
 ; which defaults to five.
 #sqlalchemy.db1.max_overflow = 10
 
 ; Connection check ping, used to detect broken database connections
 ; could be enabled to better handle cases if MySQL has gone away errors
 #sqlalchemy.db1.ping_connection = true
 
 ; ##########
 ; VCS CONFIG
 ; ##########
 vcs.server.enable = true
 vcs.server = localhost:9900
 
 ; Web server connectivity protocol, responsible for web based VCS operations
 ; Available protocols are:
 ; `http` - use http-rpc backend (default)
 vcs.server.protocol = http
 
 ; Push/Pull operations protocol, available options are:
 ; `http` - use http-rpc backend (default)
 vcs.scm_app_implementation = http
 
 ; Push/Pull operations hooks protocol, available options are:
 ; `http` - use http-rpc backend (default)
 vcs.hooks.protocol = http
 
 ; Host on which this instance is listening for hooks. vcsserver will call this host to pull/push hooks so it should be
 ; accessible via network.
 ; Use vcs.hooks.host = "*" to bind to current hostname (for Docker)
 vcs.hooks.host = *
 
 ; Start VCSServer with this instance as a subprocess, useful for development
 vcs.start_server = false
 
 ; List of enabled VCS backends, available options are:
 ; `hg`  - mercurial
 ; `git` - git
 ; `svn` - subversion
 vcs.backends = hg, git, svn
 
 ; Wait this number of seconds before killing connection to the vcsserver
 vcs.connection_timeout = 3600
 
 ; Compatibility version when creating SVN repositories. Defaults to newest version when commented out.
 ; Set a numeric version for your current SVN e.g 1.8, or 1.12
 ; Legacy available options are: pre-1.4-compatible, pre-1.5-compatible, pre-1.6-compatible, pre-1.8-compatible, pre-1.9-compatible
 #vcs.svn.compatible_version = 1.8
 
 ; Cache flag to cache vcsserver remote calls locally
 ; It uses cache_region `cache_repo`
 vcs.methods.cache = true
 
 ; ####################################################
 ; Subversion proxy support (mod_dav_svn)
 ; Maps RhodeCode repo groups into SVN paths for Apache
 ; ####################################################
 
 ; Enable or disable the config file generation.
 svn.proxy.generate_config = false
 
 ; Generate config file with `SVNListParentPath` set to `On`.
 svn.proxy.list_parent_path = true
 
 ; Set location and file name of generated config file.
 svn.proxy.config_file_path = %(here)s/mod_dav_svn.conf
 
 ; alternative mod_dav config template. This needs to be a valid mako template
 ; Example template can be found in the source code:
 ; rhodecode/apps/svn_support/templates/mod-dav-svn.conf.mako
 #svn.proxy.config_template = ~/.rccontrol/enterprise-1/custom_svn_conf.mako
 
 ; Used as a prefix to the `Location` block in the generated config file.
 ; In most cases it should be set to `/`.
 svn.proxy.location_root = /
 
 ; Command to reload the mod dav svn configuration on change.
 ; Example: `/etc/init.d/apache2 reload` or /home/USER/apache_reload.sh
 ; Make sure user who runs RhodeCode process is allowed to reload Apache
 #svn.proxy.reload_cmd = /etc/init.d/apache2 reload
 
 ; If the timeout expires before the reload command finishes, the command will
 ; be killed. Setting it to zero means no timeout. Defaults to 10 seconds.
 #svn.proxy.reload_timeout = 10
 
 ; ####################
 ; SSH Support Settings
 ; ####################
 
 ; Defines if a custom authorized_keys file should be created and written on
 ; any change user ssh keys. Setting this to false also disables possibility
 ; of adding SSH keys by users from web interface. Super admins can still
 ; manage SSH Keys.
 ssh.generate_authorized_keyfile = false
 
 ; Options for ssh, default is `no-pty,no-port-forwarding,no-X11-forwarding,no-agent-forwarding`
 # ssh.authorized_keys_ssh_opts =
 
 ; Path to the authorized_keys file where the generate entries are placed.
 ; It is possible to have multiple key files specified in `sshd_config` e.g.
 ; AuthorizedKeysFile %h/.ssh/authorized_keys %h/.ssh/authorized_keys_rhodecode
 ssh.authorized_keys_file_path = ~/.ssh/authorized_keys_rhodecode
 
 ; Command to execute the SSH wrapper. The binary is available in the
 ; RhodeCode installation directory.
 ; e.g ~/.rccontrol/community-1/profile/bin/rc-ssh-wrapper
 ssh.wrapper_cmd = ~/.rccontrol/community-1/rc-ssh-wrapper
 
 ; Allow shell when executing the ssh-wrapper command
 ssh.wrapper_cmd_allow_shell = false
 
 ; Enables logging, and detailed output send back to the client during SSH
 ; operations. Useful for debugging, shouldn't be used in production.
 ssh.enable_debug_logging = false
 
 ; Paths to binary executable, by default they are the names, but we can
 ; override them if we want to use a custom one
 ssh.executable.hg = ~/.rccontrol/vcsserver-1/profile/bin/hg
 ssh.executable.git = ~/.rccontrol/vcsserver-1/profile/bin/git
 ssh.executable.svn = ~/.rccontrol/vcsserver-1/profile/bin/svnserve
 
 ; Enables SSH key generator web interface. Disabling this still allows users
 ; to add their own keys.
 ssh.enable_ui_key_generator = true
 
 
 ; #################
 ; APPENLIGHT CONFIG
 ; #################
 
 ; Appenlight is tailored to work with RhodeCode, see
 ; http://appenlight.rhodecode.com for details how to obtain an account
 
 ; Appenlight integration enabled
 #appenlight = false
 
 #appenlight.server_url = https://api.appenlight.com
 #appenlight.api_key = YOUR_API_KEY
 #appenlight.transport_config = https://api.appenlight.com?threaded=1&timeout=5
 
 ; used for JS client
 #appenlight.api_public_key = YOUR_API_PUBLIC_KEY
 
 ; TWEAK AMOUNT OF INFO SENT HERE
 
 ; enables 404 error logging (default False)
 #appenlight.report_404 = false
 
 ; time in seconds after request is considered being slow (default 1)
 #appenlight.slow_request_time = 1
 
 ; record slow requests in application
 ; (needs to be enabled for slow datastore recording and time tracking)
 #appenlight.slow_requests = true
 
 ; enable hooking to application loggers
 #appenlight.logging = true
 
 ; minimum log level for log capture
 #ppenlight.logging.level = WARNING
 
 ; send logs only from erroneous/slow requests
 ; (saves API quota for intensive logging)
 #appenlight.logging_on_error = false
 
 ; list of additional keywords that should be grabbed from environ object
 ; can be string with comma separated list of words in lowercase
 ; (by default client will always send following info:
 ; 'REMOTE_USER', 'REMOTE_ADDR', 'SERVER_NAME', 'CONTENT_TYPE' + all keys that
 ; start with HTTP* this list be extended with additional keywords here
 #appenlight.environ_keys_whitelist =
 
 ; list of keywords that should be blanked from request object
 ; can be string with comma separated list of words in lowercase
 ; (by default client will always blank keys that contain following words
 ; 'password', 'passwd', 'pwd', 'auth_tkt', 'secret', 'csrf'
 ; this list be extended with additional keywords set here
 #appenlight.request_keys_blacklist =
 
 ; list of namespaces that should be ignores when gathering log entries
 ; can be string with comma separated list of namespaces
 ; (by default the client ignores own entries: appenlight_client.client)
 #appenlight.log_namespace_blacklist =
 
 ; Statsd client config, this is used to send metrics to statsd
 ; We recommend setting statsd_exported and scrape them using Prometheus
 #statsd.enabled = false
 #statsd.statsd_host = 0.0.0.0
 #statsd.statsd_port = 8125
 #statsd.statsd_prefix =
 #statsd.statsd_ipv6 = false
 
 ; configure logging automatically at server startup set to false
 ; to use the below custom logging config.
 ; RC_LOGGING_FORMATTER
 ; RC_LOGGING_LEVEL
 ; env variables can control the settings for logging in case of autoconfigure
 
 #logging.autoconfigure = true
 
 ; specify your own custom logging config file to configure logging
 #logging.logging_conf_file = /path/to/custom_logging.ini
 
 ; Dummy marker to add new entries after.
 ; Add any custom entries below. Please don't remove this marker.
 custom.conf = 1
 
 
 ; #####################
 ; LOGGING CONFIGURATION
 ; #####################
 
 [loggers]
 keys = root, sqlalchemy, beaker, celery, rhodecode, ssh_wrapper
 
 [handlers]
 keys = console, console_sql
 
 [formatters]
 keys = generic, json, color_formatter, color_formatter_sql
 
 ; #######
 ; LOGGERS
 ; #######
 [logger_root]
 level = NOTSET
 handlers = console
 
 [logger_sqlalchemy]
 level = INFO
 handlers = console_sql
 qualname = sqlalchemy.engine
 propagate = 0
 
 [logger_beaker]
 level = DEBUG
 handlers =
 qualname = beaker.container
 propagate = 1
 
 [logger_rhodecode]
 level = DEBUG
 handlers =
 qualname = rhodecode
 propagate = 1
 
 [logger_ssh_wrapper]
 level = DEBUG
 handlers =
 qualname = ssh_wrapper
 propagate = 1
 
 [logger_celery]
 level = DEBUG
 handlers =
 qualname = celery
 
 
 ; ########
 ; HANDLERS
 ; ########
 
 [handler_console]
 class = StreamHandler
 args = (sys.stderr, )
 level = INFO
 ; To enable JSON formatted logs replace 'generic/color_formatter' with 'json'
 ; This allows sending properly formatted logs to grafana loki or elasticsearch
 formatter = generic
 
 [handler_console_sql]
 ; "level = DEBUG" logs SQL queries and results.
 ; "level = INFO" logs SQL queries.
 ; "level = WARN" logs neither.  (Recommended for production systems.)
 class = StreamHandler
 args = (sys.stderr, )
 level = WARN
 ; To enable JSON formatted logs replace 'generic/color_formatter_sql' with 'json'
 ; This allows sending properly formatted logs to grafana loki or elasticsearch
 formatter = generic
 
 ; ##########
 ; FORMATTERS
 ; ##########
 
 [formatter_generic]
 class = rhodecode.lib.logging_formatter.ExceptionAwareFormatter
 format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s
 datefmt = %Y-%m-%d %H:%M:%S
 
 [formatter_color_formatter]
 class = rhodecode.lib.logging_formatter.ColorFormatter
 format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s
 datefmt = %Y-%m-%d %H:%M:%S
 
 [formatter_color_formatter_sql]
 class = rhodecode.lib.logging_formatter.ColorFormatterSql
 format = %(asctime)s.%(msecs)03d [%(process)d] %(levelname)-5.5s [%(name)s] %(message)s
 datefmt = %Y-%m-%d %H:%M:%S
 
 [formatter_json]
 format = %(timestamp)s %(levelname)s %(name)s %(message)s %(req_id)s
 class = rhodecode.lib._vendor.jsonlogger.JsonFormatter