##// END OF EJS Templates
switch to ruff from black
super-admin -
r1170:e15a1319 default
parent child Browse files
Show More
@@ -3,21 +3,24 b' Gunicorn config extension and hooks. Thi'
3 Gunicorn configuration should be managed by .ini files entries of RhodeCode or VCSServer
3 Gunicorn configuration should be managed by .ini files entries of RhodeCode or VCSServer
4 """
4 """
5
5
6 import contextlib
7 import dataclasses
6 import gc
8 import gc
7 import os
8 import sys
9 import math
9 import math
10 import time
10 import os
11 import threading
12 import traceback
13 import random
11 import random
14 import socket
12 import socket
15 import dataclasses
13 import sys
14 import threading
15 import time
16 import traceback
17
16 from gunicorn.glogging import Logger
18 from gunicorn.glogging import Logger
17
19
18
20
19 def get_workers():
21 def get_workers():
20 import multiprocessing
22 import multiprocessing
23
21 return multiprocessing.cpu_count() * 2 + 1
24 return multiprocessing.cpu_count() * 2 + 1
22
25
23
26
@@ -25,10 +28,10 b' bind = "127.0.0.1:10010"'
25
28
26
29
27 # Error logging output for gunicorn (-) is stdout
30 # Error logging output for gunicorn (-) is stdout
28 errorlog = '-'
31 errorlog = "-"
29
32
30 # Access logging output for gunicorn (-) is stdout
33 # Access logging output for gunicorn (-) is stdout
31 accesslog = '-'
34 accesslog = "-"
32
35
33
36
34 # SERVER MECHANICS
37 # SERVER MECHANICS
@@ -45,8 +48,7 b' tmp_upload_dir = None'
45 # '%(t)s %(p)s INFO [GNCRN] %(h)-15s rqt:%(L)s %(s)s %(b)-6s "%(m)s:%(U)s %(q)s" usr:%(u)s "%(f)s" "%(a)s"')
48 # '%(t)s %(p)s INFO [GNCRN] %(h)-15s rqt:%(L)s %(s)s %(b)-6s "%(m)s:%(U)s %(q)s" usr:%(u)s "%(f)s" "%(a)s"')
46
49
47 # loki format for easier parsing in grafana
50 # loki format for easier parsing in grafana
48 access_log_format = (
51 access_log_format = 'time="%(t)s" pid=%(p)s level="INFO" type="[GNCRN]" ip="%(h)-15s" rqt="%(L)s" response_code="%(s)s" response_bytes="%(b)-6s" uri="%(m)s:%(U)s %(q)s" user=":%(u)s" user_agent="%(a)s"'
49 'time="%(t)s" pid=%(p)s level="INFO" type="[GNCRN]" ip="%(h)-15s" rqt="%(L)s" response_code="%(s)s" response_bytes="%(b)-6s" uri="%(m)s:%(U)s %(q)s" user=":%(u)s" user_agent="%(a)s"')
50
52
51
53
52 # Sets the number of process workers. More workers means more concurrent connections
54 # Sets the number of process workers. More workers means more concurrent connections
@@ -62,14 +64,14 b' workers = 6'
62 # workers = get_workers()
64 # workers = get_workers()
63
65
64 # Gunicorn access log level
66 # Gunicorn access log level
65 loglevel = 'info'
67 loglevel = "info"
66
68
67 # Process name visible in a process list
69 # Process name visible in a process list
68 proc_name = 'rhodecode_vcsserver'
70 proc_name = "rhodecode_vcsserver"
69
71
70 # Type of worker class, one of `sync`, `gevent`
72 # Type of worker class, one of `sync`, `gevent`
71 # currently `sync` is the only option allowed.
73 # currently `sync` is the only option allowed.
72 worker_class = 'sync'
74 worker_class = "sync"
73
75
74 # The maximum number of simultaneous clients. Valid only for gevent
76 # The maximum number of simultaneous clients. Valid only for gevent
75 worker_connections = 10
77 worker_connections = 10
@@ -136,25 +138,26 b' class MemoryCheckConfig:'
136
138
137
139
138 def _get_process_rss(pid=None):
140 def _get_process_rss(pid=None):
139 try:
141 with contextlib.suppress(Exception):
140 import psutil
142 import psutil
143
141 if pid:
144 if pid:
142 proc = psutil.Process(pid)
145 proc = psutil.Process(pid)
143 else:
146 else:
144 proc = psutil.Process()
147 proc = psutil.Process()
145 return proc.memory_info().rss
148 return proc.memory_info().rss
146 except Exception:
149
147 return None
150 return None
148
151
149
152
150 def _get_config(ini_path):
153 def _get_config(ini_path):
151 import configparser
154 import configparser
152
155
153 try:
156 with contextlib.suppress(Exception):
154 config = configparser.RawConfigParser()
157 config = configparser.RawConfigParser()
155 config.read(ini_path)
158 config.read(ini_path)
156 return config
159 return config
157 except Exception:
160
158 return None
161 return None
159
162
160
163
@@ -168,24 +171,24 b' def get_memory_usage_params(config=None)'
168 ini_path = os.path.abspath(config)
171 ini_path = os.path.abspath(config)
169 conf = _get_config(ini_path)
172 conf = _get_config(ini_path)
170
173
171 section = 'server:main'
174 section = "server:main"
172 if conf and conf.has_section(section):
175 if conf and conf.has_section(section):
176 if conf.has_option(section, "memory_max_usage"):
177 _memory_max_usage = conf.getint(section, "memory_max_usage")
173
178
174 if conf.has_option(section, 'memory_max_usage'):
179 if conf.has_option(section, "memory_usage_check_interval"):
175 _memory_max_usage = conf.getint(section, 'memory_max_usage')
180 _memory_usage_check_interval = conf.getint(section, "memory_usage_check_interval")
176
177 if conf.has_option(section, 'memory_usage_check_interval'):
178 _memory_usage_check_interval = conf.getint(section, 'memory_usage_check_interval')
179
181
180 if conf.has_option(section, 'memory_usage_recovery_threshold'):
182 if conf.has_option(section, "memory_usage_recovery_threshold"):
181 _memory_usage_recovery_threshold = conf.getfloat(section, 'memory_usage_recovery_threshold')
183 _memory_usage_recovery_threshold = conf.getfloat(section, "memory_usage_recovery_threshold")
182
184
183 _memory_max_usage = int(os.environ.get('RC_GUNICORN_MEMORY_MAX_USAGE', '')
185 _memory_max_usage = int(os.environ.get("RC_GUNICORN_MEMORY_MAX_USAGE", "") or _memory_max_usage)
184 or _memory_max_usage)
186 _memory_usage_check_interval = int(
185 _memory_usage_check_interval = int(os.environ.get('RC_GUNICORN_MEMORY_USAGE_CHECK_INTERVAL', '')
187 os.environ.get("RC_GUNICORN_MEMORY_USAGE_CHECK_INTERVAL", "") or _memory_usage_check_interval
186 or _memory_usage_check_interval)
188 )
187 _memory_usage_recovery_threshold = float(os.environ.get('RC_GUNICORN_MEMORY_USAGE_RECOVERY_THRESHOLD', '')
189 _memory_usage_recovery_threshold = float(
188 or _memory_usage_recovery_threshold)
190 os.environ.get("RC_GUNICORN_MEMORY_USAGE_RECOVERY_THRESHOLD", "") or _memory_usage_recovery_threshold
191 )
189
192
190 return MemoryCheckConfig(_memory_max_usage, _memory_usage_check_interval, _memory_usage_recovery_threshold)
193 return MemoryCheckConfig(_memory_max_usage, _memory_usage_check_interval, _memory_usage_recovery_threshold)
191
194
@@ -199,26 +202,27 b' def pre_fork(server, worker):'
199
202
200
203
201 def post_fork(server, worker):
204 def post_fork(server, worker):
202
203 memory_conf = get_memory_usage_params()
205 memory_conf = get_memory_usage_params()
204 _memory_max_usage = memory_conf.max_usage
206 _memory_max_usage = memory_conf.max_usage
205 _memory_usage_check_interval = memory_conf.check_interval
207 _memory_usage_check_interval = memory_conf.check_interval
206 _memory_usage_recovery_threshold = memory_conf.recovery_threshold
208 _memory_usage_recovery_threshold = memory_conf.recovery_threshold
207
209
208 worker._memory_max_usage = int(os.environ.get('RC_GUNICORN_MEMORY_MAX_USAGE', '')
210 worker._memory_max_usage = int(os.environ.get("RC_GUNICORN_MEMORY_MAX_USAGE", "") or _memory_max_usage)
209 or _memory_max_usage)
211 worker._memory_usage_check_interval = int(
210 worker._memory_usage_check_interval = int(os.environ.get('RC_GUNICORN_MEMORY_USAGE_CHECK_INTERVAL', '')
212 os.environ.get("RC_GUNICORN_MEMORY_USAGE_CHECK_INTERVAL", "") or _memory_usage_check_interval
211 or _memory_usage_check_interval)
213 )
212 worker._memory_usage_recovery_threshold = float(os.environ.get('RC_GUNICORN_MEMORY_USAGE_RECOVERY_THRESHOLD', '')
214 worker._memory_usage_recovery_threshold = float(
213 or _memory_usage_recovery_threshold)
215 os.environ.get("RC_GUNICORN_MEMORY_USAGE_RECOVERY_THRESHOLD", "") or _memory_usage_recovery_threshold
216 )
214
217
215 # register memory last check time, with some random offset so we don't recycle all
218 # register memory last check time, with some random offset so we don't recycle all
216 # at once
219 # at once
217 worker._last_memory_check_time = _time_with_offset(_memory_usage_check_interval)
220 worker._last_memory_check_time = _time_with_offset(_memory_usage_check_interval)
218
221
219 if _memory_max_usage:
222 if _memory_max_usage:
220 server.log.info("pid=[%-10s] WORKER spawned with max memory set at %s", worker.pid,
223 server.log.info(
221 _format_data_size(_memory_max_usage))
224 "pid=[%-10s] WORKER spawned with max memory set at %s", worker.pid, _format_data_size(_memory_max_usage)
225 )
222 else:
226 else:
223 server.log.info("pid=[%-10s] WORKER spawned", worker.pid)
227 server.log.info("pid=[%-10s] WORKER spawned", worker.pid)
224
228
@@ -228,9 +232,9 b' def pre_exec(server):'
228
232
229
233
230 def on_starting(server):
234 def on_starting(server):
231 server_lbl = '{} {}'.format(server.proc_name, server.address)
235 server_lbl = "{} {}".format(server.proc_name, server.address)
232 server.log.info("Server %s is starting.", server_lbl)
236 server.log.info("Server %s is starting.", server_lbl)
233 server.log.info('Config:')
237 server.log.info("Config:")
234 server.log.info(f"\n{server.cfg}")
238 server.log.info(f"\n{server.cfg}")
235 server.log.info(get_memory_usage_params())
239 server.log.info(get_memory_usage_params())
236
240
@@ -264,10 +268,10 b' def _format_data_size(size, unit="B", pr'
264
268
265 if not binary:
269 if not binary:
266 base = 1000
270 base = 1000
267 multiples = ('', 'k', 'M', 'G', 'T', 'P', 'E', 'Z', 'Y')
271 multiples = ("", "k", "M", "G", "T", "P", "E", "Z", "Y")
268 else:
272 else:
269 base = 1024
273 base = 1024
270 multiples = ('', 'Ki', 'Mi', 'Gi', 'Ti', 'Pi', 'Ei', 'Zi', 'Yi')
274 multiples = ("", "Ki", "Mi", "Gi", "Ti", "Pi", "Ei", "Zi", "Yi")
271
275
272 sign = ""
276 sign = ""
273 if size > 0:
277 if size > 0:
@@ -282,13 +286,13 b' def _format_data_size(size, unit="B", pr'
282 m = 8
286 m = 8
283
287
284 if m == 0:
288 if m == 0:
285 precision = '%.0f'
289 precision = "%.0f"
286 else:
290 else:
287 precision = '%%.%df' % precision
291 precision = "%%.%df" % precision
288
292
289 size = precision % (size / math.pow(base, m))
293 size = precision % (size / math.pow(base, m))
290
294
291 return '%s%s %s%s' % (sign, size.strip(), multiples[m], unit)
295 return "%s%s %s%s" % (sign, size.strip(), multiples[m], unit)
292
296
293
297
294 def _check_memory_usage(worker):
298 def _check_memory_usage(worker):
@@ -304,8 +308,8 b' def _check_memory_usage(worker):'
304 mem_usage = _get_process_rss()
308 mem_usage = _get_process_rss()
305 if mem_usage and mem_usage > _memory_max_usage:
309 if mem_usage and mem_usage > _memory_max_usage:
306 worker.log.info(
310 worker.log.info(
307 "memory usage %s > %s, forcing gc",
311 "memory usage %s > %s, forcing gc", _format_data_size(mem_usage), _format_data_size(_memory_max_usage)
308 _format_data_size(mem_usage), _format_data_size(_memory_max_usage))
312 )
309 # Try to clean it up by forcing a full collection.
313 # Try to clean it up by forcing a full collection.
310 gc.collect()
314 gc.collect()
311 mem_usage = _get_process_rss()
315 mem_usage = _get_process_rss()
@@ -313,7 +317,9 b' def _check_memory_usage(worker):'
313 # Didn't clean up enough, we'll have to terminate.
317 # Didn't clean up enough, we'll have to terminate.
314 worker.log.warning(
318 worker.log.warning(
315 "memory usage %s > %s after gc, quitting",
319 "memory usage %s > %s after gc, quitting",
316 _format_data_size(mem_usage), _format_data_size(_memory_max_usage))
320 _format_data_size(mem_usage),
321 _format_data_size(_memory_max_usage),
322 )
317 # This will cause worker to auto-restart itself
323 # This will cause worker to auto-restart itself
318 worker.alive = False
324 worker.alive = False
319 worker._last_memory_check_time = time.time()
325 worker._last_memory_check_time = time.time()
@@ -329,8 +335,7 b' def worker_int(worker):'
329
335
330 code = []
336 code = []
331 for thread_id, stack in sys._current_frames().items(): # noqa
337 for thread_id, stack in sys._current_frames().items(): # noqa
332 code.append(
338 code.append("\n# Thread: %s(%d)" % (get_thread_id(thread_id), thread_id))
333 "\n# Thread: %s(%d)" % (get_thread_id(thread_id), thread_id))
334 for fname, lineno, name, line in traceback.extract_stack(stack):
339 for fname, lineno, name, line in traceback.extract_stack(stack):
335 code.append('File: "%s", line %d, in %s' % (fname, lineno, name))
340 code.append('File: "%s", line %d, in %s' % (fname, lineno, name))
336 if line:
341 if line:
@@ -352,17 +357,21 b' def child_exit(server, worker):'
352
357
353 def pre_request(worker, req):
358 def pre_request(worker, req):
354 worker.start_time = time.time()
359 worker.start_time = time.time()
355 worker.log.debug(
360 worker.log.debug("GNCRN PRE WORKER [cnt:%s]: %s %s", worker.nr, req.method, req.path)
356 "GNCRN PRE WORKER [cnt:%s]: %s %s", worker.nr, req.method, req.path)
357
361
358
362
359 def post_request(worker, req, environ, resp):
363 def post_request(worker, req, environ, resp):
360 total_time = time.time() - worker.start_time
364 total_time = time.time() - worker.start_time
361 # Gunicorn sometimes has problems with reading the status_code
365 # Gunicorn sometimes has problems with reading the status_code
362 status_code = getattr(resp, 'status_code', '')
366 status_code = getattr(resp, "status_code", "")
363 worker.log.debug(
367 worker.log.debug(
364 "GNCRN POST WORKER [cnt:%s]: %s %s resp: %s, Load Time: %.4fs",
368 "GNCRN POST WORKER [cnt:%s]: %s %s resp: %s, Load Time: %.4fs",
365 worker.nr, req.method, req.path, status_code, total_time)
369 worker.nr,
370 req.method,
371 req.path,
372 status_code,
373 total_time,
374 )
366 _check_memory_usage(worker)
375 _check_memory_usage(worker)
367
376
368
377
@@ -375,8 +384,8 b' def _filter_proxy(ip):'
375
384
376 :param ip: ip string from headers
385 :param ip: ip string from headers
377 """
386 """
378 if ',' in ip:
387 if "," in ip:
379 _ips = ip.split(',')
388 _ips = ip.split(",")
380 _first_ip = _ips[0].strip()
389 _first_ip = _ips[0].strip()
381 return _first_ip
390 return _first_ip
382 return ip
391 return ip
@@ -392,8 +401,9 b' def _filter_port(ip):'
392
401
393 :param ip:
402 :param ip:
394 """
403 """
404
395 def is_ipv6(ip_addr):
405 def is_ipv6(ip_addr):
396 if hasattr(socket, 'inet_pton'):
406 if hasattr(socket, "inet_pton"):
397 try:
407 try:
398 socket.inet_pton(socket.AF_INET6, ip_addr)
408 socket.inet_pton(socket.AF_INET6, ip_addr)
399 except socket.error:
409 except socket.error:
@@ -402,24 +412,24 b' def _filter_port(ip):'
402 return False
412 return False
403 return True
413 return True
404
414
405 if ':' not in ip: # must be ipv4 pure ip
415 if ":" not in ip: # must be ipv4 pure ip
406 return ip
416 return ip
407
417
408 if '[' in ip and ']' in ip: # ipv6 with port
418 if "[" in ip and "]" in ip: # ipv6 with port
409 return ip.split(']')[0][1:].lower()
419 return ip.split("]")[0][1:].lower()
410
420
411 # must be ipv6 or ipv4 with port
421 # must be ipv6 or ipv4 with port
412 if is_ipv6(ip):
422 if is_ipv6(ip):
413 return ip
423 return ip
414 else:
424 else:
415 ip, _port = ip.split(':')[:2] # means ipv4+port
425 ip, _port = ip.split(":")[:2] # means ipv4+port
416 return ip
426 return ip
417
427
418
428
419 def get_ip_addr(environ):
429 def get_ip_addr(environ):
420 proxy_key = 'HTTP_X_REAL_IP'
430 proxy_key = "HTTP_X_REAL_IP"
421 proxy_key2 = 'HTTP_X_FORWARDED_FOR'
431 proxy_key2 = "HTTP_X_FORWARDED_FOR"
422 def_key = 'REMOTE_ADDR'
432 def_key = "REMOTE_ADDR"
423
433
424 def _filters(x):
434 def _filters(x):
425 return _filter_port(_filter_proxy(x))
435 return _filter_port(_filter_proxy(x))
@@ -432,7 +442,7 b' def get_ip_addr(environ):'
432 if ip:
442 if ip:
433 return _filters(ip)
443 return _filters(ip)
434
444
435 ip = environ.get(def_key, '0.0.0.0')
445 ip = environ.get(def_key, "0.0.0.0")
436 return _filters(ip)
446 return _filters(ip)
437
447
438
448
@@ -450,40 +460,37 b' class RhodeCodeLogger(Logger):'
450 """ return date in RhodeCode Log format """
460 """return date in RhodeCode Log format"""
451 now = time.time()
461 now = time.time()
452 msecs = int((now - int(now)) * 1000)
462 msecs = int((now - int(now)) * 1000)
453 return time.strftime(self.datefmt, time.localtime(now)) + '.{0:03d}'.format(msecs)
463 return time.strftime(self.datefmt, time.localtime(now)) + ".{0:03d}".format(msecs)
454
464
455 def atoms(self, resp, req, environ, request_time):
465 def atoms(self, resp, req, environ, request_time):
456 """ Gets atoms for log formatting.
466 """Gets atoms for log formatting."""
457 """
458 status = resp.status
467 status = resp.status
459 if isinstance(status, str):
468 if isinstance(status, str):
460 status = status.split(None, 1)[0]
469 status = status.split(None, 1)[0]
461 atoms = {
470 atoms = {
462 'h': get_ip_addr(environ),
471 "h": get_ip_addr(environ),
463 'l': '-',
472 "l": "-",
464 'u': self._get_user(environ) or '-',
473 "u": self._get_user(environ) or "-",
465 't': self.now(),
474 "t": self.now(),
466 'r': "%s %s %s" % (environ['REQUEST_METHOD'],
475 "r": "%s %s %s" % (environ["REQUEST_METHOD"], environ["RAW_URI"], environ["SERVER_PROTOCOL"]),
467 environ['RAW_URI'],
476 "s": status,
468 environ["SERVER_PROTOCOL"]),
477 "m": environ.get("REQUEST_METHOD"),
469 's': status,
478 "U": environ.get("PATH_INFO"),
470 'm': environ.get('REQUEST_METHOD'),
479 "q": environ.get("QUERY_STRING"),
471 'U': environ.get('PATH_INFO'),
480 "H": environ.get("SERVER_PROTOCOL"),
472 'q': environ.get('QUERY_STRING'),
481 "b": getattr(resp, "sent", None) is not None and str(resp.sent) or "-",
473 'H': environ.get('SERVER_PROTOCOL'),
482 "B": getattr(resp, "sent", None),
474 'b': getattr(resp, 'sent', None) is not None and str(resp.sent) or '-',
483 "f": environ.get("HTTP_REFERER", "-"),
475 'B': getattr(resp, 'sent', None),
484 "a": environ.get("HTTP_USER_AGENT", "-"),
476 'f': environ.get('HTTP_REFERER', '-'),
485 "T": request_time.seconds,
477 'a': environ.get('HTTP_USER_AGENT', '-'),
486 "D": (request_time.seconds * 1000000) + request_time.microseconds,
478 'T': request_time.seconds,
487 "M": (request_time.seconds * 1000) + int(request_time.microseconds / 1000),
479 'D': (request_time.seconds * 1000000) + request_time.microseconds,
488 "L": "%d.%06d" % (request_time.seconds, request_time.microseconds),
480 'M': (request_time.seconds * 1000) + int(request_time.microseconds/1000),
489 "p": "<%s>" % os.getpid(),
481 'L': "%d.%06d" % (request_time.seconds, request_time.microseconds),
482 'p': "<%s>" % os.getpid()
483 }
490 }
484
491
485 # add request headers
492 # add request headers
486 if hasattr(req, 'headers'):
493 if hasattr(req, "headers"):
487 req_headers = req.headers
494 req_headers = req.headers
488 else:
495 else:
489 req_headers = req
496 req_headers = req
@@ -35,6 +35,7 b' dependencies = {file = ["requirements.tx'
35 optional-dependencies.tests = {file = ["requirements_test.txt"]}
35 optional-dependencies.tests = {file = ["requirements_test.txt"]}
36
36
37 [tool.ruff]
37 [tool.ruff]
38
38 select = [
39 select = [
39 # Pyflakes
40 # Pyflakes
40 "F",
41 "F",
@@ -44,16 +45,29 b' select = ['
44 # isort
45 # isort
45 "I001"
46 "I001"
46 ]
47 ]
48
47 ignore = [
49 ignore = [
48 "E501", # line too long, handled by black
50 "E501", # line too long, handled by black
49 ]
51 ]
52
50 # Same as Black.
53 # Same as Black.
51 line-length = 120
54 line-length = 120
52
55
56 [tool.ruff.isort]
53
57
54 [tool.ruff.isort]
55 known-first-party = ["vcsserver"]
58 known-first-party = ["vcsserver"]
56
59
57 [tool.black]
60 [tool.ruff.format]
58 line-length = 120
61
59 target-version = ['py310', 'py311']
62 # Like Black, use double quotes for strings.
63 quote-style = "double"
64
65 # Like Black, indent with spaces, rather than tabs.
66 indent-style = "space"
67
68 # Like Black, respect magic trailing commas.
69 skip-magic-trailing-comma = false
70
71 # Like Black, automatically detect the appropriate line ending.
72 line-ending = "auto"
73
General Comments 0
You need to be logged in to leave comments. Login now