##// END OF EJS Templates
gunicorn: updated config with memory usage monitoring
marcink -
r3875:c115b183 default
parent child Browse files
Show More
@@ -13,19 +13,24 b' available post the .ini config.'
13 13
14 14 """
15 15
16 import multiprocessing
16 import math
17 import gc
17 18 import sys
18 19 import time
19 import datetime
20 20 import threading
21 21 import traceback
22 import random
22 23 from gunicorn.glogging import Logger
23 24
24 25
26 def get_workers():
27 import multiprocessing
28 return multiprocessing.cpu_count() * 2 + 1
29
25 30 # GLOBAL
26 31 errorlog = '-'
27 32 accesslog = '-'
28 loglevel = 'debug'
33 loglevel = 'info'
29 34
30 35 # SECURITY
31 36
@@ -35,26 +40,23 b' limit_request_line = 0'
35 40
36 41 # Limit the number of HTTP headers fields in a request.
37 42 # By default this value is 100 and can't be larger than 32768.
38 limit_request_fields = 10240
43 limit_request_fields = 32768
39 44
40 45 # Limit the allowed size of an HTTP request header field.
41 46 # Value is a positive number or 0.
42 47 # Setting it to 0 will allow unlimited header field sizes.
43 48 limit_request_field_size = 0
44 49
45
46 50 # Timeout for graceful workers restart.
47 51 # After receiving a restart signal, workers have this much time to finish
48 52 # serving requests. Workers still alive after the timeout (starting from the
49 53 # receipt of the restart signal) are force killed.
50 graceful_timeout = 30
51
54 graceful_timeout = 60 * 60
52 55
53 56 # The number of seconds to wait for requests on a Keep-Alive connection.
54 57 # Generally set in the 1-5 seconds range.
55 58 keepalive = 2
56 59
57
58 60 # SERVER MECHANICS
59 61 # None == system temp dir
60 62 # worker_tmp_dir is recommended to be set to some tmpfs
@@ -63,36 +65,138 b' tmp_upload_dir = None'
63 65
64 66 # Custom log format
65 67 access_log_format = (
66 '%(t)s [%(p)-8s] GNCRN %(h)-15s rqt:%(L)s %(s)s %(b)-6s "%(m)s:%(U)s %(q)s" usr:%(u)s "%(f)s" "%(a)s"')
68 '%(t)s %(p)s INFO [GNCRN] %(h)-15s rqt:%(L)s %(s)s %(b)-6s "%(m)s:%(U)s %(q)s" usr:%(u)s "%(f)s" "%(a)s"')
67 69
68 70 # self adjust workers based on CPU count
69 # workers = multiprocessing.cpu_count() * 2 + 1
71 # workers = get_workers()
72
73 # n * 1024 * 0124 == n MBs, 0 = memory monitoring is disabled
74 MAX_MEMORY_USAGE = 0 * 1024 * 1024
75
76 # How often in seconds to check for memory usage
77 MEMORY_USAGE_CHECK_INTERVAL = 30
78
79 # If a gc brings us back below this threshold, we can avoid termination.
80 MEMORY_USAGE_RECOVERY_THRESHOLD = MAX_MEMORY_USAGE * 0.8
70 81
71 82
72 def post_fork(server, worker):
73 server.log.info("[<%-10s>] WORKER spawned", worker.pid)
83 def _get_process_rss(pid=None):
84 try:
85 import psutil
86 if pid:
87 proc = psutil.Process(pid)
88 else:
89 proc = psutil.Process()
90 return proc.memory_info().rss
91 except Exception:
92 return None
93
94
95 def _time_with_offset():
96 return time.time() - random.randint(0, MEMORY_USAGE_CHECK_INTERVAL/2.0)
74 97
75 98
76 99 def pre_fork(server, worker):
77 100 pass
78 101
79 102
103 def post_fork(server, worker):
104 server.log.info("<%s> WORKER spawned", worker.pid)
105 # register memory last check time, with some random offset so we don't recycle all
106 # at once
107 worker._last_memory_check_time = _time_with_offset()
108
109
80 110 def pre_exec(server):
81 111 server.log.info("Forked child, re-executing.")
82 112
83 113
84 114 def on_starting(server):
85 server.log.info("Server is starting.")
115 server_lbl = '{} {}'.format(server.proc_name, server.address)
116 server.log.info("Server %s is starting.", server_lbl)
86 117
87 118
88 119 def when_ready(server):
89 server.log.info("Server is ready. Spawning workers")
120 server.log.info("Server %s is ready. Spawning workers", server)
90 121
91 122
92 123 def on_reload(server):
93 124 pass
94 125
95 126
127 def _format_data_size(size, unit="B", precision=1, binary=True):
128 """Format a number using SI units (kilo, mega, etc.).
129
130 ``size``: The number as a float or int.
131
132 ``unit``: The unit name in plural form. Examples: "bytes", "B".
133
134 ``precision``: How many digits to the right of the decimal point. Default
135 is 1. 0 suppresses the decimal point.
136
137 ``binary``: If false, use base-10 decimal prefixes (kilo = K = 1000).
138 If true, use base-2 binary prefixes (kibi = Ki = 1024).
139
140 ``full_name``: If false (default), use the prefix abbreviation ("k" or
141 "Ki"). If true, use the full prefix ("kilo" or "kibi"). If false,
142 use abbreviation ("k" or "Ki").
143
144 """
145
146 if not binary:
147 base = 1000
148 multiples = ('', 'k', 'M', 'G', 'T', 'P', 'E', 'Z', 'Y')
149 else:
150 base = 1024
151 multiples = ('', 'Ki', 'Mi', 'Gi', 'Ti', 'Pi', 'Ei', 'Zi', 'Yi')
152
153 sign = ""
154 if size > 0:
155 m = int(math.log(size, base))
156 elif size < 0:
157 sign = "-"
158 size = -size
159 m = int(math.log(size, base))
160 else:
161 m = 0
162 if m > 8:
163 m = 8
164
165 if m == 0:
166 precision = '%.0f'
167 else:
168 precision = '%%.%df' % precision
169
170 size = precision % (size / math.pow(base, m))
171
172 return '%s%s %s%s' % (sign, size.strip(), multiples[m], unit)
173
174
175 def _check_memory_usage(worker):
176
177 if not MAX_MEMORY_USAGE:
178 return
179
180 elapsed = time.time() - worker._last_memory_check_time
181 if elapsed > MEMORY_USAGE_CHECK_INTERVAL:
182 mem_usage = _get_process_rss()
183 if mem_usage and mem_usage > MAX_MEMORY_USAGE:
184 worker.log.info(
185 "memory usage %s > %s, forcing gc",
186 _format_data_size(mem_usage), _format_data_size(MAX_MEMORY_USAGE))
187 # Try to clean it up by forcing a full collection.
188 gc.collect()
189 mem_usage = _get_process_rss()
190 if mem_usage > MEMORY_USAGE_RECOVERY_THRESHOLD:
191 # Didn't clean up enough, we'll have to terminate.
192 worker.log.warning(
193 "memory usage %s > %s after gc, quitting",
194 _format_data_size(mem_usage), _format_data_size(MAX_MEMORY_USAGE))
195 # This will cause worker to auto-restart itself
196 worker.alive = False
197 worker._last_memory_check_time = time.time()
198
199
96 200 def worker_int(worker):
97 201 worker.log.info("[<%-10s>] worker received INT or QUIT signal", worker.pid)
98 202
@@ -132,6 +236,7 b' def post_request(worker, req, environ, r'
132 236 worker.log.debug(
133 237 "GNCRN POST WORKER [cnt:%s]: %s %s resp: %s, Load Time: %.4fs",
134 238 worker.nr, req.method, req.path, resp.status_code, total_time)
239 _check_memory_usage(worker)
135 240
136 241
137 242 class RhodeCodeLogger(Logger):
General Comments 0
You need to be logged in to leave comments. Login now