##// END OF EJS Templates
gunicorn: updated gunicorn config to add memory monitoring
marcink -
r761:c5e7292a default
parent child Browse files
Show More
@@ -13,19 +13,24 b' available post the .ini config.'
13
13
14 """
14 """
15
15
16 import multiprocessing
16 import math
17 import gc
17 import sys
18 import sys
18 import time
19 import time
19 import datetime
20 import threading
20 import threading
21 import traceback
21 import traceback
22 import random
22 from gunicorn.glogging import Logger
23 from gunicorn.glogging import Logger
23
24
24
25
26 def get_workers():
27 import multiprocessing
28 return multiprocessing.cpu_count() * 2 + 1
29
25 # GLOBAL
30 # GLOBAL
26 errorlog = '-'
31 errorlog = '-'
27 accesslog = '-'
32 accesslog = '-'
28 loglevel = 'debug'
33 loglevel = 'info'
29
34
30 # SECURITY
35 # SECURITY
31
36
@@ -35,26 +40,23 b' limit_request_line = 0'
35
40
36 # Limit the number of HTTP headers fields in a request.
41 # Limit the number of HTTP headers fields in a request.
37 # By default this value is 100 and can't be larger than 32768.
42 # By default this value is 100 and can't be larger than 32768.
38 limit_request_fields = 10240
43 limit_request_fields = 32768
39
44
40 # Limit the allowed size of an HTTP request header field.
45 # Limit the allowed size of an HTTP request header field.
41 # Value is a positive number or 0.
46 # Value is a positive number or 0.
42 # Setting it to 0 will allow unlimited header field sizes.
47 # Setting it to 0 will allow unlimited header field sizes.
43 limit_request_field_size = 0
48 limit_request_field_size = 0
44
49
45
46 # Timeout for graceful workers restart.
50 # Timeout for graceful workers restart.
47 # After receiving a restart signal, workers have this much time to finish
51 # After receiving a restart signal, workers have this much time to finish
48 # serving requests. Workers still alive after the timeout (starting from the
52 # serving requests. Workers still alive after the timeout (starting from the
49 # receipt of the restart signal) are force killed.
53 # receipt of the restart signal) are force killed.
50 graceful_timeout = 30
54 graceful_timeout = 60 * 60
51
52
55
53 # The number of seconds to wait for requests on a Keep-Alive connection.
56 # The number of seconds to wait for requests on a Keep-Alive connection.
54 # Generally set in the 1-5 seconds range.
57 # Generally set in the 1-5 seconds range.
55 keepalive = 2
58 keepalive = 2
56
59
57
58 # SERVER MECHANICS
60 # SERVER MECHANICS
59 # None == system temp dir
61 # None == system temp dir
60 # worker_tmp_dir is recommended to be set to some tmpfs
62 # worker_tmp_dir is recommended to be set to some tmpfs
@@ -63,36 +65,138 b' tmp_upload_dir = None'
63
65
64 # Custom log format
66 # Custom log format
65 access_log_format = (
67 access_log_format = (
66 '%(t)s [%(p)-8s] GNCRN %(h)-15s rqt:%(L)s %(s)s %(b)-6s "%(m)s:%(U)s %(q)s" usr:%(u)s "%(f)s" "%(a)s"')
68 '%(t)s %(p)s INFO [GNCRN] %(h)-15s rqt:%(L)s %(s)s %(b)-6s "%(m)s:%(U)s %(q)s" usr:%(u)s "%(f)s" "%(a)s"')
67
69
68 # self adjust workers based on CPU count
70 # self adjust workers based on CPU count
69 # workers = multiprocessing.cpu_count() * 2 + 1
71 # workers = get_workers()
72
73 # n * 1024 * 0124 == n MBs, 0 = memory monitoring is disabled
74 MAX_MEMORY_USAGE = 0 * 1024 * 1024
75
76 # How often in seconds to check for memory usage
77 MEMORY_USAGE_CHECK_INTERVAL = 30
78
79 # If a gc brings us back below this threshold, we can avoid termination.
80 MEMORY_USAGE_RECOVERY_THRESHOLD = MAX_MEMORY_USAGE * 0.8
70
81
71
82
72 def post_fork(server, worker):
83 def _get_process_rss(pid=None):
73 server.log.info("[<%-10s>] WORKER spawned", worker.pid)
84 try:
85 import psutil
86 if pid:
87 proc = psutil.Process(pid)
88 else:
89 proc = psutil.Process()
90 return proc.memory_info().rss
91 except Exception:
92 return None
93
94
95 def _time_with_offset():
96 return time.time() - random.randint(0, MEMORY_USAGE_CHECK_INTERVAL/2.0)
74
97
75
98
76 def pre_fork(server, worker):
99 def pre_fork(server, worker):
77 pass
100 pass
78
101
79
102
103 def post_fork(server, worker):
104 server.log.info("<%s> WORKER spawned", worker.pid)
105 # register memory last check time, with some random offset so we don't recycle all
106 # at once
107 worker._last_memory_check_time = _time_with_offset()
108
109
80 def pre_exec(server):
110 def pre_exec(server):
81 server.log.info("Forked child, re-executing.")
111 server.log.info("Forked child, re-executing.")
82
112
83
113
84 def on_starting(server):
114 def on_starting(server):
85 server.log.info("Server is starting.")
115 server_lbl = '{} {}'.format(server.proc_name, server.address)
116 server.log.info("Server %s is starting.", server_lbl)
86
117
87
118
88 def when_ready(server):
119 def when_ready(server):
89 server.log.info("Server is ready. Spawning workers")
120 server.log.info("Server %s is ready. Spawning workers", server)
90
121
91
122
92 def on_reload(server):
123 def on_reload(server):
93 pass
124 pass
94
125
95
126
127 def _format_data_size(size, unit="B", precision=1, binary=True):
128 """Format a number using SI units (kilo, mega, etc.).
129
130 ``size``: The number as a float or int.
131
132 ``unit``: The unit name in plural form. Examples: "bytes", "B".
133
134 ``precision``: How many digits to the right of the decimal point. Default
135 is 1. 0 suppresses the decimal point.
136
137 ``binary``: If false, use base-10 decimal prefixes (kilo = K = 1000).
138 If true, use base-2 binary prefixes (kibi = Ki = 1024).
139
140 ``full_name``: If false (default), use the prefix abbreviation ("k" or
141 "Ki"). If true, use the full prefix ("kilo" or "kibi"). If false,
142 use abbreviation ("k" or "Ki").
143
144 """
145
146 if not binary:
147 base = 1000
148 multiples = ('', 'k', 'M', 'G', 'T', 'P', 'E', 'Z', 'Y')
149 else:
150 base = 1024
151 multiples = ('', 'Ki', 'Mi', 'Gi', 'Ti', 'Pi', 'Ei', 'Zi', 'Yi')
152
153 sign = ""
154 if size > 0:
155 m = int(math.log(size, base))
156 elif size < 0:
157 sign = "-"
158 size = -size
159 m = int(math.log(size, base))
160 else:
161 m = 0
162 if m > 8:
163 m = 8
164
165 if m == 0:
166 precision = '%.0f'
167 else:
168 precision = '%%.%df' % precision
169
170 size = precision % (size / math.pow(base, m))
171
172 return '%s%s %s%s' % (sign, size.strip(), multiples[m], unit)
173
174
175 def _check_memory_usage(worker):
176
177 if not MAX_MEMORY_USAGE:
178 return
179
180 elapsed = time.time() - worker._last_memory_check_time
181 if elapsed > MEMORY_USAGE_CHECK_INTERVAL:
182 mem_usage = _get_process_rss()
183 if mem_usage and mem_usage > MAX_MEMORY_USAGE:
184 worker.log.info(
185 "memory usage %s > %s, forcing gc",
186 _format_data_size(mem_usage), _format_data_size(MAX_MEMORY_USAGE))
187 # Try to clean it up by forcing a full collection.
188 gc.collect()
189 mem_usage = _get_process_rss()
190 if mem_usage > MEMORY_USAGE_RECOVERY_THRESHOLD:
191 # Didn't clean up enough, we'll have to terminate.
192 worker.log.warning(
193 "memory usage %s > %s after gc, quitting",
194 _format_data_size(mem_usage), _format_data_size(MAX_MEMORY_USAGE))
195 # This will cause worker to auto-restart itself
196 worker.alive = False
197 worker._last_memory_check_time = time.time()
198
199
96 def worker_int(worker):
200 def worker_int(worker):
97 worker.log.info("[<%-10s>] worker received INT or QUIT signal", worker.pid)
201 worker.log.info("[<%-10s>] worker received INT or QUIT signal", worker.pid)
98
202
@@ -132,6 +236,7 b' def post_request(worker, req, environ, r'
132 worker.log.debug(
236 worker.log.debug(
133 "GNCRN POST WORKER [cnt:%s]: %s %s resp: %s, Load Time: %.4fs",
237 "GNCRN POST WORKER [cnt:%s]: %s %s resp: %s, Load Time: %.4fs",
134 worker.nr, req.method, req.path, resp.status_code, total_time)
238 worker.nr, req.method, req.path, resp.status_code, total_time)
239 _check_memory_usage(worker)
135
240
136
241
137 class RhodeCodeLogger(Logger):
242 class RhodeCodeLogger(Logger):
General Comments 0
You need to be logged in to leave comments. Login now