##// END OF EJS Templates
gunicorn: updated gunicorn config based on release changes
marcink -
r808:418e3613 default
parent child Browse files
Show More
@@ -1,273 +1,265 b''
1 """
1 """
2 gunicorn config extension and hooks. Sets additional configuration that is
2 Gunicorn config extension and hooks. This config file adds some extra settings and memory management.
3 available post the .ini config.
3 Gunicorn configuration should be managed by .ini files entries of RhodeCode or VCSServer
4
5 - workers = ${cpu_number}
6 - threads = 1
7 - proc_name = ${gunicorn_proc_name}
8 - worker_class = sync
9 - worker_connections = 10
10 - max_requests = 1000
11 - max_requests_jitter = 30
12 - timeout = 21600
13
14 """
4 """
15
5
16 import gc
6 import gc
17 import os
7 import os
18 import sys
8 import sys
19 import math
9 import math
20 import time
10 import time
21 import threading
11 import threading
22 import traceback
12 import traceback
23 import random
13 import random
24 from gunicorn.glogging import Logger
14 from gunicorn.glogging import Logger
25
15
26
16
27 def get_workers():
17 def get_workers():
28 import multiprocessing
18 import multiprocessing
29 return multiprocessing.cpu_count() * 2 + 1
19 return multiprocessing.cpu_count() * 2 + 1
30
20
31 # GLOBAL
21 # GLOBAL
32 errorlog = '-'
22 errorlog = '-'
33 accesslog = '-'
23 accesslog = '-'
34
24
35
25
36 # SERVER MECHANICS
26 # SERVER MECHANICS
37 # None == system temp dir
27 # None == system temp dir
38 # worker_tmp_dir is recommended to be set to some tmpfs
28 # worker_tmp_dir is recommended to be set to some tmpfs
39 worker_tmp_dir = None
29 worker_tmp_dir = None
40 tmp_upload_dir = None
30 tmp_upload_dir = None
41
31
42 # Custom log format
32 # Custom log format
43 access_log_format = (
33 access_log_format = (
44 '%(t)s %(p)s INFO [GNCRN] %(h)-15s rqt:%(L)s %(s)s %(b)-6s "%(m)s:%(U)s %(q)s" usr:%(u)s "%(f)s" "%(a)s"')
34 '%(t)s %(p)s INFO [GNCRN] %(h)-15s rqt:%(L)s %(s)s %(b)-6s "%(m)s:%(U)s %(q)s" usr:%(u)s "%(f)s" "%(a)s"')
45
35
46 # self adjust workers based on CPU count
36 # self adjust workers based on CPU count
47 # workers = get_workers()
37 # workers = get_workers()
48
38
49
39
50 def _get_process_rss(pid=None):
40 def _get_process_rss(pid=None):
51 try:
41 try:
52 import psutil
42 import psutil
53 if pid:
43 if pid:
54 proc = psutil.Process(pid)
44 proc = psutil.Process(pid)
55 else:
45 else:
56 proc = psutil.Process()
46 proc = psutil.Process()
57 return proc.memory_info().rss
47 return proc.memory_info().rss
58 except Exception:
48 except Exception:
59 return None
49 return None
60
50
61
51
62 def _get_config(ini_path):
52 def _get_config(ini_path):
63
53
64 try:
54 try:
65 import configparser
55 import configparser
66 except ImportError:
56 except ImportError:
67 import ConfigParser as configparser
57 import ConfigParser as configparser
68 try:
58 try:
69 config = configparser.RawConfigParser()
59 config = configparser.RawConfigParser()
70 config.read(ini_path)
60 config.read(ini_path)
71 return config
61 return config
72 except Exception:
62 except Exception:
73 return None
63 return None
74
64
75
65
76 def _time_with_offset(memory_usage_check_interval):
66 def _time_with_offset(memory_usage_check_interval):
77 return time.time() - random.randint(0, memory_usage_check_interval/2.0)
67 return time.time() - random.randint(0, memory_usage_check_interval/2.0)
78
68
79
69
80 def pre_fork(server, worker):
70 def pre_fork(server, worker):
81 pass
71 pass
82
72
83
73
84 def post_fork(server, worker):
74 def post_fork(server, worker):
85
75
86 # memory spec defaults
76 # memory spec defaults
87 _memory_max_usage = 0
77 _memory_max_usage = 0
88 _memory_usage_check_interval = 60
78 _memory_usage_check_interval = 60
89 _memory_usage_recovery_threshold = 0.8
79 _memory_usage_recovery_threshold = 0.8
90
80
91 ini_path = os.path.abspath(server.cfg.paste)
81 ini_path = os.path.abspath(server.cfg.paste)
92 conf = _get_config(ini_path)
82 conf = _get_config(ini_path)
93
83
94 section = 'server:main'
84 section = 'server:main'
95 if conf and conf.has_section(section):
85 if conf and conf.has_section(section):
96
86
97 if conf.has_option(section, 'memory_max_usage'):
87 if conf.has_option(section, 'memory_max_usage'):
98 _memory_max_usage = conf.getint(section, 'memory_max_usage')
88 _memory_max_usage = conf.getint(section, 'memory_max_usage')
99
89
100 if conf.has_option(section, 'memory_usage_check_interval'):
90 if conf.has_option(section, 'memory_usage_check_interval'):
101 _memory_usage_check_interval = conf.getint(section, 'memory_usage_check_interval')
91 _memory_usage_check_interval = conf.getint(section, 'memory_usage_check_interval')
102
92
103 if conf.has_option(section, 'memory_usage_recovery_threshold'):
93 if conf.has_option(section, 'memory_usage_recovery_threshold'):
104 _memory_usage_recovery_threshold = conf.getfloat(section, 'memory_usage_recovery_threshold')
94 _memory_usage_recovery_threshold = conf.getfloat(section, 'memory_usage_recovery_threshold')
105
95
106 worker._memory_max_usage = _memory_max_usage
96 worker._memory_max_usage = _memory_max_usage
107 worker._memory_usage_check_interval = _memory_usage_check_interval
97 worker._memory_usage_check_interval = _memory_usage_check_interval
108 worker._memory_usage_recovery_threshold = _memory_usage_recovery_threshold
98 worker._memory_usage_recovery_threshold = _memory_usage_recovery_threshold
109
99
110 # register memory last check time, with some random offset so we don't recycle all
100 # register memory last check time, with some random offset so we don't recycle all
111 # at once
101 # at once
112 worker._last_memory_check_time = _time_with_offset(_memory_usage_check_interval)
102 worker._last_memory_check_time = _time_with_offset(_memory_usage_check_interval)
113
103
114 if _memory_max_usage:
104 if _memory_max_usage:
115 server.log.info("[%-10s] WORKER spawned with max memory set at %s", worker.pid,
105 server.log.info("[%-10s] WORKER spawned with max memory set at %s", worker.pid,
116 _format_data_size(_memory_max_usage))
106 _format_data_size(_memory_max_usage))
117 else:
107 else:
118 server.log.info("[%-10s] WORKER spawned", worker.pid)
108 server.log.info("[%-10s] WORKER spawned", worker.pid)
119
109
120
110
121 def pre_exec(server):
111 def pre_exec(server):
122 server.log.info("Forked child, re-executing.")
112 server.log.info("Forked child, re-executing.")
123
113
124
114
125 def on_starting(server):
115 def on_starting(server):
126 server_lbl = '{} {}'.format(server.proc_name, server.address)
116 server_lbl = '{} {}'.format(server.proc_name, server.address)
127 server.log.info("Server %s is starting.", server_lbl)
117 server.log.info("Server %s is starting.", server_lbl)
128
118
129
119
130 def when_ready(server):
120 def when_ready(server):
131 server.log.info("Server %s is ready. Spawning workers", server)
121 server.log.info("Server %s is ready. Spawning workers", server)
132
122
133
123
134 def on_reload(server):
124 def on_reload(server):
135 pass
125 pass
136
126
137
127
138 def _format_data_size(size, unit="B", precision=1, binary=True):
128 def _format_data_size(size, unit="B", precision=1, binary=True):
139 """Format a number using SI units (kilo, mega, etc.).
129 """Format a number using SI units (kilo, mega, etc.).
140
130
141 ``size``: The number as a float or int.
131 ``size``: The number as a float or int.
142
132
143 ``unit``: The unit name in plural form. Examples: "bytes", "B".
133 ``unit``: The unit name in plural form. Examples: "bytes", "B".
144
134
145 ``precision``: How many digits to the right of the decimal point. Default
135 ``precision``: How many digits to the right of the decimal point. Default
146 is 1. 0 suppresses the decimal point.
136 is 1. 0 suppresses the decimal point.
147
137
148 ``binary``: If false, use base-10 decimal prefixes (kilo = K = 1000).
138 ``binary``: If false, use base-10 decimal prefixes (kilo = K = 1000).
149 If true, use base-2 binary prefixes (kibi = Ki = 1024).
139 If true, use base-2 binary prefixes (kibi = Ki = 1024).
150
140
151 ``full_name``: If false (default), use the prefix abbreviation ("k" or
141 ``full_name``: If false (default), use the prefix abbreviation ("k" or
152 "Ki"). If true, use the full prefix ("kilo" or "kibi"). If false,
142 "Ki"). If true, use the full prefix ("kilo" or "kibi"). If false,
153 use abbreviation ("k" or "Ki").
143 use abbreviation ("k" or "Ki").
154
144
155 """
145 """
156
146
157 if not binary:
147 if not binary:
158 base = 1000
148 base = 1000
159 multiples = ('', 'k', 'M', 'G', 'T', 'P', 'E', 'Z', 'Y')
149 multiples = ('', 'k', 'M', 'G', 'T', 'P', 'E', 'Z', 'Y')
160 else:
150 else:
161 base = 1024
151 base = 1024
162 multiples = ('', 'Ki', 'Mi', 'Gi', 'Ti', 'Pi', 'Ei', 'Zi', 'Yi')
152 multiples = ('', 'Ki', 'Mi', 'Gi', 'Ti', 'Pi', 'Ei', 'Zi', 'Yi')
163
153
164 sign = ""
154 sign = ""
165 if size > 0:
155 if size > 0:
166 m = int(math.log(size, base))
156 m = int(math.log(size, base))
167 elif size < 0:
157 elif size < 0:
168 sign = "-"
158 sign = "-"
169 size = -size
159 size = -size
170 m = int(math.log(size, base))
160 m = int(math.log(size, base))
171 else:
161 else:
172 m = 0
162 m = 0
173 if m > 8:
163 if m > 8:
174 m = 8
164 m = 8
175
165
176 if m == 0:
166 if m == 0:
177 precision = '%.0f'
167 precision = '%.0f'
178 else:
168 else:
179 precision = '%%.%df' % precision
169 precision = '%%.%df' % precision
180
170
181 size = precision % (size / math.pow(base, m))
171 size = precision % (size / math.pow(base, m))
182
172
183 return '%s%s %s%s' % (sign, size.strip(), multiples[m], unit)
173 return '%s%s %s%s' % (sign, size.strip(), multiples[m], unit)
184
174
185
175
186 def _check_memory_usage(worker):
176 def _check_memory_usage(worker):
187 memory_max_usage = worker._memory_max_usage
177 memory_max_usage = worker._memory_max_usage
188 if not memory_max_usage:
178 if not memory_max_usage:
189 return
179 return
190
180
191 memory_usage_check_interval = worker._memory_usage_check_interval
181 memory_usage_check_interval = worker._memory_usage_check_interval
192 memory_usage_recovery_threshold = memory_max_usage * worker._memory_usage_recovery_threshold
182 memory_usage_recovery_threshold = memory_max_usage * worker._memory_usage_recovery_threshold
193
183
194 elapsed = time.time() - worker._last_memory_check_time
184 elapsed = time.time() - worker._last_memory_check_time
195 if elapsed > memory_usage_check_interval:
185 if elapsed > memory_usage_check_interval:
196 mem_usage = _get_process_rss()
186 mem_usage = _get_process_rss()
197 if mem_usage and mem_usage > memory_max_usage:
187 if mem_usage and mem_usage > memory_max_usage:
198 worker.log.info(
188 worker.log.info(
199 "memory usage %s > %s, forcing gc",
189 "memory usage %s > %s, forcing gc",
200 _format_data_size(mem_usage), _format_data_size(memory_max_usage))
190 _format_data_size(mem_usage), _format_data_size(memory_max_usage))
201 # Try to clean it up by forcing a full collection.
191 # Try to clean it up by forcing a full collection.
202 gc.collect()
192 gc.collect()
203 mem_usage = _get_process_rss()
193 mem_usage = _get_process_rss()
204 if mem_usage > memory_usage_recovery_threshold:
194 if mem_usage > memory_usage_recovery_threshold:
205 # Didn't clean up enough, we'll have to terminate.
195 # Didn't clean up enough, we'll have to terminate.
206 worker.log.warning(
196 worker.log.warning(
207 "memory usage %s > %s after gc, quitting",
197 "memory usage %s > %s after gc, quitting",
208 _format_data_size(mem_usage), _format_data_size(memory_max_usage))
198 _format_data_size(mem_usage), _format_data_size(memory_max_usage))
209 # This will cause worker to auto-restart itself
199 # This will cause worker to auto-restart itself
210 worker.alive = False
200 worker.alive = False
211 worker._last_memory_check_time = time.time()
201 worker._last_memory_check_time = time.time()
212
202
213
203
214 def worker_int(worker):
204 def worker_int(worker):
215 worker.log.info("[%-10s] worker received INT or QUIT signal", worker.pid)
205 worker.log.info("[%-10s] worker received INT or QUIT signal", worker.pid)
216
206
217 # get traceback info, on worker crash
207 # get traceback info, on worker crash
218 id2name = dict([(th.ident, th.name) for th in threading.enumerate()])
208 id2name = dict([(th.ident, th.name) for th in threading.enumerate()])
219 code = []
209 code = []
220 for thread_id, stack in sys._current_frames().items():
210 for thread_id, stack in sys._current_frames().items():
221 code.append(
211 code.append(
222 "\n# Thread: %s(%d)" % (id2name.get(thread_id, ""), thread_id))
212 "\n# Thread: %s(%d)" % (id2name.get(thread_id, ""), thread_id))
223 for fname, lineno, name, line in traceback.extract_stack(stack):
213 for fname, lineno, name, line in traceback.extract_stack(stack):
224 code.append('File: "%s", line %d, in %s' % (fname, lineno, name))
214 code.append('File: "%s", line %d, in %s' % (fname, lineno, name))
225 if line:
215 if line:
226 code.append(" %s" % (line.strip()))
216 code.append(" %s" % (line.strip()))
227 worker.log.debug("\n".join(code))
217 worker.log.debug("\n".join(code))
228
218
229
219
230 def worker_abort(worker):
220 def worker_abort(worker):
231 worker.log.info("[%-10s] worker received SIGABRT signal", worker.pid)
221 worker.log.info("[%-10s] worker received SIGABRT signal", worker.pid)
232
222
233
223
234 def worker_exit(server, worker):
224 def worker_exit(server, worker):
235 worker.log.info("[%-10s] worker exit", worker.pid)
225 worker.log.info("[%-10s] worker exit", worker.pid)
236
226
237
227
238 def child_exit(server, worker):
228 def child_exit(server, worker):
239 worker.log.info("[%-10s] worker child exit", worker.pid)
229 worker.log.info("[%-10s] worker child exit", worker.pid)
240
230
241
231
242 def pre_request(worker, req):
232 def pre_request(worker, req):
243 worker.start_time = time.time()
233 worker.start_time = time.time()
244 worker.log.debug(
234 worker.log.debug(
245 "GNCRN PRE WORKER [cnt:%s]: %s %s", worker.nr, req.method, req.path)
235 "GNCRN PRE WORKER [cnt:%s]: %s %s", worker.nr, req.method, req.path)
246
236
247
237
248 def post_request(worker, req, environ, resp):
238 def post_request(worker, req, environ, resp):
249 total_time = time.time() - worker.start_time
239 total_time = time.time() - worker.start_time
240 # Gunicorn sometimes has problems with reading the status_code
241 status_code = getattr(resp, 'status_code', '')
250 worker.log.debug(
242 worker.log.debug(
251 "GNCRN POST WORKER [cnt:%s]: %s %s resp: %s, Load Time: %.4fs",
243 "GNCRN POST WORKER [cnt:%s]: %s %s resp: %s, Load Time: %.4fs",
252 worker.nr, req.method, req.path, resp.status_code, total_time)
244 worker.nr, req.method, req.path, status_code, total_time)
253 _check_memory_usage(worker)
245 _check_memory_usage(worker)
254
246
255
247
256 class RhodeCodeLogger(Logger):
248 class RhodeCodeLogger(Logger):
257 """
249 """
258 Custom Logger that allows some customization that gunicorn doesn't allow
250 Custom Logger that allows some customization that gunicorn doesn't allow
259 """
251 """
260
252
261 datefmt = r"%Y-%m-%d %H:%M:%S"
253 datefmt = r"%Y-%m-%d %H:%M:%S"
262
254
263 def __init__(self, cfg):
255 def __init__(self, cfg):
264 Logger.__init__(self, cfg)
256 Logger.__init__(self, cfg)
265
257
266 def now(self):
258 def now(self):
267 """ return date in RhodeCode Log format """
259 """ return date in RhodeCode Log format """
268 now = time.time()
260 now = time.time()
269 msecs = int((now - long(now)) * 1000)
261 msecs = int((now - long(now)) * 1000)
270 return time.strftime(self.datefmt, time.localtime(now)) + '.{0:03d}'.format(msecs)
262 return time.strftime(self.datefmt, time.localtime(now)) + '.{0:03d}'.format(msecs)
271
263
272
264
273 logger_class = RhodeCodeLogger
265 logger_class = RhodeCodeLogger
General Comments 0
You need to be logged in to leave comments. Login now