##// END OF EJS Templates
more py25 compat fixes
marcink -
r2730:7949bc80 beta
parent child Browse files
Show More
@@ -1,532 +1,585
1 # -*- coding: utf-8 -*-
1 # -*- coding: utf-8 -*-
2 """
2 """
3 rhodecode.lib.compat
3 rhodecode.lib.compat
4 ~~~~~~~~~~~~~~~~~~~~
4 ~~~~~~~~~~~~~~~~~~~~
5
5
6 Python backward compatibility functions and common libs
6 Python backward compatibility functions and common libs
7
7
8
8
9 :created_on: Oct 7, 2011
9 :created_on: Oct 7, 2011
10 :author: marcink
10 :author: marcink
11 :copyright: (C) 2010-2010 Marcin Kuzminski <marcin@python-works.com>
11 :copyright: (C) 2010-2010 Marcin Kuzminski <marcin@python-works.com>
12 :license: GPLv3, see COPYING for more details.
12 :license: GPLv3, see COPYING for more details.
13 """
13 """
14 # This program is free software: you can redistribute it and/or modify
14 # This program is free software: you can redistribute it and/or modify
15 # it under the terms of the GNU General Public License as published by
15 # it under the terms of the GNU General Public License as published by
16 # the Free Software Foundation, either version 3 of the License, or
16 # the Free Software Foundation, either version 3 of the License, or
17 # (at your option) any later version.
17 # (at your option) any later version.
18 #
18 #
19 # This program is distributed in the hope that it will be useful,
19 # This program is distributed in the hope that it will be useful,
20 # but WITHOUT ANY WARRANTY; without even the implied warranty of
20 # but WITHOUT ANY WARRANTY; without even the implied warranty of
21 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
22 # GNU General Public License for more details.
22 # GNU General Public License for more details.
23 #
23 #
24 # You should have received a copy of the GNU General Public License
24 # You should have received a copy of the GNU General Public License
25 # along with this program. If not, see <http://www.gnu.org/licenses/>.
25 # along with this program. If not, see <http://www.gnu.org/licenses/>.
26
26
27 import os
27 import os
28 from rhodecode import __platform__, PLATFORM_WIN, __py_version__
28 from rhodecode import __platform__, PLATFORM_WIN, __py_version__
29
29
30 #==============================================================================
30 #==============================================================================
31 # json
31 # json
32 #==============================================================================
32 #==============================================================================
33 from rhodecode.lib.ext_json import json
33 from rhodecode.lib.ext_json import json
34
34
35
35
36 #==============================================================================
36 #==============================================================================
37 # izip_longest
37 # izip_longest
38 #==============================================================================
38 #==============================================================================
39 try:
39 try:
40 from itertools import izip_longest
40 from itertools import izip_longest
41 except ImportError:
41 except ImportError:
42 import itertools
42 import itertools
43
43
44 def izip_longest(*args, **kwds):
44 def izip_longest(*args, **kwds):
45 fillvalue = kwds.get("fillvalue")
45 fillvalue = kwds.get("fillvalue")
46
46
47 def sentinel(counter=([fillvalue] * (len(args) - 1)).pop):
47 def sentinel(counter=([fillvalue] * (len(args) - 1)).pop):
48 yield counter() # yields the fillvalue, or raises IndexError
48 yield counter() # yields the fillvalue, or raises IndexError
49
49
50 fillers = itertools.repeat(fillvalue)
50 fillers = itertools.repeat(fillvalue)
51 iters = [itertools.chain(it, sentinel(), fillers)
51 iters = [itertools.chain(it, sentinel(), fillers)
52 for it in args]
52 for it in args]
53 try:
53 try:
54 for tup in itertools.izip(*iters):
54 for tup in itertools.izip(*iters):
55 yield tup
55 yield tup
56 except IndexError:
56 except IndexError:
57 pass
57 pass
58
58
59
59
60 #==============================================================================
60 #==============================================================================
61 # OrderedDict
61 # OrderedDict
62 #==============================================================================
62 #==============================================================================
63
63
64 # Python Software Foundation License
64 # Python Software Foundation License
65
65
66 # XXX: it feels like using the class with "is" and "is not" instead of "==" and
66 # XXX: it feels like using the class with "is" and "is not" instead of "==" and
67 # "!=" should be faster.
67 # "!=" should be faster.
68 class _Nil(object):
68 class _Nil(object):
69
69
70 def __repr__(self):
70 def __repr__(self):
71 return "nil"
71 return "nil"
72
72
73 def __eq__(self, other):
73 def __eq__(self, other):
74 if (isinstance(other, _Nil)):
74 if (isinstance(other, _Nil)):
75 return True
75 return True
76 else:
76 else:
77 return NotImplemented
77 return NotImplemented
78
78
79 def __ne__(self, other):
79 def __ne__(self, other):
80 if (isinstance(other, _Nil)):
80 if (isinstance(other, _Nil)):
81 return False
81 return False
82 else:
82 else:
83 return NotImplemented
83 return NotImplemented
84
84
85 _nil = _Nil()
85 _nil = _Nil()
86
86
87
87
88 class _odict(object):
88 class _odict(object):
89 """Ordered dict data structure, with O(1) complexity for dict operations
89 """Ordered dict data structure, with O(1) complexity for dict operations
90 that modify one element.
90 that modify one element.
91
91
92 Overwriting values doesn't change their original sequential order.
92 Overwriting values doesn't change their original sequential order.
93 """
93 """
94
94
95 def _dict_impl(self):
95 def _dict_impl(self):
96 return None
96 return None
97
97
98 def __init__(self, data=(), **kwds):
98 def __init__(self, data=(), **kwds):
99 """This doesn't accept keyword initialization as normal dicts to avoid
99 """This doesn't accept keyword initialization as normal dicts to avoid
100 a trap - inside a function or method the keyword args are accessible
100 a trap - inside a function or method the keyword args are accessible
101 only as a dict, without a defined order, so their original order is
101 only as a dict, without a defined order, so their original order is
102 lost.
102 lost.
103 """
103 """
104 if kwds:
104 if kwds:
105 raise TypeError("__init__() of ordered dict takes no keyword "
105 raise TypeError("__init__() of ordered dict takes no keyword "
106 "arguments to avoid an ordering trap.")
106 "arguments to avoid an ordering trap.")
107 self._dict_impl().__init__(self)
107 self._dict_impl().__init__(self)
108 # If you give a normal dict, then the order of elements is undefined
108 # If you give a normal dict, then the order of elements is undefined
109 if hasattr(data, "iteritems"):
109 if hasattr(data, "iteritems"):
110 for key, val in data.iteritems():
110 for key, val in data.iteritems():
111 self[key] = val
111 self[key] = val
112 else:
112 else:
113 for key, val in data:
113 for key, val in data:
114 self[key] = val
114 self[key] = val
115
115
116 # Double-linked list header
116 # Double-linked list header
117 def _get_lh(self):
117 def _get_lh(self):
118 dict_impl = self._dict_impl()
118 dict_impl = self._dict_impl()
119 if not hasattr(self, '_lh'):
119 if not hasattr(self, '_lh'):
120 dict_impl.__setattr__(self, '_lh', _nil)
120 dict_impl.__setattr__(self, '_lh', _nil)
121 return dict_impl.__getattribute__(self, '_lh')
121 return dict_impl.__getattribute__(self, '_lh')
122
122
123 def _set_lh(self, val):
123 def _set_lh(self, val):
124 self._dict_impl().__setattr__(self, '_lh', val)
124 self._dict_impl().__setattr__(self, '_lh', val)
125
125
126 lh = property(_get_lh, _set_lh)
126 lh = property(_get_lh, _set_lh)
127
127
128 # Double-linked list tail
128 # Double-linked list tail
129 def _get_lt(self):
129 def _get_lt(self):
130 dict_impl = self._dict_impl()
130 dict_impl = self._dict_impl()
131 if not hasattr(self, '_lt'):
131 if not hasattr(self, '_lt'):
132 dict_impl.__setattr__(self, '_lt', _nil)
132 dict_impl.__setattr__(self, '_lt', _nil)
133 return dict_impl.__getattribute__(self, '_lt')
133 return dict_impl.__getattribute__(self, '_lt')
134
134
135 def _set_lt(self, val):
135 def _set_lt(self, val):
136 self._dict_impl().__setattr__(self, '_lt', val)
136 self._dict_impl().__setattr__(self, '_lt', val)
137
137
138 lt = property(_get_lt, _set_lt)
138 lt = property(_get_lt, _set_lt)
139
139
140 def __getitem__(self, key):
140 def __getitem__(self, key):
141 return self._dict_impl().__getitem__(self, key)[1]
141 return self._dict_impl().__getitem__(self, key)[1]
142
142
143 def __setitem__(self, key, val):
143 def __setitem__(self, key, val):
144 dict_impl = self._dict_impl()
144 dict_impl = self._dict_impl()
145 try:
145 try:
146 dict_impl.__getitem__(self, key)[1] = val
146 dict_impl.__getitem__(self, key)[1] = val
147 except KeyError:
147 except KeyError:
148 new = [dict_impl.__getattribute__(self, 'lt'), val, _nil]
148 new = [dict_impl.__getattribute__(self, 'lt'), val, _nil]
149 dict_impl.__setitem__(self, key, new)
149 dict_impl.__setitem__(self, key, new)
150 if dict_impl.__getattribute__(self, 'lt') == _nil:
150 if dict_impl.__getattribute__(self, 'lt') == _nil:
151 dict_impl.__setattr__(self, 'lh', key)
151 dict_impl.__setattr__(self, 'lh', key)
152 else:
152 else:
153 dict_impl.__getitem__(
153 dict_impl.__getitem__(
154 self, dict_impl.__getattribute__(self, 'lt'))[2] = key
154 self, dict_impl.__getattribute__(self, 'lt'))[2] = key
155 dict_impl.__setattr__(self, 'lt', key)
155 dict_impl.__setattr__(self, 'lt', key)
156
156
157 def __delitem__(self, key):
157 def __delitem__(self, key):
158 dict_impl = self._dict_impl()
158 dict_impl = self._dict_impl()
159 pred, _, succ = self._dict_impl().__getitem__(self, key)
159 pred, _, succ = self._dict_impl().__getitem__(self, key)
160 if pred == _nil:
160 if pred == _nil:
161 dict_impl.__setattr__(self, 'lh', succ)
161 dict_impl.__setattr__(self, 'lh', succ)
162 else:
162 else:
163 dict_impl.__getitem__(self, pred)[2] = succ
163 dict_impl.__getitem__(self, pred)[2] = succ
164 if succ == _nil:
164 if succ == _nil:
165 dict_impl.__setattr__(self, 'lt', pred)
165 dict_impl.__setattr__(self, 'lt', pred)
166 else:
166 else:
167 dict_impl.__getitem__(self, succ)[0] = pred
167 dict_impl.__getitem__(self, succ)[0] = pred
168 dict_impl.__delitem__(self, key)
168 dict_impl.__delitem__(self, key)
169
169
170 def __contains__(self, key):
170 def __contains__(self, key):
171 return key in self.keys()
171 return key in self.keys()
172
172
173 def __len__(self):
173 def __len__(self):
174 return len(self.keys())
174 return len(self.keys())
175
175
176 def __str__(self):
176 def __str__(self):
177 pairs = ("%r: %r" % (k, v) for k, v in self.iteritems())
177 pairs = ("%r: %r" % (k, v) for k, v in self.iteritems())
178 return "{%s}" % ", ".join(pairs)
178 return "{%s}" % ", ".join(pairs)
179
179
180 def __repr__(self):
180 def __repr__(self):
181 if self:
181 if self:
182 pairs = ("(%r, %r)" % (k, v) for k, v in self.iteritems())
182 pairs = ("(%r, %r)" % (k, v) for k, v in self.iteritems())
183 return "odict([%s])" % ", ".join(pairs)
183 return "odict([%s])" % ", ".join(pairs)
184 else:
184 else:
185 return "odict()"
185 return "odict()"
186
186
187 def get(self, k, x=None):
187 def get(self, k, x=None):
188 if k in self:
188 if k in self:
189 return self._dict_impl().__getitem__(self, k)[1]
189 return self._dict_impl().__getitem__(self, k)[1]
190 else:
190 else:
191 return x
191 return x
192
192
193 def __iter__(self):
193 def __iter__(self):
194 dict_impl = self._dict_impl()
194 dict_impl = self._dict_impl()
195 curr_key = dict_impl.__getattribute__(self, 'lh')
195 curr_key = dict_impl.__getattribute__(self, 'lh')
196 while curr_key != _nil:
196 while curr_key != _nil:
197 yield curr_key
197 yield curr_key
198 curr_key = dict_impl.__getitem__(self, curr_key)[2]
198 curr_key = dict_impl.__getitem__(self, curr_key)[2]
199
199
200 iterkeys = __iter__
200 iterkeys = __iter__
201
201
202 def keys(self):
202 def keys(self):
203 return list(self.iterkeys())
203 return list(self.iterkeys())
204
204
205 def itervalues(self):
205 def itervalues(self):
206 dict_impl = self._dict_impl()
206 dict_impl = self._dict_impl()
207 curr_key = dict_impl.__getattribute__(self, 'lh')
207 curr_key = dict_impl.__getattribute__(self, 'lh')
208 while curr_key != _nil:
208 while curr_key != _nil:
209 _, val, curr_key = dict_impl.__getitem__(self, curr_key)
209 _, val, curr_key = dict_impl.__getitem__(self, curr_key)
210 yield val
210 yield val
211
211
212 def values(self):
212 def values(self):
213 return list(self.itervalues())
213 return list(self.itervalues())
214
214
215 def iteritems(self):
215 def iteritems(self):
216 dict_impl = self._dict_impl()
216 dict_impl = self._dict_impl()
217 curr_key = dict_impl.__getattribute__(self, 'lh')
217 curr_key = dict_impl.__getattribute__(self, 'lh')
218 while curr_key != _nil:
218 while curr_key != _nil:
219 _, val, next_key = dict_impl.__getitem__(self, curr_key)
219 _, val, next_key = dict_impl.__getitem__(self, curr_key)
220 yield curr_key, val
220 yield curr_key, val
221 curr_key = next_key
221 curr_key = next_key
222
222
223 def items(self):
223 def items(self):
224 return list(self.iteritems())
224 return list(self.iteritems())
225
225
226 def sort(self, cmp=None, key=None, reverse=False):
226 def sort(self, cmp=None, key=None, reverse=False):
227 items = [(k, v) for k, v in self.items()]
227 items = [(k, v) for k, v in self.items()]
228 if cmp is not None:
228 if cmp is not None:
229 items = sorted(items, cmp=cmp)
229 items = sorted(items, cmp=cmp)
230 elif key is not None:
230 elif key is not None:
231 items = sorted(items, key=key)
231 items = sorted(items, key=key)
232 else:
232 else:
233 items = sorted(items, key=lambda x: x[1])
233 items = sorted(items, key=lambda x: x[1])
234 if reverse:
234 if reverse:
235 items.reverse()
235 items.reverse()
236 self.clear()
236 self.clear()
237 self.__init__(items)
237 self.__init__(items)
238
238
239 def clear(self):
239 def clear(self):
240 dict_impl = self._dict_impl()
240 dict_impl = self._dict_impl()
241 dict_impl.clear(self)
241 dict_impl.clear(self)
242 dict_impl.__setattr__(self, 'lh', _nil)
242 dict_impl.__setattr__(self, 'lh', _nil)
243 dict_impl.__setattr__(self, 'lt', _nil)
243 dict_impl.__setattr__(self, 'lt', _nil)
244
244
245 def copy(self):
245 def copy(self):
246 return self.__class__(self)
246 return self.__class__(self)
247
247
248 def update(self, data=(), **kwds):
248 def update(self, data=(), **kwds):
249 if kwds:
249 if kwds:
250 raise TypeError("update() of ordered dict takes no keyword "
250 raise TypeError("update() of ordered dict takes no keyword "
251 "arguments to avoid an ordering trap.")
251 "arguments to avoid an ordering trap.")
252 if hasattr(data, "iteritems"):
252 if hasattr(data, "iteritems"):
253 data = data.iteritems()
253 data = data.iteritems()
254 for key, val in data:
254 for key, val in data:
255 self[key] = val
255 self[key] = val
256
256
257 def setdefault(self, k, x=None):
257 def setdefault(self, k, x=None):
258 try:
258 try:
259 return self[k]
259 return self[k]
260 except KeyError:
260 except KeyError:
261 self[k] = x
261 self[k] = x
262 return x
262 return x
263
263
264 def pop(self, k, x=_nil):
264 def pop(self, k, x=_nil):
265 try:
265 try:
266 val = self[k]
266 val = self[k]
267 del self[k]
267 del self[k]
268 return val
268 return val
269 except KeyError:
269 except KeyError:
270 if x == _nil:
270 if x == _nil:
271 raise
271 raise
272 return x
272 return x
273
273
274 def popitem(self):
274 def popitem(self):
275 try:
275 try:
276 dict_impl = self._dict_impl()
276 dict_impl = self._dict_impl()
277 key = dict_impl.__getattribute__(self, 'lt')
277 key = dict_impl.__getattribute__(self, 'lt')
278 return key, self.pop(key)
278 return key, self.pop(key)
279 except KeyError:
279 except KeyError:
280 raise KeyError("'popitem(): ordered dictionary is empty'")
280 raise KeyError("'popitem(): ordered dictionary is empty'")
281
281
282 def riterkeys(self):
282 def riterkeys(self):
283 """To iterate on keys in reversed order.
283 """To iterate on keys in reversed order.
284 """
284 """
285 dict_impl = self._dict_impl()
285 dict_impl = self._dict_impl()
286 curr_key = dict_impl.__getattribute__(self, 'lt')
286 curr_key = dict_impl.__getattribute__(self, 'lt')
287 while curr_key != _nil:
287 while curr_key != _nil:
288 yield curr_key
288 yield curr_key
289 curr_key = dict_impl.__getitem__(self, curr_key)[0]
289 curr_key = dict_impl.__getitem__(self, curr_key)[0]
290
290
291 __reversed__ = riterkeys
291 __reversed__ = riterkeys
292
292
293 def rkeys(self):
293 def rkeys(self):
294 """List of the keys in reversed order.
294 """List of the keys in reversed order.
295 """
295 """
296 return list(self.riterkeys())
296 return list(self.riterkeys())
297
297
298 def ritervalues(self):
298 def ritervalues(self):
299 """To iterate on values in reversed order.
299 """To iterate on values in reversed order.
300 """
300 """
301 dict_impl = self._dict_impl()
301 dict_impl = self._dict_impl()
302 curr_key = dict_impl.__getattribute__(self, 'lt')
302 curr_key = dict_impl.__getattribute__(self, 'lt')
303 while curr_key != _nil:
303 while curr_key != _nil:
304 curr_key, val, _ = dict_impl.__getitem__(self, curr_key)
304 curr_key, val, _ = dict_impl.__getitem__(self, curr_key)
305 yield val
305 yield val
306
306
307 def rvalues(self):
307 def rvalues(self):
308 """List of the values in reversed order.
308 """List of the values in reversed order.
309 """
309 """
310 return list(self.ritervalues())
310 return list(self.ritervalues())
311
311
312 def riteritems(self):
312 def riteritems(self):
313 """To iterate on (key, value) in reversed order.
313 """To iterate on (key, value) in reversed order.
314 """
314 """
315 dict_impl = self._dict_impl()
315 dict_impl = self._dict_impl()
316 curr_key = dict_impl.__getattribute__(self, 'lt')
316 curr_key = dict_impl.__getattribute__(self, 'lt')
317 while curr_key != _nil:
317 while curr_key != _nil:
318 pred_key, val, _ = dict_impl.__getitem__(self, curr_key)
318 pred_key, val, _ = dict_impl.__getitem__(self, curr_key)
319 yield curr_key, val
319 yield curr_key, val
320 curr_key = pred_key
320 curr_key = pred_key
321
321
322 def ritems(self):
322 def ritems(self):
323 """List of the (key, value) in reversed order.
323 """List of the (key, value) in reversed order.
324 """
324 """
325 return list(self.riteritems())
325 return list(self.riteritems())
326
326
327 def firstkey(self):
327 def firstkey(self):
328 if self:
328 if self:
329 return self._dict_impl().__getattribute__(self, 'lh')
329 return self._dict_impl().__getattribute__(self, 'lh')
330 else:
330 else:
331 raise KeyError("'firstkey(): ordered dictionary is empty'")
331 raise KeyError("'firstkey(): ordered dictionary is empty'")
332
332
333 def lastkey(self):
333 def lastkey(self):
334 if self:
334 if self:
335 return self._dict_impl().__getattribute__(self, 'lt')
335 return self._dict_impl().__getattribute__(self, 'lt')
336 else:
336 else:
337 raise KeyError("'lastkey(): ordered dictionary is empty'")
337 raise KeyError("'lastkey(): ordered dictionary is empty'")
338
338
339 def as_dict(self):
339 def as_dict(self):
340 return self._dict_impl()(self.items())
340 return self._dict_impl()(self.items())
341
341
342 def _repr(self):
342 def _repr(self):
343 """_repr(): low level repr of the whole data contained in the odict.
343 """_repr(): low level repr of the whole data contained in the odict.
344 Useful for debugging.
344 Useful for debugging.
345 """
345 """
346 dict_impl = self._dict_impl()
346 dict_impl = self._dict_impl()
347 form = "odict low level repr lh,lt,data: %r, %r, %s"
347 form = "odict low level repr lh,lt,data: %r, %r, %s"
348 return form % (dict_impl.__getattribute__(self, 'lh'),
348 return form % (dict_impl.__getattribute__(self, 'lh'),
349 dict_impl.__getattribute__(self, 'lt'),
349 dict_impl.__getattribute__(self, 'lt'),
350 dict_impl.__repr__(self))
350 dict_impl.__repr__(self))
351
351
352
352
353 class OrderedDict(_odict, dict):
353 class OrderedDict(_odict, dict):
354
354
355 def _dict_impl(self):
355 def _dict_impl(self):
356 return dict
356 return dict
357
357
358
358
359 #==============================================================================
359 #==============================================================================
360 # OrderedSet
360 # OrderedSet
361 #==============================================================================
361 #==============================================================================
362 from sqlalchemy.util import OrderedSet
362 from sqlalchemy.util import OrderedSet
363
363
364
364
365 #==============================================================================
365 #==============================================================================
366 # kill FUNCTIONS
366 # kill FUNCTIONS
367 #==============================================================================
367 #==============================================================================
368 if __platform__ in PLATFORM_WIN:
368 if __platform__ in PLATFORM_WIN:
369 import ctypes
369 import ctypes
370
370
371 def kill(pid, sig):
371 def kill(pid, sig):
372 """kill function for Win32"""
372 """kill function for Win32"""
373 kernel32 = ctypes.windll.kernel32
373 kernel32 = ctypes.windll.kernel32
374 handle = kernel32.OpenProcess(1, 0, pid)
374 handle = kernel32.OpenProcess(1, 0, pid)
375 return (0 != kernel32.TerminateProcess(handle, 0))
375 return (0 != kernel32.TerminateProcess(handle, 0))
376
376
377 else:
377 else:
378 kill = os.kill
378 kill = os.kill
379
379
380
380
381 #==============================================================================
381 #==============================================================================
382 # itertools.product
382 # itertools.product
383 #==============================================================================
383 #==============================================================================
384
384
385 try:
385 try:
386 from itertools import product
386 from itertools import product
387 except ImportError:
387 except ImportError:
388 def product(*args, **kwds):
388 def product(*args, **kwds):
389 # product('ABCD', 'xy') --> Ax Ay Bx By Cx Cy Dx Dy
389 # product('ABCD', 'xy') --> Ax Ay Bx By Cx Cy Dx Dy
390 # product(range(2), repeat=3) --> 000 001 010 011 100 101 110 111
390 # product(range(2), repeat=3) --> 000 001 010 011 100 101 110 111
391 pools = map(tuple, args) * kwds.get('repeat', 1)
391 pools = map(tuple, args) * kwds.get('repeat', 1)
392 result = [[]]
392 result = [[]]
393 for pool in pools:
393 for pool in pools:
394 result = [x + [y] for x in result for y in pool]
394 result = [x + [y] for x in result for y in pool]
395 for prod in result:
395 for prod in result:
396 yield tuple(prod)
396 yield tuple(prod)
397
397
398
398
399 #==============================================================================
399 #==============================================================================
400 # BytesIO
400 # BytesIO
401 #==============================================================================
401 #==============================================================================
402
402
403 try:
403 try:
404 from io import BytesIO
404 from io import BytesIO
405 except ImportError:
405 except ImportError:
406 from cStringIO import StringIO as BytesIO
406 from cStringIO import StringIO as BytesIO
407
407
408
408
409 #==============================================================================
409 #==============================================================================
410 # deque
410 # deque
411 #==============================================================================
411 #==============================================================================
412
412
413 if __py_version__ >= (2, 6):
413 if __py_version__ >= (2, 6):
414 from collections import deque
414 from collections import deque
415 else:
415 else:
416 #need to implement our own deque with maxlen
416 #need to implement our own deque with maxlen
417 class deque(object):
417 class deque(object):
418
418
419 def __init__(self, iterable=(), maxlen=-1):
419 def __init__(self, iterable=(), maxlen=-1):
420 if not hasattr(self, 'data'):
420 if not hasattr(self, 'data'):
421 self.left = self.right = 0
421 self.left = self.right = 0
422 self.data = {}
422 self.data = {}
423 self.maxlen = maxlen
423 self.maxlen = maxlen
424 self.extend(iterable)
424 self.extend(iterable)
425
425
426 def append(self, x):
426 def append(self, x):
427 self.data[self.right] = x
427 self.data[self.right] = x
428 self.right += 1
428 self.right += 1
429 if self.maxlen != -1 and len(self) > self.maxlen:
429 if self.maxlen != -1 and len(self) > self.maxlen:
430 self.popleft()
430 self.popleft()
431
431
432 def appendleft(self, x):
432 def appendleft(self, x):
433 self.left -= 1
433 self.left -= 1
434 self.data[self.left] = x
434 self.data[self.left] = x
435 if self.maxlen != -1 and len(self) > self.maxlen:
435 if self.maxlen != -1 and len(self) > self.maxlen:
436 self.pop()
436 self.pop()
437
437
438 def pop(self):
438 def pop(self):
439 if self.left == self.right:
439 if self.left == self.right:
440 raise IndexError('cannot pop from empty deque')
440 raise IndexError('cannot pop from empty deque')
441 self.right -= 1
441 self.right -= 1
442 elem = self.data[self.right]
442 elem = self.data[self.right]
443 del self.data[self.right]
443 del self.data[self.right]
444 return elem
444 return elem
445
445
446 def popleft(self):
446 def popleft(self):
447 if self.left == self.right:
447 if self.left == self.right:
448 raise IndexError('cannot pop from empty deque')
448 raise IndexError('cannot pop from empty deque')
449 elem = self.data[self.left]
449 elem = self.data[self.left]
450 del self.data[self.left]
450 del self.data[self.left]
451 self.left += 1
451 self.left += 1
452 return elem
452 return elem
453
453
454 def clear(self):
454 def clear(self):
455 self.data.clear()
455 self.data.clear()
456 self.left = self.right = 0
456 self.left = self.right = 0
457
457
458 def extend(self, iterable):
458 def extend(self, iterable):
459 for elem in iterable:
459 for elem in iterable:
460 self.append(elem)
460 self.append(elem)
461
461
462 def extendleft(self, iterable):
462 def extendleft(self, iterable):
463 for elem in iterable:
463 for elem in iterable:
464 self.appendleft(elem)
464 self.appendleft(elem)
465
465
466 def rotate(self, n=1):
466 def rotate(self, n=1):
467 if self:
467 if self:
468 n %= len(self)
468 n %= len(self)
469 for i in xrange(n):
469 for i in xrange(n):
470 self.appendleft(self.pop())
470 self.appendleft(self.pop())
471
471
472 def __getitem__(self, i):
472 def __getitem__(self, i):
473 if i < 0:
473 if i < 0:
474 i += len(self)
474 i += len(self)
475 try:
475 try:
476 return self.data[i + self.left]
476 return self.data[i + self.left]
477 except KeyError:
477 except KeyError:
478 raise IndexError
478 raise IndexError
479
479
480 def __setitem__(self, i, value):
480 def __setitem__(self, i, value):
481 if i < 0:
481 if i < 0:
482 i += len(self)
482 i += len(self)
483 try:
483 try:
484 self.data[i + self.left] = value
484 self.data[i + self.left] = value
485 except KeyError:
485 except KeyError:
486 raise IndexError
486 raise IndexError
487
487
488 def __delitem__(self, i):
488 def __delitem__(self, i):
489 size = len(self)
489 size = len(self)
490 if not (-size <= i < size):
490 if not (-size <= i < size):
491 raise IndexError
491 raise IndexError
492 data = self.data
492 data = self.data
493 if i < 0:
493 if i < 0:
494 i += size
494 i += size
495 for j in xrange(self.left + i, self.right - 1):
495 for j in xrange(self.left + i, self.right - 1):
496 data[j] = data[j + 1]
496 data[j] = data[j + 1]
497 self.pop()
497 self.pop()
498
498
499 def __len__(self):
499 def __len__(self):
500 return self.right - self.left
500 return self.right - self.left
501
501
502 def __cmp__(self, other):
502 def __cmp__(self, other):
503 if type(self) != type(other):
503 if type(self) != type(other):
504 return cmp(type(self), type(other))
504 return cmp(type(self), type(other))
505 return cmp(list(self), list(other))
505 return cmp(list(self), list(other))
506
506
507 def __repr__(self, _track=[]):
507 def __repr__(self, _track=[]):
508 if id(self) in _track:
508 if id(self) in _track:
509 return '...'
509 return '...'
510 _track.append(id(self))
510 _track.append(id(self))
511 r = 'deque(%r, maxlen=%s)' % (list(self), self.maxlen)
511 r = 'deque(%r, maxlen=%s)' % (list(self), self.maxlen)
512 _track.remove(id(self))
512 _track.remove(id(self))
513 return r
513 return r
514
514
515 def __getstate__(self):
515 def __getstate__(self):
516 return (tuple(self),)
516 return (tuple(self),)
517
517
518 def __setstate__(self, s):
518 def __setstate__(self, s):
519 self.__init__(s[0])
519 self.__init__(s[0])
520
520
521 def __hash__(self):
521 def __hash__(self):
522 raise TypeError
522 raise TypeError
523
523
524 def __copy__(self):
524 def __copy__(self):
525 return self.__class__(self)
525 return self.__class__(self)
526
526
527 def __deepcopy__(self, memo={}):
527 def __deepcopy__(self, memo={}):
528 from copy import deepcopy
528 from copy import deepcopy
529 result = self.__class__()
529 result = self.__class__()
530 memo[id(self)] = result
530 memo[id(self)] = result
531 result.__init__(deepcopy(tuple(self), memo))
531 result.__init__(deepcopy(tuple(self), memo))
532 return result
532 return result
533
534
535 #==============================================================================
536 # threading.Event
537 #==============================================================================
538
539 if __py_version__ >= (2, 6):
540 from threading import Event
541 else:
542 from threading import _Verbose, Condition, Lock
543
544 def Event(*args, **kwargs):
545 return _Event(*args, **kwargs)
546
547 class _Event(_Verbose):
548
549 # After Tim Peters' event class (without is_posted())
550
551 def __init__(self, verbose=None):
552 _Verbose.__init__(self, verbose)
553 self.__cond = Condition(Lock())
554 self.__flag = False
555
556 def isSet(self):
557 return self.__flag
558
559 is_set = isSet
560
561 def set(self):
562 self.__cond.acquire()
563 try:
564 self.__flag = True
565 self.__cond.notify_all()
566 finally:
567 self.__cond.release()
568
569 def clear(self):
570 self.__cond.acquire()
571 try:
572 self.__flag = False
573 finally:
574 self.__cond.release()
575
576 def wait(self, timeout=None):
577 self.__cond.acquire()
578 try:
579 if not self.__flag:
580 self.__cond.wait(timeout)
581 finally:
582 self.__cond.release()
583
584
585
@@ -1,408 +1,408
1 '''
1 '''
2 Module provides a class allowing to wrap communication over subprocess.Popen
2 Module provides a class allowing to wrap communication over subprocess.Popen
3 input, output, error streams into a meaningfull, non-blocking, concurrent
3 input, output, error streams into a meaningfull, non-blocking, concurrent
4 stream processor exposing the output data as an iterator fitting to be a
4 stream processor exposing the output data as an iterator fitting to be a
5 return value passed by a WSGI applicaiton to a WSGI server per PEP 3333.
5 return value passed by a WSGI applicaiton to a WSGI server per PEP 3333.
6
6
7 Copyright (c) 2011 Daniel Dotsenko <dotsa@hotmail.com>
7 Copyright (c) 2011 Daniel Dotsenko <dotsa@hotmail.com>
8
8
9 This file is part of git_http_backend.py Project.
9 This file is part of git_http_backend.py Project.
10
10
11 git_http_backend.py Project is free software: you can redistribute it and/or
11 git_http_backend.py Project is free software: you can redistribute it and/or
12 modify it under the terms of the GNU Lesser General Public License as
12 modify it under the terms of the GNU Lesser General Public License as
13 published by the Free Software Foundation, either version 2.1 of the License,
13 published by the Free Software Foundation, either version 2.1 of the License,
14 or (at your option) any later version.
14 or (at your option) any later version.
15
15
16 git_http_backend.py Project is distributed in the hope that it will be useful,
16 git_http_backend.py Project is distributed in the hope that it will be useful,
17 but WITHOUT ANY WARRANTY; without even the implied warranty of
17 but WITHOUT ANY WARRANTY; without even the implied warranty of
18 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19 GNU Lesser General Public License for more details.
19 GNU Lesser General Public License for more details.
20
20
21 You should have received a copy of the GNU Lesser General Public License
21 You should have received a copy of the GNU Lesser General Public License
22 along with git_http_backend.py Project.
22 along with git_http_backend.py Project.
23 If not, see <http://www.gnu.org/licenses/>.
23 If not, see <http://www.gnu.org/licenses/>.
24 '''
24 '''
25 import os
25 import os
26 import subprocess
26 import subprocess
27 import threading
27 import threading
28 from rhodecode.lib.compat import deque
28 from rhodecode.lib.compat import deque, Event
29
29
30
30
31 class StreamFeeder(threading.Thread):
31 class StreamFeeder(threading.Thread):
32 """
32 """
33 Normal writing into pipe-like is blocking once the buffer is filled.
33 Normal writing into pipe-like is blocking once the buffer is filled.
34 This thread allows a thread to seep data from a file-like into a pipe
34 This thread allows a thread to seep data from a file-like into a pipe
35 without blocking the main thread.
35 without blocking the main thread.
36 We close inpipe once the end of the source stream is reached.
36 We close inpipe once the end of the source stream is reached.
37 """
37 """
38 def __init__(self, source):
38 def __init__(self, source):
39 super(StreamFeeder, self).__init__()
39 super(StreamFeeder, self).__init__()
40 self.daemon = True
40 self.daemon = True
41 filelike = False
41 filelike = False
42 self.bytes = bytes()
42 self.bytes = bytes()
43 if type(source) in (type(''), bytes, bytearray): # string-like
43 if type(source) in (type(''), bytes, bytearray): # string-like
44 self.bytes = bytes(source)
44 self.bytes = bytes(source)
45 else: # can be either file pointer or file-like
45 else: # can be either file pointer or file-like
46 if type(source) in (int, long): # file pointer it is
46 if type(source) in (int, long): # file pointer it is
47 ## converting file descriptor (int) stdin into file-like
47 ## converting file descriptor (int) stdin into file-like
48 try:
48 try:
49 source = os.fdopen(source, 'rb', 16384)
49 source = os.fdopen(source, 'rb', 16384)
50 except Exception:
50 except Exception:
51 pass
51 pass
52 # let's see if source is file-like by now
52 # let's see if source is file-like by now
53 try:
53 try:
54 filelike = source.read
54 filelike = source.read
55 except Exception:
55 except Exception:
56 pass
56 pass
57 if not filelike and not self.bytes:
57 if not filelike and not self.bytes:
58 raise TypeError("StreamFeeder's source object must be a readable "
58 raise TypeError("StreamFeeder's source object must be a readable "
59 "file-like, a file descriptor, or a string-like.")
59 "file-like, a file descriptor, or a string-like.")
60 self.source = source
60 self.source = source
61 self.readiface, self.writeiface = os.pipe()
61 self.readiface, self.writeiface = os.pipe()
62
62
63 def run(self):
63 def run(self):
64 t = self.writeiface
64 t = self.writeiface
65 if self.bytes:
65 if self.bytes:
66 os.write(t, self.bytes)
66 os.write(t, self.bytes)
67 else:
67 else:
68 s = self.source
68 s = self.source
69 b = s.read(4096)
69 b = s.read(4096)
70 while b:
70 while b:
71 os.write(t, b)
71 os.write(t, b)
72 b = s.read(4096)
72 b = s.read(4096)
73 os.close(t)
73 os.close(t)
74
74
75 @property
75 @property
76 def output(self):
76 def output(self):
77 return self.readiface
77 return self.readiface
78
78
79
79
80 class InputStreamChunker(threading.Thread):
80 class InputStreamChunker(threading.Thread):
81 def __init__(self, source, target, buffer_size, chunk_size):
81 def __init__(self, source, target, buffer_size, chunk_size):
82
82
83 super(InputStreamChunker, self).__init__()
83 super(InputStreamChunker, self).__init__()
84
84
85 self.daemon = True # die die die.
85 self.daemon = True # die die die.
86
86
87 self.source = source
87 self.source = source
88 self.target = target
88 self.target = target
89 self.chunk_count_max = int(buffer_size / chunk_size) + 1
89 self.chunk_count_max = int(buffer_size / chunk_size) + 1
90 self.chunk_size = chunk_size
90 self.chunk_size = chunk_size
91
91
92 self.data_added = threading.Event()
92 self.data_added = Event()
93 self.data_added.clear()
93 self.data_added.clear()
94
94
95 self.keep_reading = threading.Event()
95 self.keep_reading = Event()
96 self.keep_reading.set()
96 self.keep_reading.set()
97
97
98 self.EOF = threading.Event()
98 self.EOF = Event()
99 self.EOF.clear()
99 self.EOF.clear()
100
100
101 self.go = threading.Event()
101 self.go = Event()
102 self.go.set()
102 self.go.set()
103
103
104 def stop(self):
104 def stop(self):
105 self.go.clear()
105 self.go.clear()
106 self.EOF.set()
106 self.EOF.set()
107 try:
107 try:
108 # this is not proper, but is done to force the reader thread let
108 # this is not proper, but is done to force the reader thread let
109 # go of the input because, if successful, .close() will send EOF
109 # go of the input because, if successful, .close() will send EOF
110 # down the pipe.
110 # down the pipe.
111 self.source.close()
111 self.source.close()
112 except:
112 except:
113 pass
113 pass
114
114
115 def run(self):
115 def run(self):
116 s = self.source
116 s = self.source
117 t = self.target
117 t = self.target
118 cs = self.chunk_size
118 cs = self.chunk_size
119 ccm = self.chunk_count_max
119 ccm = self.chunk_count_max
120 kr = self.keep_reading
120 kr = self.keep_reading
121 da = self.data_added
121 da = self.data_added
122 go = self.go
122 go = self.go
123 b = s.read(cs)
123 b = s.read(cs)
124 while b and go.is_set():
124 while b and go.is_set():
125 if len(t) > ccm:
125 if len(t) > ccm:
126 kr.clear()
126 kr.clear()
127 kr.wait(2)
127 kr.wait(2)
128 # # this only works on 2.7.x and up
128 # # this only works on 2.7.x and up
129 # if not kr.wait(10):
129 # if not kr.wait(10):
130 # raise Exception("Timed out while waiting for input to be read.")
130 # raise Exception("Timed out while waiting for input to be read.")
131 # instead we'll use this
131 # instead we'll use this
132 if len(t) > ccm + 3:
132 if len(t) > ccm + 3:
133 raise IOError("Timed out while waiting for input from subprocess.")
133 raise IOError("Timed out while waiting for input from subprocess.")
134 t.append(b)
134 t.append(b)
135 da.set()
135 da.set()
136 b = s.read(cs)
136 b = s.read(cs)
137 self.EOF.set()
137 self.EOF.set()
138 da.set() # for cases when done but there was no input.
138 da.set() # for cases when done but there was no input.
139
139
140
140
141 class BufferedGenerator():
141 class BufferedGenerator():
142 '''
142 '''
143 Class behaves as a non-blocking, buffered pipe reader.
143 Class behaves as a non-blocking, buffered pipe reader.
144 Reads chunks of data (through a thread)
144 Reads chunks of data (through a thread)
145 from a blocking pipe, and attaches these to an array (Deque) of chunks.
145 from a blocking pipe, and attaches these to an array (Deque) of chunks.
146 Reading is halted in the thread when max chunks is internally buffered.
146 Reading is halted in the thread when max chunks is internally buffered.
147 The .next() may operate in blocking or non-blocking fashion by yielding
147 The .next() may operate in blocking or non-blocking fashion by yielding
148 '' if no data is ready
148 '' if no data is ready
149 to be sent or by not returning until there is some data to send
149 to be sent or by not returning until there is some data to send
150 When we get EOF from underlying source pipe we raise the marker to raise
150 When we get EOF from underlying source pipe we raise the marker to raise
151 StopIteration after the last chunk of data is yielded.
151 StopIteration after the last chunk of data is yielded.
152 '''
152 '''
153
153
154 def __init__(self, source, buffer_size=65536, chunk_size=4096,
154 def __init__(self, source, buffer_size=65536, chunk_size=4096,
155 starting_values=[], bottomless=False):
155 starting_values=[], bottomless=False):
156
156
157 if bottomless:
157 if bottomless:
158 maxlen = int(buffer_size / chunk_size)
158 maxlen = int(buffer_size / chunk_size)
159 else:
159 else:
160 maxlen = None
160 maxlen = None
161
161
162 self.data = deque(starting_values, maxlen)
162 self.data = deque(starting_values, maxlen)
163
163
164 self.worker = InputStreamChunker(source, self.data, buffer_size,
164 self.worker = InputStreamChunker(source, self.data, buffer_size,
165 chunk_size)
165 chunk_size)
166 if starting_values:
166 if starting_values:
167 self.worker.data_added.set()
167 self.worker.data_added.set()
168 self.worker.start()
168 self.worker.start()
169
169
170 ####################
170 ####################
171 # Generator's methods
171 # Generator's methods
172 ####################
172 ####################
173
173
174 def __iter__(self):
174 def __iter__(self):
175 return self
175 return self
176
176
177 def next(self):
177 def next(self):
178 while not len(self.data) and not self.worker.EOF.is_set():
178 while not len(self.data) and not self.worker.EOF.is_set():
179 self.worker.data_added.clear()
179 self.worker.data_added.clear()
180 self.worker.data_added.wait(0.2)
180 self.worker.data_added.wait(0.2)
181 if len(self.data):
181 if len(self.data):
182 self.worker.keep_reading.set()
182 self.worker.keep_reading.set()
183 return bytes(self.data.popleft())
183 return bytes(self.data.popleft())
184 elif self.worker.EOF.is_set():
184 elif self.worker.EOF.is_set():
185 raise StopIteration
185 raise StopIteration
186
186
187 def throw(self, type, value=None, traceback=None):
187 def throw(self, type, value=None, traceback=None):
188 if not self.worker.EOF.is_set():
188 if not self.worker.EOF.is_set():
189 raise type(value)
189 raise type(value)
190
190
191 def start(self):
191 def start(self):
192 self.worker.start()
192 self.worker.start()
193
193
194 def stop(self):
194 def stop(self):
195 self.worker.stop()
195 self.worker.stop()
196
196
197 def close(self):
197 def close(self):
198 try:
198 try:
199 self.worker.stop()
199 self.worker.stop()
200 self.throw(GeneratorExit)
200 self.throw(GeneratorExit)
201 except (GeneratorExit, StopIteration):
201 except (GeneratorExit, StopIteration):
202 pass
202 pass
203
203
204 def __del__(self):
204 def __del__(self):
205 self.close()
205 self.close()
206
206
207 ####################
207 ####################
208 # Threaded reader's infrastructure.
208 # Threaded reader's infrastructure.
209 ####################
209 ####################
210 @property
210 @property
211 def input(self):
211 def input(self):
212 return self.worker.w
212 return self.worker.w
213
213
214 @property
214 @property
215 def data_added_event(self):
215 def data_added_event(self):
216 return self.worker.data_added
216 return self.worker.data_added
217
217
218 @property
218 @property
219 def data_added(self):
219 def data_added(self):
220 return self.worker.data_added.is_set()
220 return self.worker.data_added.is_set()
221
221
222 @property
222 @property
223 def reading_paused(self):
223 def reading_paused(self):
224 return not self.worker.keep_reading.is_set()
224 return not self.worker.keep_reading.is_set()
225
225
226 @property
226 @property
227 def done_reading_event(self):
227 def done_reading_event(self):
228 '''
228 '''
229 Done_reding does not mean that the iterator's buffer is empty.
229 Done_reding does not mean that the iterator's buffer is empty.
230 Iterator might have done reading from underlying source, but the read
230 Iterator might have done reading from underlying source, but the read
231 chunks might still be available for serving through .next() method.
231 chunks might still be available for serving through .next() method.
232
232
233 @return An Event class instance.
233 @return An Event class instance.
234 '''
234 '''
235 return self.worker.EOF
235 return self.worker.EOF
236
236
237 @property
237 @property
238 def done_reading(self):
238 def done_reading(self):
239 '''
239 '''
240 Done_reding does not mean that the iterator's buffer is empty.
240 Done_reding does not mean that the iterator's buffer is empty.
241 Iterator might have done reading from underlying source, but the read
241 Iterator might have done reading from underlying source, but the read
242 chunks might still be available for serving through .next() method.
242 chunks might still be available for serving through .next() method.
243
243
244 @return An Bool value.
244 @return An Bool value.
245 '''
245 '''
246 return self.worker.EOF.is_set()
246 return self.worker.EOF.is_set()
247
247
248 @property
248 @property
249 def length(self):
249 def length(self):
250 '''
250 '''
251 returns int.
251 returns int.
252
252
253 This is the lenght of the que of chunks, not the length of
253 This is the lenght of the que of chunks, not the length of
254 the combined contents in those chunks.
254 the combined contents in those chunks.
255
255
256 __len__() cannot be meaningfully implemented because this
256 __len__() cannot be meaningfully implemented because this
257 reader is just flying throuh a bottomless pit content and
257 reader is just flying throuh a bottomless pit content and
258 can only know the lenght of what it already saw.
258 can only know the lenght of what it already saw.
259
259
260 If __len__() on WSGI server per PEP 3333 returns a value,
260 If __len__() on WSGI server per PEP 3333 returns a value,
261 the responce's length will be set to that. In order not to
261 the responce's length will be set to that. In order not to
262 confuse WSGI PEP3333 servers, we will not implement __len__
262 confuse WSGI PEP3333 servers, we will not implement __len__
263 at all.
263 at all.
264 '''
264 '''
265 return len(self.data)
265 return len(self.data)
266
266
267 def prepend(self, x):
267 def prepend(self, x):
268 self.data.appendleft(x)
268 self.data.appendleft(x)
269
269
270 def append(self, x):
270 def append(self, x):
271 self.data.append(x)
271 self.data.append(x)
272
272
273 def extend(self, o):
273 def extend(self, o):
274 self.data.extend(o)
274 self.data.extend(o)
275
275
276 def __getitem__(self, i):
276 def __getitem__(self, i):
277 return self.data[i]
277 return self.data[i]
278
278
279
279
280 class SubprocessIOChunker(object):
280 class SubprocessIOChunker(object):
281 '''
281 '''
282 Processor class wrapping handling of subprocess IO.
282 Processor class wrapping handling of subprocess IO.
283
283
284 In a way, this is a "communicate()" replacement with a twist.
284 In a way, this is a "communicate()" replacement with a twist.
285
285
286 - We are multithreaded. Writing in and reading out, err are all sep threads.
286 - We are multithreaded. Writing in and reading out, err are all sep threads.
287 - We support concurrent (in and out) stream processing.
287 - We support concurrent (in and out) stream processing.
288 - The output is not a stream. It's a queue of read string (bytes, not unicode)
288 - The output is not a stream. It's a queue of read string (bytes, not unicode)
289 chunks. The object behaves as an iterable. You can "for chunk in obj:" us.
289 chunks. The object behaves as an iterable. You can "for chunk in obj:" us.
290 - We are non-blocking in more respects than communicate()
290 - We are non-blocking in more respects than communicate()
291 (reading from subprocess out pauses when internal buffer is full, but
291 (reading from subprocess out pauses when internal buffer is full, but
292 does not block the parent calling code. On the flip side, reading from
292 does not block the parent calling code. On the flip side, reading from
293 slow-yielding subprocess may block the iteration until data shows up. This
293 slow-yielding subprocess may block the iteration until data shows up. This
294 does not block the parallel inpipe reading occurring parallel thread.)
294 does not block the parallel inpipe reading occurring parallel thread.)
295
295
296 The purpose of the object is to allow us to wrap subprocess interactions into
296 The purpose of the object is to allow us to wrap subprocess interactions into
297 and interable that can be passed to a WSGI server as the application's return
297 and interable that can be passed to a WSGI server as the application's return
298 value. Because of stream-processing-ability, WSGI does not have to read ALL
298 value. Because of stream-processing-ability, WSGI does not have to read ALL
299 of the subprocess's output and buffer it, before handing it to WSGI server for
299 of the subprocess's output and buffer it, before handing it to WSGI server for
300 HTTP response. Instead, the class initializer reads just a bit of the stream
300 HTTP response. Instead, the class initializer reads just a bit of the stream
301 to figure out if error ocurred or likely to occur and if not, just hands the
301 to figure out if error ocurred or likely to occur and if not, just hands the
302 further iteration over subprocess output to the server for completion of HTTP
302 further iteration over subprocess output to the server for completion of HTTP
303 response.
303 response.
304
304
305 The real or perceived subprocess error is trapped and raised as one of
305 The real or perceived subprocess error is trapped and raised as one of
306 EnvironmentError family of exceptions
306 EnvironmentError family of exceptions
307
307
308 Example usage:
308 Example usage:
309 # try:
309 # try:
310 # answer = SubprocessIOChunker(
310 # answer = SubprocessIOChunker(
311 # cmd,
311 # cmd,
312 # input,
312 # input,
313 # buffer_size = 65536,
313 # buffer_size = 65536,
314 # chunk_size = 4096
314 # chunk_size = 4096
315 # )
315 # )
316 # except (EnvironmentError) as e:
316 # except (EnvironmentError) as e:
317 # print str(e)
317 # print str(e)
318 # raise e
318 # raise e
319 #
319 #
320 # return answer
320 # return answer
321
321
322
322
323 '''
323 '''
324 def __init__(self, cmd, inputstream=None, buffer_size=65536,
324 def __init__(self, cmd, inputstream=None, buffer_size=65536,
325 chunk_size=4096, starting_values=[], **kwargs):
325 chunk_size=4096, starting_values=[], **kwargs):
326 '''
326 '''
327 Initializes SubprocessIOChunker
327 Initializes SubprocessIOChunker
328
328
329 :param cmd: A Subprocess.Popen style "cmd". Can be string or array of strings
329 :param cmd: A Subprocess.Popen style "cmd". Can be string or array of strings
330 :param inputstream: (Default: None) A file-like, string, or file pointer.
330 :param inputstream: (Default: None) A file-like, string, or file pointer.
331 :param buffer_size: (Default: 65536) A size of total buffer per stream in bytes.
331 :param buffer_size: (Default: 65536) A size of total buffer per stream in bytes.
332 :param chunk_size: (Default: 4096) A max size of a chunk. Actual chunk may be smaller.
332 :param chunk_size: (Default: 4096) A max size of a chunk. Actual chunk may be smaller.
333 :param starting_values: (Default: []) An array of strings to put in front of output que.
333 :param starting_values: (Default: []) An array of strings to put in front of output que.
334 '''
334 '''
335
335
336 if inputstream:
336 if inputstream:
337 input_streamer = StreamFeeder(inputstream)
337 input_streamer = StreamFeeder(inputstream)
338 input_streamer.start()
338 input_streamer.start()
339 inputstream = input_streamer.output
339 inputstream = input_streamer.output
340
340
341 if isinstance(cmd, (list, tuple)):
341 if isinstance(cmd, (list, tuple)):
342 cmd = ' '.join(cmd)
342 cmd = ' '.join(cmd)
343
343
344 _p = subprocess.Popen(cmd,
344 _p = subprocess.Popen(cmd,
345 bufsize=-1,
345 bufsize=-1,
346 shell=True,
346 shell=True,
347 stdin=inputstream,
347 stdin=inputstream,
348 stdout=subprocess.PIPE,
348 stdout=subprocess.PIPE,
349 stderr=subprocess.PIPE,
349 stderr=subprocess.PIPE,
350 **kwargs
350 **kwargs
351 )
351 )
352
352
353 bg_out = BufferedGenerator(_p.stdout, buffer_size, chunk_size, starting_values)
353 bg_out = BufferedGenerator(_p.stdout, buffer_size, chunk_size, starting_values)
354 bg_err = BufferedGenerator(_p.stderr, 16000, 1, bottomless=True)
354 bg_err = BufferedGenerator(_p.stderr, 16000, 1, bottomless=True)
355
355
356 while not bg_out.done_reading and not bg_out.reading_paused and not bg_err.length:
356 while not bg_out.done_reading and not bg_out.reading_paused and not bg_err.length:
357 # doing this until we reach either end of file, or end of buffer.
357 # doing this until we reach either end of file, or end of buffer.
358 bg_out.data_added_event.wait(1)
358 bg_out.data_added_event.wait(1)
359 bg_out.data_added_event.clear()
359 bg_out.data_added_event.clear()
360
360
361 # at this point it's still ambiguous if we are done reading or just full buffer.
361 # at this point it's still ambiguous if we are done reading or just full buffer.
362 # Either way, if error (returned by ended process, or implied based on
362 # Either way, if error (returned by ended process, or implied based on
363 # presence of stuff in stderr output) we error out.
363 # presence of stuff in stderr output) we error out.
364 # Else, we are happy.
364 # Else, we are happy.
365 _returncode = _p.poll()
365 _returncode = _p.poll()
366 if _returncode or (_returncode == None and bg_err.length):
366 if _returncode or (_returncode == None and bg_err.length):
367 try:
367 try:
368 _p.terminate()
368 _p.terminate()
369 except:
369 except:
370 pass
370 pass
371 bg_out.stop()
371 bg_out.stop()
372 bg_err.stop()
372 bg_err.stop()
373 err = '%s' % ''.join(bg_err)
373 err = '%s' % ''.join(bg_err)
374 raise EnvironmentError("Subprocess exited due to an error:\n" + err)
374 raise EnvironmentError("Subprocess exited due to an error:\n" + err)
375
375
376 self.process = _p
376 self.process = _p
377 self.output = bg_out
377 self.output = bg_out
378 self.error = bg_err
378 self.error = bg_err
379
379
380 def __iter__(self):
380 def __iter__(self):
381 return self
381 return self
382
382
383 def next(self):
383 def next(self):
384 if self.process.poll():
384 if self.process.poll():
385 err = '%s' % ''.join(self.error)
385 err = '%s' % ''.join(self.error)
386 raise EnvironmentError("Subprocess exited due to an error:\n" + err)
386 raise EnvironmentError("Subprocess exited due to an error:\n" + err)
387 return self.output.next()
387 return self.output.next()
388
388
389 def throw(self, type, value=None, traceback=None):
389 def throw(self, type, value=None, traceback=None):
390 if self.output.length or not self.output.done_reading:
390 if self.output.length or not self.output.done_reading:
391 raise type(value)
391 raise type(value)
392
392
393 def close(self):
393 def close(self):
394 try:
394 try:
395 self.process.terminate()
395 self.process.terminate()
396 except:
396 except:
397 pass
397 pass
398 try:
398 try:
399 self.output.close()
399 self.output.close()
400 except:
400 except:
401 pass
401 pass
402 try:
402 try:
403 self.error.close()
403 self.error.close()
404 except:
404 except:
405 pass
405 pass
406
406
407 def __del__(self):
407 def __del__(self):
408 self.close()
408 self.close()
General Comments 0
You need to be logged in to leave comments. Login now