##// END OF EJS Templates
py3: remove dead code to make file descriptors non-inheritable...
Manuel Jacob -
r50191:b6b6ae9e default
parent child Browse files
Show More
@@ -1,365 +1,340
1 #!/usr/bin/env python
1 #!/usr/bin/env python
2 """
2 """
3 Tests the buffering behavior of stdio streams in `mercurial.utils.procutil`.
3 Tests the buffering behavior of stdio streams in `mercurial.utils.procutil`.
4 """
4 """
5
5
6 import contextlib
6 import contextlib
7 import errno
7 import errno
8 import os
8 import os
9 import pickle
9 import pickle
10 import signal
10 import signal
11 import subprocess
11 import subprocess
12 import sys
12 import sys
13 import tempfile
13 import tempfile
14 import unittest
14 import unittest
15
15
16 from mercurial import pycompat, util
16 from mercurial import pycompat, util
17
17
18
18
19 if pycompat.ispy3:
20
21 def set_noninheritable(fd):
22 # On Python 3, file descriptors are non-inheritable by default.
23 pass
24
25
26 else:
27 if pycompat.iswindows:
28 # unused
29 set_noninheritable = None
30 else:
31 import fcntl
32
33 def set_noninheritable(fd):
34 old = fcntl.fcntl(fd, fcntl.F_GETFD)
35 fcntl.fcntl(fd, fcntl.F_SETFD, old | fcntl.FD_CLOEXEC)
36
37
38 TEST_BUFFERING_CHILD_SCRIPT = r'''
19 TEST_BUFFERING_CHILD_SCRIPT = r'''
39 import os
20 import os
40
21
41 from mercurial import dispatch
22 from mercurial import dispatch
42 from mercurial.utils import procutil
23 from mercurial.utils import procutil
43
24
44 dispatch.initstdio()
25 dispatch.initstdio()
45 procutil.{stream}.write(b'aaa')
26 procutil.{stream}.write(b'aaa')
46 os.write(procutil.{stream}.fileno(), b'[written aaa]')
27 os.write(procutil.{stream}.fileno(), b'[written aaa]')
47 procutil.{stream}.write(b'bbb\n')
28 procutil.{stream}.write(b'bbb\n')
48 os.write(procutil.{stream}.fileno(), b'[written bbb\\n]')
29 os.write(procutil.{stream}.fileno(), b'[written bbb\\n]')
49 '''
30 '''
50 UNBUFFERED = b'aaa[written aaa]bbb\n[written bbb\\n]'
31 UNBUFFERED = b'aaa[written aaa]bbb\n[written bbb\\n]'
51 LINE_BUFFERED = b'[written aaa]aaabbb\n[written bbb\\n]'
32 LINE_BUFFERED = b'[written aaa]aaabbb\n[written bbb\\n]'
52 FULLY_BUFFERED = b'[written aaa][written bbb\\n]aaabbb\n'
33 FULLY_BUFFERED = b'[written aaa][written bbb\\n]aaabbb\n'
53
34
54
35
55 TEST_LARGE_WRITE_CHILD_SCRIPT = r'''
36 TEST_LARGE_WRITE_CHILD_SCRIPT = r'''
56 import os
37 import os
57 import signal
38 import signal
58 import sys
39 import sys
59
40
60 from mercurial import dispatch
41 from mercurial import dispatch
61 from mercurial.utils import procutil
42 from mercurial.utils import procutil
62
43
63 signal.signal(signal.SIGINT, lambda *x: None)
44 signal.signal(signal.SIGINT, lambda *x: None)
64 dispatch.initstdio()
45 dispatch.initstdio()
65 write_result = procutil.{stream}.write(b'x' * 1048576)
46 write_result = procutil.{stream}.write(b'x' * 1048576)
66 with os.fdopen(
47 with os.fdopen(
67 os.open({write_result_fn!r}, os.O_WRONLY | getattr(os, 'O_TEMPORARY', 0)),
48 os.open({write_result_fn!r}, os.O_WRONLY | getattr(os, 'O_TEMPORARY', 0)),
68 'w',
49 'w',
69 ) as write_result_f:
50 ) as write_result_f:
70 write_result_f.write(str(write_result))
51 write_result_f.write(str(write_result))
71 '''
52 '''
72
53
73
54
74 TEST_BROKEN_PIPE_CHILD_SCRIPT = r'''
55 TEST_BROKEN_PIPE_CHILD_SCRIPT = r'''
75 import os
56 import os
76 import pickle
57 import pickle
77
58
78 from mercurial import dispatch
59 from mercurial import dispatch
79 from mercurial.utils import procutil
60 from mercurial.utils import procutil
80
61
81 dispatch.initstdio()
62 dispatch.initstdio()
82 procutil.stdin.read(1) # wait until parent process closed pipe
63 procutil.stdin.read(1) # wait until parent process closed pipe
83 try:
64 try:
84 procutil.{stream}.write(b'test')
65 procutil.{stream}.write(b'test')
85 procutil.{stream}.flush()
66 procutil.{stream}.flush()
86 except EnvironmentError as e:
67 except EnvironmentError as e:
87 with os.fdopen(
68 with os.fdopen(
88 os.open(
69 os.open(
89 {err_fn!r},
70 {err_fn!r},
90 os.O_WRONLY
71 os.O_WRONLY
91 | getattr(os, 'O_BINARY', 0)
72 | getattr(os, 'O_BINARY', 0)
92 | getattr(os, 'O_TEMPORARY', 0),
73 | getattr(os, 'O_TEMPORARY', 0),
93 ),
74 ),
94 'wb',
75 'wb',
95 ) as err_f:
76 ) as err_f:
96 pickle.dump(e, err_f)
77 pickle.dump(e, err_f)
97 # Exit early to suppress further broken pipe errors at interpreter shutdown.
78 # Exit early to suppress further broken pipe errors at interpreter shutdown.
98 os._exit(0)
79 os._exit(0)
99 '''
80 '''
100
81
101
82
102 @contextlib.contextmanager
83 @contextlib.contextmanager
103 def _closing(fds):
84 def _closing(fds):
104 try:
85 try:
105 yield
86 yield
106 finally:
87 finally:
107 for fd in fds:
88 for fd in fds:
108 try:
89 try:
109 os.close(fd)
90 os.close(fd)
110 except EnvironmentError:
91 except EnvironmentError:
111 pass
92 pass
112
93
113
94
114 # In the following, we set the FDs non-inheritable mainly to make it possible
95 # In the following, we set the FDs non-inheritable mainly to make it possible
115 # for tests to close the receiving end of the pipe / PTYs.
96 # for tests to close the receiving end of the pipe / PTYs.
116
97
117
98
118 @contextlib.contextmanager
99 @contextlib.contextmanager
119 def _devnull():
100 def _devnull():
120 devnull = os.open(os.devnull, os.O_WRONLY)
101 devnull = os.open(os.devnull, os.O_WRONLY)
121 # We don't have a receiving end, so it's not worth the effort on Python 2
102 # We don't have a receiving end, so it's not worth the effort on Python 2
122 # on Windows to make the FD non-inheritable.
103 # on Windows to make the FD non-inheritable.
123 with _closing([devnull]):
104 with _closing([devnull]):
124 yield (None, devnull)
105 yield (None, devnull)
125
106
126
107
127 @contextlib.contextmanager
108 @contextlib.contextmanager
128 def _pipes():
109 def _pipes():
129 rwpair = os.pipe()
110 rwpair = os.pipe()
130 # Pipes are already non-inheritable on Windows.
131 if not pycompat.iswindows:
132 set_noninheritable(rwpair[0])
133 set_noninheritable(rwpair[1])
134 with _closing(rwpair):
111 with _closing(rwpair):
135 yield rwpair
112 yield rwpair
136
113
137
114
138 @contextlib.contextmanager
115 @contextlib.contextmanager
139 def _ptys():
116 def _ptys():
140 if pycompat.iswindows:
117 if pycompat.iswindows:
141 raise unittest.SkipTest("PTYs are not supported on Windows")
118 raise unittest.SkipTest("PTYs are not supported on Windows")
142 import pty
119 import pty
143 import tty
120 import tty
144
121
145 rwpair = pty.openpty()
122 rwpair = pty.openpty()
146 set_noninheritable(rwpair[0])
147 set_noninheritable(rwpair[1])
148 with _closing(rwpair):
123 with _closing(rwpair):
149 tty.setraw(rwpair[0])
124 tty.setraw(rwpair[0])
150 yield rwpair
125 yield rwpair
151
126
152
127
153 def _readall(fd, buffer_size, initial_buf=None):
128 def _readall(fd, buffer_size, initial_buf=None):
154 buf = initial_buf or []
129 buf = initial_buf or []
155 while True:
130 while True:
156 try:
131 try:
157 s = os.read(fd, buffer_size)
132 s = os.read(fd, buffer_size)
158 except OSError as e:
133 except OSError as e:
159 if e.errno == errno.EIO:
134 if e.errno == errno.EIO:
160 # If the child-facing PTY got closed, reading from the
135 # If the child-facing PTY got closed, reading from the
161 # parent-facing PTY raises EIO.
136 # parent-facing PTY raises EIO.
162 break
137 break
163 raise
138 raise
164 if not s:
139 if not s:
165 break
140 break
166 buf.append(s)
141 buf.append(s)
167 return b''.join(buf)
142 return b''.join(buf)
168
143
169
144
170 class TestStdio(unittest.TestCase):
145 class TestStdio(unittest.TestCase):
171 def _test(
146 def _test(
172 self,
147 self,
173 child_script,
148 child_script,
174 stream,
149 stream,
175 rwpair_generator,
150 rwpair_generator,
176 check_output,
151 check_output,
177 python_args=[],
152 python_args=[],
178 post_child_check=None,
153 post_child_check=None,
179 stdin_generator=None,
154 stdin_generator=None,
180 ):
155 ):
181 assert stream in ('stdout', 'stderr')
156 assert stream in ('stdout', 'stderr')
182 if stdin_generator is None:
157 if stdin_generator is None:
183 stdin_generator = open(os.devnull, 'rb')
158 stdin_generator = open(os.devnull, 'rb')
184 with rwpair_generator() as (
159 with rwpair_generator() as (
185 stream_receiver,
160 stream_receiver,
186 child_stream,
161 child_stream,
187 ), stdin_generator as child_stdin:
162 ), stdin_generator as child_stdin:
188 proc = subprocess.Popen(
163 proc = subprocess.Popen(
189 [sys.executable] + python_args + ['-c', child_script],
164 [sys.executable] + python_args + ['-c', child_script],
190 stdin=child_stdin,
165 stdin=child_stdin,
191 stdout=child_stream if stream == 'stdout' else None,
166 stdout=child_stream if stream == 'stdout' else None,
192 stderr=child_stream if stream == 'stderr' else None,
167 stderr=child_stream if stream == 'stderr' else None,
193 )
168 )
194 try:
169 try:
195 os.close(child_stream)
170 os.close(child_stream)
196 if stream_receiver is not None:
171 if stream_receiver is not None:
197 check_output(stream_receiver, proc)
172 check_output(stream_receiver, proc)
198 except: # re-raises
173 except: # re-raises
199 proc.terminate()
174 proc.terminate()
200 raise
175 raise
201 finally:
176 finally:
202 retcode = proc.wait()
177 retcode = proc.wait()
203 self.assertEqual(retcode, 0)
178 self.assertEqual(retcode, 0)
204 if post_child_check is not None:
179 if post_child_check is not None:
205 post_child_check()
180 post_child_check()
206
181
207 def _test_buffering(
182 def _test_buffering(
208 self, stream, rwpair_generator, expected_output, python_args=[]
183 self, stream, rwpair_generator, expected_output, python_args=[]
209 ):
184 ):
210 def check_output(stream_receiver, proc):
185 def check_output(stream_receiver, proc):
211 self.assertEqual(_readall(stream_receiver, 1024), expected_output)
186 self.assertEqual(_readall(stream_receiver, 1024), expected_output)
212
187
213 self._test(
188 self._test(
214 TEST_BUFFERING_CHILD_SCRIPT.format(stream=stream),
189 TEST_BUFFERING_CHILD_SCRIPT.format(stream=stream),
215 stream,
190 stream,
216 rwpair_generator,
191 rwpair_generator,
217 check_output,
192 check_output,
218 python_args,
193 python_args,
219 )
194 )
220
195
221 def test_buffering_stdout_devnull(self):
196 def test_buffering_stdout_devnull(self):
222 self._test_buffering('stdout', _devnull, None)
197 self._test_buffering('stdout', _devnull, None)
223
198
224 def test_buffering_stdout_pipes(self):
199 def test_buffering_stdout_pipes(self):
225 self._test_buffering('stdout', _pipes, FULLY_BUFFERED)
200 self._test_buffering('stdout', _pipes, FULLY_BUFFERED)
226
201
227 def test_buffering_stdout_ptys(self):
202 def test_buffering_stdout_ptys(self):
228 self._test_buffering('stdout', _ptys, LINE_BUFFERED)
203 self._test_buffering('stdout', _ptys, LINE_BUFFERED)
229
204
230 def test_buffering_stdout_devnull_unbuffered(self):
205 def test_buffering_stdout_devnull_unbuffered(self):
231 self._test_buffering('stdout', _devnull, None, python_args=['-u'])
206 self._test_buffering('stdout', _devnull, None, python_args=['-u'])
232
207
233 def test_buffering_stdout_pipes_unbuffered(self):
208 def test_buffering_stdout_pipes_unbuffered(self):
234 self._test_buffering('stdout', _pipes, UNBUFFERED, python_args=['-u'])
209 self._test_buffering('stdout', _pipes, UNBUFFERED, python_args=['-u'])
235
210
236 def test_buffering_stdout_ptys_unbuffered(self):
211 def test_buffering_stdout_ptys_unbuffered(self):
237 self._test_buffering('stdout', _ptys, UNBUFFERED, python_args=['-u'])
212 self._test_buffering('stdout', _ptys, UNBUFFERED, python_args=['-u'])
238
213
239 if not pycompat.ispy3 and not pycompat.iswindows:
214 if not pycompat.ispy3 and not pycompat.iswindows:
240 # On Python 2 on non-Windows, we manually open stdout in line-buffered
215 # On Python 2 on non-Windows, we manually open stdout in line-buffered
241 # mode if connected to a TTY. We should check if Python was configured
216 # mode if connected to a TTY. We should check if Python was configured
242 # to use unbuffered stdout, but it's hard to do that.
217 # to use unbuffered stdout, but it's hard to do that.
243 test_buffering_stdout_ptys_unbuffered = unittest.expectedFailure(
218 test_buffering_stdout_ptys_unbuffered = unittest.expectedFailure(
244 test_buffering_stdout_ptys_unbuffered
219 test_buffering_stdout_ptys_unbuffered
245 )
220 )
246
221
247 def _test_large_write(self, stream, rwpair_generator, python_args=[]):
222 def _test_large_write(self, stream, rwpair_generator, python_args=[]):
248 if not pycompat.ispy3 and pycompat.isdarwin:
223 if not pycompat.ispy3 and pycompat.isdarwin:
249 # Python 2 doesn't always retry on EINTR, but the libc might retry.
224 # Python 2 doesn't always retry on EINTR, but the libc might retry.
250 # So far, it was observed only on macOS that EINTR is raised at the
225 # So far, it was observed only on macOS that EINTR is raised at the
251 # Python level. As Python 2 support will be dropped soon-ish, we
226 # Python level. As Python 2 support will be dropped soon-ish, we
252 # won't attempt to fix it.
227 # won't attempt to fix it.
253 raise unittest.SkipTest("raises EINTR on macOS")
228 raise unittest.SkipTest("raises EINTR on macOS")
254
229
255 def check_output(stream_receiver, proc):
230 def check_output(stream_receiver, proc):
256 if not pycompat.iswindows:
231 if not pycompat.iswindows:
257 # On Unix, we can provoke a partial write() by interrupting it
232 # On Unix, we can provoke a partial write() by interrupting it
258 # by a signal handler as soon as a bit of data was written.
233 # by a signal handler as soon as a bit of data was written.
259 # We test that write() is called until all data is written.
234 # We test that write() is called until all data is written.
260 buf = [os.read(stream_receiver, 1)]
235 buf = [os.read(stream_receiver, 1)]
261 proc.send_signal(signal.SIGINT)
236 proc.send_signal(signal.SIGINT)
262 else:
237 else:
263 # On Windows, there doesn't seem to be a way to cause partial
238 # On Windows, there doesn't seem to be a way to cause partial
264 # writes.
239 # writes.
265 buf = []
240 buf = []
266 self.assertEqual(
241 self.assertEqual(
267 _readall(stream_receiver, 131072, buf), b'x' * 1048576
242 _readall(stream_receiver, 131072, buf), b'x' * 1048576
268 )
243 )
269
244
270 def post_child_check():
245 def post_child_check():
271 write_result_str = write_result_f.read()
246 write_result_str = write_result_f.read()
272 if pycompat.ispy3:
247 if pycompat.ispy3:
273 # On Python 3, we test that the correct number of bytes is
248 # On Python 3, we test that the correct number of bytes is
274 # claimed to have been written.
249 # claimed to have been written.
275 expected_write_result_str = '1048576'
250 expected_write_result_str = '1048576'
276 else:
251 else:
277 # On Python 2, we only check that the large write does not
252 # On Python 2, we only check that the large write does not
278 # crash.
253 # crash.
279 expected_write_result_str = 'None'
254 expected_write_result_str = 'None'
280 self.assertEqual(write_result_str, expected_write_result_str)
255 self.assertEqual(write_result_str, expected_write_result_str)
281
256
282 with tempfile.NamedTemporaryFile('r') as write_result_f:
257 with tempfile.NamedTemporaryFile('r') as write_result_f:
283 self._test(
258 self._test(
284 TEST_LARGE_WRITE_CHILD_SCRIPT.format(
259 TEST_LARGE_WRITE_CHILD_SCRIPT.format(
285 stream=stream, write_result_fn=write_result_f.name
260 stream=stream, write_result_fn=write_result_f.name
286 ),
261 ),
287 stream,
262 stream,
288 rwpair_generator,
263 rwpair_generator,
289 check_output,
264 check_output,
290 python_args,
265 python_args,
291 post_child_check=post_child_check,
266 post_child_check=post_child_check,
292 )
267 )
293
268
294 def test_large_write_stdout_devnull(self):
269 def test_large_write_stdout_devnull(self):
295 self._test_large_write('stdout', _devnull)
270 self._test_large_write('stdout', _devnull)
296
271
297 def test_large_write_stdout_pipes(self):
272 def test_large_write_stdout_pipes(self):
298 self._test_large_write('stdout', _pipes)
273 self._test_large_write('stdout', _pipes)
299
274
300 def test_large_write_stdout_ptys(self):
275 def test_large_write_stdout_ptys(self):
301 self._test_large_write('stdout', _ptys)
276 self._test_large_write('stdout', _ptys)
302
277
303 def test_large_write_stdout_devnull_unbuffered(self):
278 def test_large_write_stdout_devnull_unbuffered(self):
304 self._test_large_write('stdout', _devnull, python_args=['-u'])
279 self._test_large_write('stdout', _devnull, python_args=['-u'])
305
280
306 def test_large_write_stdout_pipes_unbuffered(self):
281 def test_large_write_stdout_pipes_unbuffered(self):
307 self._test_large_write('stdout', _pipes, python_args=['-u'])
282 self._test_large_write('stdout', _pipes, python_args=['-u'])
308
283
309 def test_large_write_stdout_ptys_unbuffered(self):
284 def test_large_write_stdout_ptys_unbuffered(self):
310 self._test_large_write('stdout', _ptys, python_args=['-u'])
285 self._test_large_write('stdout', _ptys, python_args=['-u'])
311
286
312 def test_large_write_stderr_devnull(self):
287 def test_large_write_stderr_devnull(self):
313 self._test_large_write('stderr', _devnull)
288 self._test_large_write('stderr', _devnull)
314
289
315 def test_large_write_stderr_pipes(self):
290 def test_large_write_stderr_pipes(self):
316 self._test_large_write('stderr', _pipes)
291 self._test_large_write('stderr', _pipes)
317
292
318 def test_large_write_stderr_ptys(self):
293 def test_large_write_stderr_ptys(self):
319 self._test_large_write('stderr', _ptys)
294 self._test_large_write('stderr', _ptys)
320
295
321 def test_large_write_stderr_devnull_unbuffered(self):
296 def test_large_write_stderr_devnull_unbuffered(self):
322 self._test_large_write('stderr', _devnull, python_args=['-u'])
297 self._test_large_write('stderr', _devnull, python_args=['-u'])
323
298
324 def test_large_write_stderr_pipes_unbuffered(self):
299 def test_large_write_stderr_pipes_unbuffered(self):
325 self._test_large_write('stderr', _pipes, python_args=['-u'])
300 self._test_large_write('stderr', _pipes, python_args=['-u'])
326
301
327 def test_large_write_stderr_ptys_unbuffered(self):
302 def test_large_write_stderr_ptys_unbuffered(self):
328 self._test_large_write('stderr', _ptys, python_args=['-u'])
303 self._test_large_write('stderr', _ptys, python_args=['-u'])
329
304
330 def _test_broken_pipe(self, stream):
305 def _test_broken_pipe(self, stream):
331 assert stream in ('stdout', 'stderr')
306 assert stream in ('stdout', 'stderr')
332
307
333 def check_output(stream_receiver, proc):
308 def check_output(stream_receiver, proc):
334 os.close(stream_receiver)
309 os.close(stream_receiver)
335 proc.stdin.write(b'x')
310 proc.stdin.write(b'x')
336 proc.stdin.close()
311 proc.stdin.close()
337
312
338 def post_child_check():
313 def post_child_check():
339 err = pickle.load(err_f)
314 err = pickle.load(err_f)
340 self.assertEqual(err.errno, errno.EPIPE)
315 self.assertEqual(err.errno, errno.EPIPE)
341 self.assertEqual(err.strerror, "Broken pipe")
316 self.assertEqual(err.strerror, "Broken pipe")
342
317
343 with tempfile.NamedTemporaryFile('rb') as err_f:
318 with tempfile.NamedTemporaryFile('rb') as err_f:
344 self._test(
319 self._test(
345 TEST_BROKEN_PIPE_CHILD_SCRIPT.format(
320 TEST_BROKEN_PIPE_CHILD_SCRIPT.format(
346 stream=stream, err_fn=err_f.name
321 stream=stream, err_fn=err_f.name
347 ),
322 ),
348 stream,
323 stream,
349 _pipes,
324 _pipes,
350 check_output,
325 check_output,
351 post_child_check=post_child_check,
326 post_child_check=post_child_check,
352 stdin_generator=util.nullcontextmanager(subprocess.PIPE),
327 stdin_generator=util.nullcontextmanager(subprocess.PIPE),
353 )
328 )
354
329
355 def test_broken_pipe_stdout(self):
330 def test_broken_pipe_stdout(self):
356 self._test_broken_pipe('stdout')
331 self._test_broken_pipe('stdout')
357
332
358 def test_broken_pipe_stderr(self):
333 def test_broken_pipe_stderr(self):
359 self._test_broken_pipe('stderr')
334 self._test_broken_pipe('stderr')
360
335
361
336
362 if __name__ == '__main__':
337 if __name__ == '__main__':
363 import silenttestrunner
338 import silenttestrunner
364
339
365 silenttestrunner.main(__name__)
340 silenttestrunner.main(__name__)
General Comments 0
You need to be logged in to leave comments. Login now