##// END OF EJS Templates
Merge branch 'win32-shlex'
Thomas Kluyver -
r5539:d1997d96 merge
parent child Browse files
Show More
@@ -1,145 +1,170
1 """Common utilities for the various process_* implementations.
1 """Common utilities for the various process_* implementations.
2
2
3 This file is only meant to be imported by the platform-specific implementations
3 This file is only meant to be imported by the platform-specific implementations
4 of subprocess utilities, and it contains tools that are common to all of them.
4 of subprocess utilities, and it contains tools that are common to all of them.
5 """
5 """
6
6
7 #-----------------------------------------------------------------------------
7 #-----------------------------------------------------------------------------
8 # Copyright (C) 2010-2011 The IPython Development Team
8 # Copyright (C) 2010-2011 The IPython Development Team
9 #
9 #
10 # Distributed under the terms of the BSD License. The full license is in
10 # Distributed under the terms of the BSD License. The full license is in
11 # the file COPYING, distributed as part of this software.
11 # the file COPYING, distributed as part of this software.
12 #-----------------------------------------------------------------------------
12 #-----------------------------------------------------------------------------
13
13
14 #-----------------------------------------------------------------------------
14 #-----------------------------------------------------------------------------
15 # Imports
15 # Imports
16 #-----------------------------------------------------------------------------
16 #-----------------------------------------------------------------------------
17 import subprocess
17 import subprocess
18 import shlex
18 import sys
19 import sys
19
20
20 from IPython.utils import py3compat
21 from IPython.utils import py3compat
21
22
22 #-----------------------------------------------------------------------------
23 #-----------------------------------------------------------------------------
23 # Function definitions
24 # Function definitions
24 #-----------------------------------------------------------------------------
25 #-----------------------------------------------------------------------------
25
26
26 def read_no_interrupt(p):
27 def read_no_interrupt(p):
27 """Read from a pipe ignoring EINTR errors.
28 """Read from a pipe ignoring EINTR errors.
28
29
29 This is necessary because when reading from pipes with GUI event loops
30 This is necessary because when reading from pipes with GUI event loops
30 running in the background, often interrupts are raised that stop the
31 running in the background, often interrupts are raised that stop the
31 command from completing."""
32 command from completing."""
32 import errno
33 import errno
33
34
34 try:
35 try:
35 return p.read()
36 return p.read()
36 except IOError, err:
37 except IOError, err:
37 if err.errno != errno.EINTR:
38 if err.errno != errno.EINTR:
38 raise
39 raise
39
40
40
41
41 def process_handler(cmd, callback, stderr=subprocess.PIPE):
42 def process_handler(cmd, callback, stderr=subprocess.PIPE):
42 """Open a command in a shell subprocess and execute a callback.
43 """Open a command in a shell subprocess and execute a callback.
43
44
44 This function provides common scaffolding for creating subprocess.Popen()
45 This function provides common scaffolding for creating subprocess.Popen()
45 calls. It creates a Popen object and then calls the callback with it.
46 calls. It creates a Popen object and then calls the callback with it.
46
47
47 Parameters
48 Parameters
48 ----------
49 ----------
49 cmd : str
50 cmd : str
50 A string to be executed with the underlying system shell (by calling
51 A string to be executed with the underlying system shell (by calling
51 :func:`Popen` with ``shell=True``.
52 :func:`Popen` with ``shell=True``.
52
53
53 callback : callable
54 callback : callable
54 A one-argument function that will be called with the Popen object.
55 A one-argument function that will be called with the Popen object.
55
56
56 stderr : file descriptor number, optional
57 stderr : file descriptor number, optional
57 By default this is set to ``subprocess.PIPE``, but you can also pass the
58 By default this is set to ``subprocess.PIPE``, but you can also pass the
58 value ``subprocess.STDOUT`` to force the subprocess' stderr to go into
59 value ``subprocess.STDOUT`` to force the subprocess' stderr to go into
59 the same file descriptor as its stdout. This is useful to read stdout
60 the same file descriptor as its stdout. This is useful to read stdout
60 and stderr combined in the order they are generated.
61 and stderr combined in the order they are generated.
61
62
62 Returns
63 Returns
63 -------
64 -------
64 The return value of the provided callback is returned.
65 The return value of the provided callback is returned.
65 """
66 """
66 sys.stdout.flush()
67 sys.stdout.flush()
67 sys.stderr.flush()
68 sys.stderr.flush()
68 # On win32, close_fds can't be true when using pipes for stdin/out/err
69 # On win32, close_fds can't be true when using pipes for stdin/out/err
69 close_fds = sys.platform != 'win32'
70 close_fds = sys.platform != 'win32'
70 p = subprocess.Popen(cmd, shell=True,
71 p = subprocess.Popen(cmd, shell=True,
71 stdin=subprocess.PIPE,
72 stdin=subprocess.PIPE,
72 stdout=subprocess.PIPE,
73 stdout=subprocess.PIPE,
73 stderr=stderr,
74 stderr=stderr,
74 close_fds=close_fds)
75 close_fds=close_fds)
75
76
76 try:
77 try:
77 out = callback(p)
78 out = callback(p)
78 except KeyboardInterrupt:
79 except KeyboardInterrupt:
79 print('^C')
80 print('^C')
80 sys.stdout.flush()
81 sys.stdout.flush()
81 sys.stderr.flush()
82 sys.stderr.flush()
82 out = None
83 out = None
83 finally:
84 finally:
84 # Make really sure that we don't leave processes behind, in case the
85 # Make really sure that we don't leave processes behind, in case the
85 # call above raises an exception
86 # call above raises an exception
86 # We start by assuming the subprocess finished (to avoid NameErrors
87 # We start by assuming the subprocess finished (to avoid NameErrors
87 # later depending on the path taken)
88 # later depending on the path taken)
88 if p.returncode is None:
89 if p.returncode is None:
89 try:
90 try:
90 p.terminate()
91 p.terminate()
91 p.poll()
92 p.poll()
92 except OSError:
93 except OSError:
93 pass
94 pass
94 # One last try on our way out
95 # One last try on our way out
95 if p.returncode is None:
96 if p.returncode is None:
96 try:
97 try:
97 p.kill()
98 p.kill()
98 except OSError:
99 except OSError:
99 pass
100 pass
100
101
101 return out
102 return out
102
103
103
104
104 def getoutput(cmd):
105 def getoutput(cmd):
105 """Return standard output of executing cmd in a shell.
106 """Return standard output of executing cmd in a shell.
106
107
107 Accepts the same arguments as os.system().
108 Accepts the same arguments as os.system().
108
109
109 Parameters
110 Parameters
110 ----------
111 ----------
111 cmd : str
112 cmd : str
112 A command to be executed in the system shell.
113 A command to be executed in the system shell.
113
114
114 Returns
115 Returns
115 -------
116 -------
116 stdout : str
117 stdout : str
117 """
118 """
118
119
119 out = process_handler(cmd, lambda p: p.communicate()[0], subprocess.STDOUT)
120 out = process_handler(cmd, lambda p: p.communicate()[0], subprocess.STDOUT)
120 if out is None:
121 if out is None:
121 return ''
122 return ''
122 return py3compat.bytes_to_str(out)
123 return py3compat.bytes_to_str(out)
123
124
124
125
125 def getoutputerror(cmd):
126 def getoutputerror(cmd):
126 """Return (standard output, standard error) of executing cmd in a shell.
127 """Return (standard output, standard error) of executing cmd in a shell.
127
128
128 Accepts the same arguments as os.system().
129 Accepts the same arguments as os.system().
129
130
130 Parameters
131 Parameters
131 ----------
132 ----------
132 cmd : str
133 cmd : str
133 A command to be executed in the system shell.
134 A command to be executed in the system shell.
134
135
135 Returns
136 Returns
136 -------
137 -------
137 stdout : str
138 stdout : str
138 stderr : str
139 stderr : str
139 """
140 """
140
141
141 out_err = process_handler(cmd, lambda p: p.communicate())
142 out_err = process_handler(cmd, lambda p: p.communicate())
142 if out_err is None:
143 if out_err is None:
143 return '', ''
144 return '', ''
144 out, err = out_err
145 out, err = out_err
145 return py3compat.bytes_to_str(out), py3compat.bytes_to_str(err)
146 return py3compat.bytes_to_str(out), py3compat.bytes_to_str(err)
147
148
149 def arg_split(s, posix=False):
150 """Split a command line's arguments in a shell-like manner.
151
152 This is a modified version of the standard library's shlex.split()
153 function, but with a default of posix=False for splitting, so that quotes
154 in inputs are respected."""
155
156 # Unfortunately, python's shlex module is buggy with unicode input:
157 # http://bugs.python.org/issue1170
158 # At least encoding the input when it's unicode seems to help, but there
159 # may be more problems lurking. Apparently this is fixed in python3.
160 is_unicode = False
161 if (not py3compat.PY3) and isinstance(s, unicode):
162 is_unicode = True
163 s = s.encode('utf-8')
164 lex = shlex.shlex(s, posix=posix)
165 lex.whitespace_split = True
166 tokens = list(lex)
167 if is_unicode:
168 # Convert the tokens back to unicode.
169 tokens = [x.decode('utf-8') for x in tokens]
170 return tokens
@@ -1,194 +1,197
1 """Posix-specific implementation of process utilities.
1 """Posix-specific implementation of process utilities.
2
2
3 This file is only meant to be imported by process.py, not by end-users.
3 This file is only meant to be imported by process.py, not by end-users.
4 """
4 """
5
5
6 #-----------------------------------------------------------------------------
6 #-----------------------------------------------------------------------------
7 # Copyright (C) 2010-2011 The IPython Development Team
7 # Copyright (C) 2010-2011 The IPython Development Team
8 #
8 #
9 # Distributed under the terms of the BSD License. The full license is in
9 # Distributed under the terms of the BSD License. The full license is in
10 # the file COPYING, distributed as part of this software.
10 # the file COPYING, distributed as part of this software.
11 #-----------------------------------------------------------------------------
11 #-----------------------------------------------------------------------------
12
12
13 #-----------------------------------------------------------------------------
13 #-----------------------------------------------------------------------------
14 # Imports
14 # Imports
15 #-----------------------------------------------------------------------------
15 #-----------------------------------------------------------------------------
16 from __future__ import print_function
16 from __future__ import print_function
17
17
18 # Stdlib
18 # Stdlib
19 import subprocess as sp
19 import subprocess as sp
20 import sys
20 import sys
21
21
22 from IPython.external import pexpect
22 from IPython.external import pexpect
23
23
24 # Our own
24 # Our own
25 from .autoattr import auto_attr
25 from .autoattr import auto_attr
26 from ._process_common import getoutput
26 from ._process_common import getoutput, arg_split
27 from IPython.utils import text
27 from IPython.utils import text
28 from IPython.utils import py3compat
28 from IPython.utils import py3compat
29
29
30 #-----------------------------------------------------------------------------
30 #-----------------------------------------------------------------------------
31 # Function definitions
31 # Function definitions
32 #-----------------------------------------------------------------------------
32 #-----------------------------------------------------------------------------
33
33
34 def _find_cmd(cmd):
34 def _find_cmd(cmd):
35 """Find the full path to a command using which."""
35 """Find the full path to a command using which."""
36
36
37 path = sp.Popen(['/usr/bin/env', 'which', cmd],
37 path = sp.Popen(['/usr/bin/env', 'which', cmd],
38 stdout=sp.PIPE).communicate()[0]
38 stdout=sp.PIPE).communicate()[0]
39 return py3compat.bytes_to_str(path)
39 return py3compat.bytes_to_str(path)
40
40
41
41
42 class ProcessHandler(object):
42 class ProcessHandler(object):
43 """Execute subprocesses under the control of pexpect.
43 """Execute subprocesses under the control of pexpect.
44 """
44 """
45 # Timeout in seconds to wait on each reading of the subprocess' output.
45 # Timeout in seconds to wait on each reading of the subprocess' output.
46 # This should not be set too low to avoid cpu overusage from our side,
46 # This should not be set too low to avoid cpu overusage from our side,
47 # since we read in a loop whose period is controlled by this timeout.
47 # since we read in a loop whose period is controlled by this timeout.
48 read_timeout = 0.05
48 read_timeout = 0.05
49
49
50 # Timeout to give a process if we receive SIGINT, between sending the
50 # Timeout to give a process if we receive SIGINT, between sending the
51 # SIGINT to the process and forcefully terminating it.
51 # SIGINT to the process and forcefully terminating it.
52 terminate_timeout = 0.2
52 terminate_timeout = 0.2
53
53
54 # File object where stdout and stderr of the subprocess will be written
54 # File object where stdout and stderr of the subprocess will be written
55 logfile = None
55 logfile = None
56
56
57 # Shell to call for subprocesses to execute
57 # Shell to call for subprocesses to execute
58 sh = None
58 sh = None
59
59
60 @auto_attr
60 @auto_attr
61 def sh(self):
61 def sh(self):
62 sh = pexpect.which('sh')
62 sh = pexpect.which('sh')
63 if sh is None:
63 if sh is None:
64 raise OSError('"sh" shell not found')
64 raise OSError('"sh" shell not found')
65 return sh
65 return sh
66
66
67 def __init__(self, logfile=None, read_timeout=None, terminate_timeout=None):
67 def __init__(self, logfile=None, read_timeout=None, terminate_timeout=None):
68 """Arguments are used for pexpect calls."""
68 """Arguments are used for pexpect calls."""
69 self.read_timeout = (ProcessHandler.read_timeout if read_timeout is
69 self.read_timeout = (ProcessHandler.read_timeout if read_timeout is
70 None else read_timeout)
70 None else read_timeout)
71 self.terminate_timeout = (ProcessHandler.terminate_timeout if
71 self.terminate_timeout = (ProcessHandler.terminate_timeout if
72 terminate_timeout is None else
72 terminate_timeout is None else
73 terminate_timeout)
73 terminate_timeout)
74 self.logfile = sys.stdout if logfile is None else logfile
74 self.logfile = sys.stdout if logfile is None else logfile
75
75
76 def getoutput(self, cmd):
76 def getoutput(self, cmd):
77 """Run a command and return its stdout/stderr as a string.
77 """Run a command and return its stdout/stderr as a string.
78
78
79 Parameters
79 Parameters
80 ----------
80 ----------
81 cmd : str
81 cmd : str
82 A command to be executed in the system shell.
82 A command to be executed in the system shell.
83
83
84 Returns
84 Returns
85 -------
85 -------
86 output : str
86 output : str
87 A string containing the combination of stdout and stderr from the
87 A string containing the combination of stdout and stderr from the
88 subprocess, in whatever order the subprocess originally wrote to its
88 subprocess, in whatever order the subprocess originally wrote to its
89 file descriptors (so the order of the information in this string is the
89 file descriptors (so the order of the information in this string is the
90 correct order as would be seen if running the command in a terminal).
90 correct order as would be seen if running the command in a terminal).
91 """
91 """
92 try:
92 try:
93 return pexpect.run(self.sh, args=['-c', cmd]).replace('\r\n', '\n')
93 return pexpect.run(self.sh, args=['-c', cmd]).replace('\r\n', '\n')
94 except KeyboardInterrupt:
94 except KeyboardInterrupt:
95 print('^C', file=sys.stderr, end='')
95 print('^C', file=sys.stderr, end='')
96
96
97 def getoutput_pexpect(self, cmd):
97 def getoutput_pexpect(self, cmd):
98 """Run a command and return its stdout/stderr as a string.
98 """Run a command and return its stdout/stderr as a string.
99
99
100 Parameters
100 Parameters
101 ----------
101 ----------
102 cmd : str
102 cmd : str
103 A command to be executed in the system shell.
103 A command to be executed in the system shell.
104
104
105 Returns
105 Returns
106 -------
106 -------
107 output : str
107 output : str
108 A string containing the combination of stdout and stderr from the
108 A string containing the combination of stdout and stderr from the
109 subprocess, in whatever order the subprocess originally wrote to its
109 subprocess, in whatever order the subprocess originally wrote to its
110 file descriptors (so the order of the information in this string is the
110 file descriptors (so the order of the information in this string is the
111 correct order as would be seen if running the command in a terminal).
111 correct order as would be seen if running the command in a terminal).
112 """
112 """
113 try:
113 try:
114 return pexpect.run(self.sh, args=['-c', cmd]).replace('\r\n', '\n')
114 return pexpect.run(self.sh, args=['-c', cmd]).replace('\r\n', '\n')
115 except KeyboardInterrupt:
115 except KeyboardInterrupt:
116 print('^C', file=sys.stderr, end='')
116 print('^C', file=sys.stderr, end='')
117
117
118 def system(self, cmd):
118 def system(self, cmd):
119 """Execute a command in a subshell.
119 """Execute a command in a subshell.
120
120
121 Parameters
121 Parameters
122 ----------
122 ----------
123 cmd : str
123 cmd : str
124 A command to be executed in the system shell.
124 A command to be executed in the system shell.
125
125
126 Returns
126 Returns
127 -------
127 -------
128 int : child's exitstatus
128 int : child's exitstatus
129 """
129 """
130 # Get likely encoding for the output.
130 # Get likely encoding for the output.
131 enc = text.getdefaultencoding()
131 enc = text.getdefaultencoding()
132
132
133 # Patterns to match on the output, for pexpect. We read input and
133 # Patterns to match on the output, for pexpect. We read input and
134 # allow either a short timeout or EOF
134 # allow either a short timeout or EOF
135 patterns = [pexpect.TIMEOUT, pexpect.EOF]
135 patterns = [pexpect.TIMEOUT, pexpect.EOF]
136 # the index of the EOF pattern in the list.
136 # the index of the EOF pattern in the list.
137 # even though we know it's 1, this call means we don't have to worry if
137 # even though we know it's 1, this call means we don't have to worry if
138 # we change the above list, and forget to change this value:
138 # we change the above list, and forget to change this value:
139 EOF_index = patterns.index(pexpect.EOF)
139 EOF_index = patterns.index(pexpect.EOF)
140 # The size of the output stored so far in the process output buffer.
140 # The size of the output stored so far in the process output buffer.
141 # Since pexpect only appends to this buffer, each time we print we
141 # Since pexpect only appends to this buffer, each time we print we
142 # record how far we've printed, so that next time we only print *new*
142 # record how far we've printed, so that next time we only print *new*
143 # content from the buffer.
143 # content from the buffer.
144 out_size = 0
144 out_size = 0
145 try:
145 try:
146 # Since we're not really searching the buffer for text patterns, we
146 # Since we're not really searching the buffer for text patterns, we
147 # can set pexpect's search window to be tiny and it won't matter.
147 # can set pexpect's search window to be tiny and it won't matter.
148 # We only search for the 'patterns' timeout or EOF, which aren't in
148 # We only search for the 'patterns' timeout or EOF, which aren't in
149 # the text itself.
149 # the text itself.
150 #child = pexpect.spawn(pcmd, searchwindowsize=1)
150 #child = pexpect.spawn(pcmd, searchwindowsize=1)
151 if hasattr(pexpect, 'spawnb'):
151 if hasattr(pexpect, 'spawnb'):
152 child = pexpect.spawnb(self.sh, args=['-c', cmd]) # Pexpect-U
152 child = pexpect.spawnb(self.sh, args=['-c', cmd]) # Pexpect-U
153 else:
153 else:
154 child = pexpect.spawn(self.sh, args=['-c', cmd]) # Vanilla Pexpect
154 child = pexpect.spawn(self.sh, args=['-c', cmd]) # Vanilla Pexpect
155 flush = sys.stdout.flush
155 flush = sys.stdout.flush
156 while True:
156 while True:
157 # res is the index of the pattern that caused the match, so we
157 # res is the index of the pattern that caused the match, so we
158 # know whether we've finished (if we matched EOF) or not
158 # know whether we've finished (if we matched EOF) or not
159 res_idx = child.expect_list(patterns, self.read_timeout)
159 res_idx = child.expect_list(patterns, self.read_timeout)
160 print(child.before[out_size:].decode(enc, 'replace'), end='')
160 print(child.before[out_size:].decode(enc, 'replace'), end='')
161 flush()
161 flush()
162 if res_idx==EOF_index:
162 if res_idx==EOF_index:
163 break
163 break
164 # Update the pointer to what we've already printed
164 # Update the pointer to what we've already printed
165 out_size = len(child.before)
165 out_size = len(child.before)
166 except KeyboardInterrupt:
166 except KeyboardInterrupt:
167 # We need to send ^C to the process. The ascii code for '^C' is 3
167 # We need to send ^C to the process. The ascii code for '^C' is 3
168 # (the character is known as ETX for 'End of Text', see
168 # (the character is known as ETX for 'End of Text', see
169 # curses.ascii.ETX).
169 # curses.ascii.ETX).
170 child.sendline(chr(3))
170 child.sendline(chr(3))
171 # Read and print any more output the program might produce on its
171 # Read and print any more output the program might produce on its
172 # way out.
172 # way out.
173 try:
173 try:
174 out_size = len(child.before)
174 out_size = len(child.before)
175 child.expect_list(patterns, self.terminate_timeout)
175 child.expect_list(patterns, self.terminate_timeout)
176 print(child.before[out_size:].decode(enc, 'replace'), end='')
176 print(child.before[out_size:].decode(enc, 'replace'), end='')
177 sys.stdout.flush()
177 sys.stdout.flush()
178 except KeyboardInterrupt:
178 except KeyboardInterrupt:
179 # Impatient users tend to type it multiple times
179 # Impatient users tend to type it multiple times
180 pass
180 pass
181 finally:
181 finally:
182 # Ensure the subprocess really is terminated
182 # Ensure the subprocess really is terminated
183 child.terminate(force=True)
183 child.terminate(force=True)
184 # add isalive check, to ensure exitstatus is set:
184 # add isalive check, to ensure exitstatus is set:
185 child.isalive()
185 child.isalive()
186 return child.exitstatus
186 return child.exitstatus
187
187
188
188
189 # Make system() with a functional interface for outside use. Note that we use
189 # Make system() with a functional interface for outside use. Note that we use
190 # getoutput() from the _common utils, which is built on top of popen(). Using
190 # getoutput() from the _common utils, which is built on top of popen(). Using
191 # pexpect to get subprocess output produces difficult to parse output, since
191 # pexpect to get subprocess output produces difficult to parse output, since
192 # programs think they are talking to a tty and produce highly formatted output
192 # programs think they are talking to a tty and produce highly formatted output
193 # (ls is a good example) that makes them hard.
193 # (ls is a good example) that makes them hard.
194 system = ProcessHandler().system
194 system = ProcessHandler().system
195
196
197
@@ -1,148 +1,178
1 """Windows-specific implementation of process utilities.
1 """Windows-specific implementation of process utilities.
2
2
3 This file is only meant to be imported by process.py, not by end-users.
3 This file is only meant to be imported by process.py, not by end-users.
4 """
4 """
5
5
6 #-----------------------------------------------------------------------------
6 #-----------------------------------------------------------------------------
7 # Copyright (C) 2010-2011 The IPython Development Team
7 # Copyright (C) 2010-2011 The IPython Development Team
8 #
8 #
9 # Distributed under the terms of the BSD License. The full license is in
9 # Distributed under the terms of the BSD License. The full license is in
10 # the file COPYING, distributed as part of this software.
10 # the file COPYING, distributed as part of this software.
11 #-----------------------------------------------------------------------------
11 #-----------------------------------------------------------------------------
12
12
13 #-----------------------------------------------------------------------------
13 #-----------------------------------------------------------------------------
14 # Imports
14 # Imports
15 #-----------------------------------------------------------------------------
15 #-----------------------------------------------------------------------------
16 from __future__ import print_function
16 from __future__ import print_function
17
17
18 # stdlib
18 # stdlib
19 import os
19 import os
20 import sys
20 import sys
21 import ctypes
21
22
23 from ctypes import c_int, POINTER
24 from ctypes.wintypes import LPCWSTR, HLOCAL
22 from subprocess import STDOUT
25 from subprocess import STDOUT
23
26
24 # our own imports
27 # our own imports
25 from ._process_common import read_no_interrupt, process_handler
28 from ._process_common import read_no_interrupt, process_handler
29 from . import py3compat
26 from . import text
30 from . import text
27
31
28 #-----------------------------------------------------------------------------
32 #-----------------------------------------------------------------------------
29 # Function definitions
33 # Function definitions
30 #-----------------------------------------------------------------------------
34 #-----------------------------------------------------------------------------
31
35
32 class AvoidUNCPath(object):
36 class AvoidUNCPath(object):
33 """A context manager to protect command execution from UNC paths.
37 """A context manager to protect command execution from UNC paths.
34
38
35 In the Win32 API, commands can't be invoked with the cwd being a UNC path.
39 In the Win32 API, commands can't be invoked with the cwd being a UNC path.
36 This context manager temporarily changes directory to the 'C:' drive on
40 This context manager temporarily changes directory to the 'C:' drive on
37 entering, and restores the original working directory on exit.
41 entering, and restores the original working directory on exit.
38
42
39 The context manager returns the starting working directory *if* it made a
43 The context manager returns the starting working directory *if* it made a
40 change and None otherwise, so that users can apply the necessary adjustment
44 change and None otherwise, so that users can apply the necessary adjustment
41 to their system calls in the event of a change.
45 to their system calls in the event of a change.
42
46
43 Example
47 Example
44 -------
48 -------
45 ::
49 ::
46 cmd = 'dir'
50 cmd = 'dir'
47 with AvoidUNCPath() as path:
51 with AvoidUNCPath() as path:
48 if path is not None:
52 if path is not None:
49 cmd = '"pushd %s &&"%s' % (path, cmd)
53 cmd = '"pushd %s &&"%s' % (path, cmd)
50 os.system(cmd)
54 os.system(cmd)
51 """
55 """
52 def __enter__(self):
56 def __enter__(self):
53 self.path = os.getcwdu()
57 self.path = os.getcwdu()
54 self.is_unc_path = self.path.startswith(r"\\")
58 self.is_unc_path = self.path.startswith(r"\\")
55 if self.is_unc_path:
59 if self.is_unc_path:
56 # change to c drive (as cmd.exe cannot handle UNC addresses)
60 # change to c drive (as cmd.exe cannot handle UNC addresses)
57 os.chdir("C:")
61 os.chdir("C:")
58 return self.path
62 return self.path
59 else:
63 else:
60 # We return None to signal that there was no change in the working
64 # We return None to signal that there was no change in the working
61 # directory
65 # directory
62 return None
66 return None
63
67
64 def __exit__(self, exc_type, exc_value, traceback):
68 def __exit__(self, exc_type, exc_value, traceback):
65 if self.is_unc_path:
69 if self.is_unc_path:
66 os.chdir(self.path)
70 os.chdir(self.path)
67
71
68
72
69 def _find_cmd(cmd):
73 def _find_cmd(cmd):
70 """Find the full path to a .bat or .exe using the win32api module."""
74 """Find the full path to a .bat or .exe using the win32api module."""
71 try:
75 try:
72 from win32api import SearchPath
76 from win32api import SearchPath
73 except ImportError:
77 except ImportError:
74 raise ImportError('you need to have pywin32 installed for this to work')
78 raise ImportError('you need to have pywin32 installed for this to work')
75 else:
79 else:
76 PATH = os.environ['PATH']
80 PATH = os.environ['PATH']
77 extensions = ['.exe', '.com', '.bat', '.py']
81 extensions = ['.exe', '.com', '.bat', '.py']
78 path = None
82 path = None
79 for ext in extensions:
83 for ext in extensions:
80 try:
84 try:
81 path = SearchPath(PATH, cmd + ext)[0]
85 path = SearchPath(PATH, cmd + ext)[0]
82 except:
86 except:
83 pass
87 pass
84 if path is None:
88 if path is None:
85 raise OSError("command %r not found" % cmd)
89 raise OSError("command %r not found" % cmd)
86 else:
90 else:
87 return path
91 return path
88
92
89
93
90 def _system_body(p):
94 def _system_body(p):
91 """Callback for _system."""
95 """Callback for _system."""
92 enc = text.getdefaultencoding()
96 enc = text.getdefaultencoding()
93 for line in read_no_interrupt(p.stdout).splitlines():
97 for line in read_no_interrupt(p.stdout).splitlines():
94 line = line.decode(enc, 'replace')
98 line = line.decode(enc, 'replace')
95 print(line, file=sys.stdout)
99 print(line, file=sys.stdout)
96 for line in read_no_interrupt(p.stderr).splitlines():
100 for line in read_no_interrupt(p.stderr).splitlines():
97 line = line.decode(enc, 'replace')
101 line = line.decode(enc, 'replace')
98 print(line, file=sys.stderr)
102 print(line, file=sys.stderr)
99
103
100 # Wait to finish for returncode
104 # Wait to finish for returncode
101 return p.wait()
105 return p.wait()
102
106
103
107
104 def system(cmd):
108 def system(cmd):
105 """Win32 version of os.system() that works with network shares.
109 """Win32 version of os.system() that works with network shares.
106
110
107 Note that this implementation returns None, as meant for use in IPython.
111 Note that this implementation returns None, as meant for use in IPython.
108
112
109 Parameters
113 Parameters
110 ----------
114 ----------
111 cmd : str
115 cmd : str
112 A command to be executed in the system shell.
116 A command to be executed in the system shell.
113
117
114 Returns
118 Returns
115 -------
119 -------
116 None : we explicitly do NOT return the subprocess status code, as this
120 None : we explicitly do NOT return the subprocess status code, as this
117 utility is meant to be used extensively in IPython, where any return value
121 utility is meant to be used extensively in IPython, where any return value
118 would trigger :func:`sys.displayhook` calls.
122 would trigger :func:`sys.displayhook` calls.
119 """
123 """
120 with AvoidUNCPath() as path:
124 with AvoidUNCPath() as path:
121 if path is not None:
125 if path is not None:
122 cmd = '"pushd %s &&"%s' % (path, cmd)
126 cmd = '"pushd %s &&"%s' % (path, cmd)
123 return process_handler(cmd, _system_body)
127 return process_handler(cmd, _system_body)
124
128
125
129
126 def getoutput(cmd):
130 def getoutput(cmd):
127 """Return standard output of executing cmd in a shell.
131 """Return standard output of executing cmd in a shell.
128
132
129 Accepts the same arguments as os.system().
133 Accepts the same arguments as os.system().
130
134
131 Parameters
135 Parameters
132 ----------
136 ----------
133 cmd : str
137 cmd : str
134 A command to be executed in the system shell.
138 A command to be executed in the system shell.
135
139
136 Returns
140 Returns
137 -------
141 -------
138 stdout : str
142 stdout : str
139 """
143 """
140
144
141 with AvoidUNCPath() as path:
145 with AvoidUNCPath() as path:
142 if path is not None:
146 if path is not None:
143 cmd = '"pushd %s &&"%s' % (path, cmd)
147 cmd = '"pushd %s &&"%s' % (path, cmd)
144 out = process_handler(cmd, lambda p: p.communicate()[0], STDOUT)
148 out = process_handler(cmd, lambda p: p.communicate()[0], STDOUT)
145
149
146 if out is None:
150 if out is None:
147 out = ''
151 out = ''
148 return out
152 return out
153
154 try:
155 CommandLineToArgvW = ctypes.windll.shell32.CommandLineToArgvW
156 CommandLineToArgvW.arg_types = [LPCWSTR, POINTER(c_int)]
157 CommandLineToArgvW.res_types = [POINTER(LPCWSTR)]
158 LocalFree = ctypes.windll.kernel32.LocalFree
159 LocalFree.res_type = HLOCAL
160 LocalFree.arg_types = [HLOCAL]
161
162 def arg_split(commandline, posix=False):
163 """Split a command line's arguments in a shell-like manner.
164
165 This is a special version for windows that use a ctypes call to CommandLineToArgvW
166 to do the argv splitting. The posix paramter is ignored.
167 """
168 #CommandLineToArgvW returns path to executable if called with empty string.
169 if commandline.strip() == "":
170 return []
171 argvn = c_int()
172 result_pointer = CommandLineToArgvW(py3compat.cast_unicode(commandline.lstrip()), ctypes.byref(argvn))
173 result_array_type = LPCWSTR * argvn.value
174 result = [arg for arg in result_array_type.from_address(result_pointer)]
175 retval = LocalFree(result_pointer)
176 return result
177 except AttributeError:
178 from ._process_common import arg_split
@@ -1,147 +1,123
1 # encoding: utf-8
1 # encoding: utf-8
2 """
2 """
3 Utilities for working with external processes.
3 Utilities for working with external processes.
4 """
4 """
5
5
6 #-----------------------------------------------------------------------------
6 #-----------------------------------------------------------------------------
7 # Copyright (C) 2008-2011 The IPython Development Team
7 # Copyright (C) 2008-2011 The IPython Development Team
8 #
8 #
9 # Distributed under the terms of the BSD License. The full license is in
9 # Distributed under the terms of the BSD License. The full license is in
10 # the file COPYING, distributed as part of this software.
10 # the file COPYING, distributed as part of this software.
11 #-----------------------------------------------------------------------------
11 #-----------------------------------------------------------------------------
12
12
13 #-----------------------------------------------------------------------------
13 #-----------------------------------------------------------------------------
14 # Imports
14 # Imports
15 #-----------------------------------------------------------------------------
15 #-----------------------------------------------------------------------------
16 from __future__ import print_function
16 from __future__ import print_function
17
17
18 # Stdlib
18 # Stdlib
19 import os
19 import os
20 import sys
20 import sys
21 import shlex
21 import shlex
22
22
23 # Our own
23 # Our own
24 if sys.platform == 'win32':
24 if sys.platform == 'win32':
25 from ._process_win32 import _find_cmd, system, getoutput, AvoidUNCPath
25 from ._process_win32 import _find_cmd, system, getoutput, AvoidUNCPath, arg_split
26 else:
26 else:
27 from ._process_posix import _find_cmd, system, getoutput
27 from ._process_posix import _find_cmd, system, getoutput, arg_split
28
28
29
29 from ._process_common import getoutputerror
30 from ._process_common import getoutputerror
30 from IPython.utils import py3compat
31 from IPython.utils import py3compat
31
32
32 #-----------------------------------------------------------------------------
33 #-----------------------------------------------------------------------------
33 # Code
34 # Code
34 #-----------------------------------------------------------------------------
35 #-----------------------------------------------------------------------------
35
36
36
37
37 class FindCmdError(Exception):
38 class FindCmdError(Exception):
38 pass
39 pass
39
40
40
41
41 def find_cmd(cmd):
42 def find_cmd(cmd):
42 """Find absolute path to executable cmd in a cross platform manner.
43 """Find absolute path to executable cmd in a cross platform manner.
43
44
44 This function tries to determine the full path to a command line program
45 This function tries to determine the full path to a command line program
45 using `which` on Unix/Linux/OS X and `win32api` on Windows. Most of the
46 using `which` on Unix/Linux/OS X and `win32api` on Windows. Most of the
46 time it will use the version that is first on the users `PATH`. If
47 time it will use the version that is first on the users `PATH`. If
47 cmd is `python` return `sys.executable`.
48 cmd is `python` return `sys.executable`.
48
49
49 Warning, don't use this to find IPython command line programs as there
50 Warning, don't use this to find IPython command line programs as there
50 is a risk you will find the wrong one. Instead find those using the
51 is a risk you will find the wrong one. Instead find those using the
51 following code and looking for the application itself::
52 following code and looking for the application itself::
52
53
53 from IPython.utils.path import get_ipython_module_path
54 from IPython.utils.path import get_ipython_module_path
54 from IPython.utils.process import pycmd2argv
55 from IPython.utils.process import pycmd2argv
55 argv = pycmd2argv(get_ipython_module_path('IPython.frontend.terminal.ipapp'))
56 argv = pycmd2argv(get_ipython_module_path('IPython.frontend.terminal.ipapp'))
56
57
57 Parameters
58 Parameters
58 ----------
59 ----------
59 cmd : str
60 cmd : str
60 The command line program to look for.
61 The command line program to look for.
61 """
62 """
62 if cmd == 'python':
63 if cmd == 'python':
63 return os.path.abspath(sys.executable)
64 return os.path.abspath(sys.executable)
64 try:
65 try:
65 path = _find_cmd(cmd).rstrip()
66 path = _find_cmd(cmd).rstrip()
66 except OSError:
67 except OSError:
67 raise FindCmdError('command could not be found: %s' % cmd)
68 raise FindCmdError('command could not be found: %s' % cmd)
68 # which returns empty if not found
69 # which returns empty if not found
69 if path == '':
70 if path == '':
70 raise FindCmdError('command could not be found: %s' % cmd)
71 raise FindCmdError('command could not be found: %s' % cmd)
71 return os.path.abspath(path)
72 return os.path.abspath(path)
72
73
73
74
74 def pycmd2argv(cmd):
75 def pycmd2argv(cmd):
75 r"""Take the path of a python command and return a list (argv-style).
76 r"""Take the path of a python command and return a list (argv-style).
76
77
77 This only works on Python based command line programs and will find the
78 This only works on Python based command line programs and will find the
78 location of the ``python`` executable using ``sys.executable`` to make
79 location of the ``python`` executable using ``sys.executable`` to make
79 sure the right version is used.
80 sure the right version is used.
80
81
81 For a given path ``cmd``, this returns [cmd] if cmd's extension is .exe,
82 For a given path ``cmd``, this returns [cmd] if cmd's extension is .exe,
82 .com or .bat, and [, cmd] otherwise.
83 .com or .bat, and [, cmd] otherwise.
83
84
84 Parameters
85 Parameters
85 ----------
86 ----------
86 cmd : string
87 cmd : string
87 The path of the command.
88 The path of the command.
88
89
89 Returns
90 Returns
90 -------
91 -------
91 argv-style list.
92 argv-style list.
92 """
93 """
93 ext = os.path.splitext(cmd)[1]
94 ext = os.path.splitext(cmd)[1]
94 if ext in ['.exe', '.com', '.bat']:
95 if ext in ['.exe', '.com', '.bat']:
95 return [cmd]
96 return [cmd]
96 else:
97 else:
97 if sys.platform == 'win32':
98 if sys.platform == 'win32':
98 # The -u option here turns on unbuffered output, which is required
99 # The -u option here turns on unbuffered output, which is required
99 # on Win32 to prevent wierd conflict and problems with Twisted.
100 # on Win32 to prevent wierd conflict and problems with Twisted.
100 # Also, use sys.executable to make sure we are picking up the
101 # Also, use sys.executable to make sure we are picking up the
101 # right python exe.
102 # right python exe.
102 return [sys.executable, '-u', cmd]
103 return [sys.executable, '-u', cmd]
103 else:
104 else:
104 return [sys.executable, cmd]
105 return [sys.executable, cmd]
105
106
106
107 def arg_split(s, posix=False):
108 """Split a command line's arguments in a shell-like manner.
109
110 This is a modified version of the standard library's shlex.split()
111 function, but with a default of posix=False for splitting, so that quotes
112 in inputs are respected."""
113
114 # Unfortunately, python's shlex module is buggy with unicode input:
115 # http://bugs.python.org/issue1170
116 # At least encoding the input when it's unicode seems to help, but there
117 # may be more problems lurking. Apparently this is fixed in python3.
118 is_unicode = False
119 if (not py3compat.PY3) and isinstance(s, unicode):
120 is_unicode = True
121 s = s.encode('utf-8')
122 lex = shlex.shlex(s, posix=posix)
123 lex.whitespace_split = True
124 tokens = list(lex)
125 if is_unicode:
126 # Convert the tokens back to unicode.
127 tokens = [x.decode('utf-8') for x in tokens]
128 return tokens
129
130
131 def abbrev_cwd():
107 def abbrev_cwd():
132 """ Return abbreviated version of cwd, e.g. d:mydir """
108 """ Return abbreviated version of cwd, e.g. d:mydir """
133 cwd = os.getcwdu().replace('\\','/')
109 cwd = os.getcwdu().replace('\\','/')
134 drivepart = ''
110 drivepart = ''
135 tail = cwd
111 tail = cwd
136 if sys.platform == 'win32':
112 if sys.platform == 'win32':
137 if len(cwd) < 4:
113 if len(cwd) < 4:
138 return cwd
114 return cwd
139 drivepart,tail = os.path.splitdrive(cwd)
115 drivepart,tail = os.path.splitdrive(cwd)
140
116
141
117
142 parts = tail.split('/')
118 parts = tail.split('/')
143 if len(parts) > 2:
119 if len(parts) > 2:
144 tail = '/'.join(parts[-2:])
120 tail = '/'.join(parts[-2:])
145
121
146 return (drivepart + (
122 return (drivepart + (
147 cwd == '/' and '/' or tail))
123 cwd == '/' and '/' or tail))
@@ -1,111 +1,131
1 # encoding: utf-8
1 # encoding: utf-8
2 """
2 """
3 Tests for platutils.py
3 Tests for platutils.py
4 """
4 """
5
5
6 #-----------------------------------------------------------------------------
6 #-----------------------------------------------------------------------------
7 # Copyright (C) 2008-2011 The IPython Development Team
7 # Copyright (C) 2008-2011 The IPython Development Team
8 #
8 #
9 # Distributed under the terms of the BSD License. The full license is in
9 # Distributed under the terms of the BSD License. The full license is in
10 # the file COPYING, distributed as part of this software.
10 # the file COPYING, distributed as part of this software.
11 #-----------------------------------------------------------------------------
11 #-----------------------------------------------------------------------------
12
12
13 #-----------------------------------------------------------------------------
13 #-----------------------------------------------------------------------------
14 # Imports
14 # Imports
15 #-----------------------------------------------------------------------------
15 #-----------------------------------------------------------------------------
16
16
17 import sys
17 import sys
18 from unittest import TestCase
18 from unittest import TestCase
19
19
20 import nose.tools as nt
20 import nose.tools as nt
21
21
22 from IPython.utils.process import (find_cmd, FindCmdError, arg_split,
22 from IPython.utils.process import (find_cmd, FindCmdError, arg_split,
23 system, getoutput, getoutputerror)
23 system, getoutput, getoutputerror)
24 from IPython.testing import decorators as dec
24 from IPython.testing import decorators as dec
25 from IPython.testing import tools as tt
25 from IPython.testing import tools as tt
26
26
27 #-----------------------------------------------------------------------------
27 #-----------------------------------------------------------------------------
28 # Tests
28 # Tests
29 #-----------------------------------------------------------------------------
29 #-----------------------------------------------------------------------------
30
30
31 def test_find_cmd_python():
31 def test_find_cmd_python():
32 """Make sure we find sys.exectable for python."""
32 """Make sure we find sys.exectable for python."""
33 nt.assert_equals(find_cmd('python'), sys.executable)
33 nt.assert_equals(find_cmd('python'), sys.executable)
34
34
35
35
36 @dec.skip_win32
36 @dec.skip_win32
37 def test_find_cmd_ls():
37 def test_find_cmd_ls():
38 """Make sure we can find the full path to ls."""
38 """Make sure we can find the full path to ls."""
39 path = find_cmd('ls')
39 path = find_cmd('ls')
40 nt.assert_true(path.endswith('ls'))
40 nt.assert_true(path.endswith('ls'))
41
41
42
42
43 def has_pywin32():
43 def has_pywin32():
44 try:
44 try:
45 import win32api
45 import win32api
46 except ImportError:
46 except ImportError:
47 return False
47 return False
48 return True
48 return True
49
49
50
50
51 @dec.onlyif(has_pywin32, "This test requires win32api to run")
51 @dec.onlyif(has_pywin32, "This test requires win32api to run")
52 def test_find_cmd_pythonw():
52 def test_find_cmd_pythonw():
53 """Try to find pythonw on Windows."""
53 """Try to find pythonw on Windows."""
54 path = find_cmd('pythonw')
54 path = find_cmd('pythonw')
55 nt.assert_true(path.endswith('pythonw.exe'))
55 nt.assert_true(path.endswith('pythonw.exe'))
56
56
57
57
58 @dec.onlyif(lambda : sys.platform != 'win32' or has_pywin32(),
58 @dec.onlyif(lambda : sys.platform != 'win32' or has_pywin32(),
59 "This test runs on posix or in win32 with win32api installed")
59 "This test runs on posix or in win32 with win32api installed")
60 def test_find_cmd_fail():
60 def test_find_cmd_fail():
61 """Make sure that FindCmdError is raised if we can't find the cmd."""
61 """Make sure that FindCmdError is raised if we can't find the cmd."""
62 nt.assert_raises(FindCmdError,find_cmd,'asdfasdf')
62 nt.assert_raises(FindCmdError,find_cmd,'asdfasdf')
63
63
64
64
65 @dec.skip_win32
65 def test_arg_split():
66 def test_arg_split():
66 """Ensure that argument lines are correctly split like in a shell."""
67 """Ensure that argument lines are correctly split like in a shell."""
67 tests = [['hi', ['hi']],
68 tests = [['hi', ['hi']],
68 [u'hi', [u'hi']],
69 [u'hi', [u'hi']],
69 ['hello there', ['hello', 'there']],
70 ['hello there', ['hello', 'there']],
70 [u'h\N{LATIN SMALL LETTER A WITH CARON}llo', [u'h\N{LATIN SMALL LETTER A WITH CARON}llo']],
71 # \u01ce == \N{LATIN SMALL LETTER A WITH CARON}
72 # Do not use \N because the tests crash with syntax error in
73 # some cases, for example windows python2.6.
74 [u'h\u01cello', [u'h\u01cello']],
71 ['something "with quotes"', ['something', '"with quotes"']],
75 ['something "with quotes"', ['something', '"with quotes"']],
72 ]
76 ]
73 for argstr, argv in tests:
77 for argstr, argv in tests:
74 nt.assert_equal(arg_split(argstr), argv)
78 nt.assert_equal(arg_split(argstr), argv)
79
80 @dec.skip_if_not_win32
81 def test_arg_split_win32():
82 """Ensure that argument lines are correctly split like in a shell."""
83 tests = [['hi', ['hi']],
84 [u'hi', [u'hi']],
85 ['hello there', ['hello', 'there']],
86 [u'h\u01cello', [u'h\u01cello']],
87 ['something "with quotes"', ['something', 'with quotes']],
88 ]
89 for argstr, argv in tests:
90 nt.assert_equal(arg_split(argstr), argv)
75
91
76
92
77 class SubProcessTestCase(TestCase, tt.TempFileMixin):
93 class SubProcessTestCase(TestCase, tt.TempFileMixin):
78 def setUp(self):
94 def setUp(self):
79 """Make a valid python temp file."""
95 """Make a valid python temp file."""
80 lines = ["from __future__ import print_function",
96 lines = ["from __future__ import print_function",
81 "import sys",
97 "import sys",
82 "print('on stdout', end='', file=sys.stdout)",
98 "print('on stdout', end='', file=sys.stdout)",
83 "print('on stderr', end='', file=sys.stderr)",
99 "print('on stderr', end='', file=sys.stderr)",
84 "sys.stdout.flush()",
100 "sys.stdout.flush()",
85 "sys.stderr.flush()"]
101 "sys.stderr.flush()"]
86 self.mktmp('\n'.join(lines))
102 self.mktmp('\n'.join(lines))
87
103
88 def test_system(self):
104 def test_system(self):
89 status = system('python "%s"' % self.fname)
105 status = system('python "%s"' % self.fname)
90 self.assertEquals(status, 0)
106 self.assertEquals(status, 0)
91
107
92 def test_system_quotes(self):
108 def test_system_quotes(self):
93 status = system('python -c "import sys"')
109 status = system('python -c "import sys"')
94 self.assertEquals(status, 0)
110 self.assertEquals(status, 0)
95
111
96 def test_getoutput(self):
112 def test_getoutput(self):
97 out = getoutput('python "%s"' % self.fname)
113 out = getoutput('python "%s"' % self.fname)
98 self.assertEquals(out, 'on stdout')
114 self.assertEquals(out, 'on stdout')
99
115
100 def test_getoutput_quoted(self):
116 def test_getoutput_quoted(self):
101 out = getoutput('python -c "print (1)"')
117 out = getoutput('python -c "print (1)"')
102 self.assertEquals(out.strip(), '1')
118 self.assertEquals(out.strip(), '1')
119
120 #Invalid quoting on windows
121 @dec.skip_win32
122 def test_getoutput_quoted2(self):
103 out = getoutput("python -c 'print (1)'")
123 out = getoutput("python -c 'print (1)'")
104 self.assertEquals(out.strip(), '1')
124 self.assertEquals(out.strip(), '1')
105 out = getoutput("python -c 'print (\"1\")'")
125 out = getoutput("python -c 'print (\"1\")'")
106 self.assertEquals(out.strip(), '1')
126 self.assertEquals(out.strip(), '1')
107
127
108 def test_getoutput(self):
128 def test_getoutput(self):
109 out, err = getoutputerror('python "%s"' % self.fname)
129 out, err = getoutputerror('python "%s"' % self.fname)
110 self.assertEquals(out, 'on stdout')
130 self.assertEquals(out, 'on stdout')
111 self.assertEquals(err, 'on stderr')
131 self.assertEquals(err, 'on stderr')
General Comments 0
You need to be logged in to leave comments. Login now