##// END OF EJS Templates
skip find_cmd when setting up script magics...
MinRK -
Show More
@@ -1,288 +1,275 b''
1 """Magic functions for running cells in various scripts."""
1 """Magic functions for running cells in various scripts."""
2 #-----------------------------------------------------------------------------
2 #-----------------------------------------------------------------------------
3 # Copyright (c) 2012 The IPython Development Team.
3 # Copyright (c) 2012 The IPython Development Team.
4 #
4 #
5 # Distributed under the terms of the Modified BSD License.
5 # Distributed under the terms of the Modified BSD License.
6 #
6 #
7 # The full license is in the file COPYING.txt, distributed with this software.
7 # The full license is in the file COPYING.txt, distributed with this software.
8 #-----------------------------------------------------------------------------
8 #-----------------------------------------------------------------------------
9
9
10 #-----------------------------------------------------------------------------
10 #-----------------------------------------------------------------------------
11 # Imports
11 # Imports
12 #-----------------------------------------------------------------------------
12 #-----------------------------------------------------------------------------
13
13
14 # Stdlib
14 # Stdlib
15 import os
15 import os
16 import re
16 import re
17 import sys
17 import sys
18 import signal
18 import signal
19 import time
19 import time
20 from subprocess import Popen, PIPE
20 from subprocess import Popen, PIPE
21 import atexit
21 import atexit
22
22
23 # Our own packages
23 # Our own packages
24 from IPython.config.configurable import Configurable
24 from IPython.config.configurable import Configurable
25 from IPython.core import magic_arguments
25 from IPython.core import magic_arguments
26 from IPython.core.error import UsageError
26 from IPython.core.error import UsageError
27 from IPython.core.magic import (
27 from IPython.core.magic import (
28 Magics, magics_class, line_magic, cell_magic
28 Magics, magics_class, line_magic, cell_magic
29 )
29 )
30 from IPython.lib.backgroundjobs import BackgroundJobManager
30 from IPython.lib.backgroundjobs import BackgroundJobManager
31 from IPython.testing.skipdoctest import skip_doctest
31 from IPython.testing.skipdoctest import skip_doctest
32 from IPython.utils import py3compat
32 from IPython.utils import py3compat
33 from IPython.utils.process import find_cmd, FindCmdError, arg_split
33 from IPython.utils.process import arg_split
34 from IPython.utils.traitlets import List, Dict
34 from IPython.utils.traitlets import List, Dict
35
35
36 #-----------------------------------------------------------------------------
36 #-----------------------------------------------------------------------------
37 # Magic implementation classes
37 # Magic implementation classes
38 #-----------------------------------------------------------------------------
38 #-----------------------------------------------------------------------------
39
39
40 def script_args(f):
40 def script_args(f):
41 """single decorator for adding script args"""
41 """single decorator for adding script args"""
42 args = [
42 args = [
43 magic_arguments.argument(
43 magic_arguments.argument(
44 '--out', type=str,
44 '--out', type=str,
45 help="""The variable in which to store stdout from the script.
45 help="""The variable in which to store stdout from the script.
46 If the script is backgrounded, this will be the stdout *pipe*,
46 If the script is backgrounded, this will be the stdout *pipe*,
47 instead of the stderr text itself.
47 instead of the stderr text itself.
48 """
48 """
49 ),
49 ),
50 magic_arguments.argument(
50 magic_arguments.argument(
51 '--err', type=str,
51 '--err', type=str,
52 help="""The variable in which to store stderr from the script.
52 help="""The variable in which to store stderr from the script.
53 If the script is backgrounded, this will be the stderr *pipe*,
53 If the script is backgrounded, this will be the stderr *pipe*,
54 instead of the stderr text itself.
54 instead of the stderr text itself.
55 """
55 """
56 ),
56 ),
57 magic_arguments.argument(
57 magic_arguments.argument(
58 '--bg', action="store_true",
58 '--bg', action="store_true",
59 help="""Whether to run the script in the background.
59 help="""Whether to run the script in the background.
60 If given, the only way to see the output of the command is
60 If given, the only way to see the output of the command is
61 with --out/err.
61 with --out/err.
62 """
62 """
63 ),
63 ),
64 magic_arguments.argument(
64 magic_arguments.argument(
65 '--proc', type=str,
65 '--proc', type=str,
66 help="""The variable in which to store Popen instance.
66 help="""The variable in which to store Popen instance.
67 This is used only when --bg option is given.
67 This is used only when --bg option is given.
68 """
68 """
69 ),
69 ),
70 ]
70 ]
71 for arg in args:
71 for arg in args:
72 f = arg(f)
72 f = arg(f)
73 return f
73 return f
74
74
75 @magics_class
75 @magics_class
76 class ScriptMagics(Magics, Configurable):
76 class ScriptMagics(Magics, Configurable):
77 """Magics for talking to scripts
77 """Magics for talking to scripts
78
78
79 This defines a base `%%script` cell magic for running a cell
79 This defines a base `%%script` cell magic for running a cell
80 with a program in a subprocess, and registers a few top-level
80 with a program in a subprocess, and registers a few top-level
81 magics that call %%script with common interpreters.
81 magics that call %%script with common interpreters.
82 """
82 """
83 script_magics = List(config=True,
83 script_magics = List(config=True,
84 help="""Extra script cell magics to define
84 help="""Extra script cell magics to define
85
85
86 This generates simple wrappers of `%%script foo` as `%%foo`.
86 This generates simple wrappers of `%%script foo` as `%%foo`.
87
87
88 If you want to add script magics that aren't on your path,
88 If you want to add script magics that aren't on your path,
89 specify them in script_paths
89 specify them in script_paths
90 """,
90 """,
91 )
91 )
92 def _script_magics_default(self):
92 def _script_magics_default(self):
93 """default to a common list of programs if we find them"""
93 """default to a common list of programs"""
94
94
95 defaults = []
95 defaults = [
96 to_try = []
97 if os.name == 'nt':
98 defaults.append('cmd')
99 to_try.append('powershell')
100 to_try.extend([
101 'sh',
96 'sh',
102 'bash',
97 'bash',
103 'perl',
98 'perl',
104 'ruby',
99 'ruby',
100 'python',
105 'python3',
101 'python3',
106 'pypy',
102 'pypy',
107 ])
103 ]
104 if os.name == 'nt':
105 defaults.extend([
106 'cmd',
107 'powershell',
108 ])
108
109
109 for cmd in to_try:
110 if cmd in self.script_paths:
111 defaults.append(cmd)
112 else:
113 try:
114 find_cmd(cmd)
115 except FindCmdError:
116 # command not found, ignore it
117 pass
118 except ImportError:
119 # Windows without pywin32, find_cmd doesn't work
120 pass
121 else:
122 defaults.append(cmd)
123 return defaults
110 return defaults
124
111
125 script_paths = Dict(config=True,
112 script_paths = Dict(config=True,
126 help="""Dict mapping short 'ruby' names to full paths, such as '/opt/secret/bin/ruby'
113 help="""Dict mapping short 'ruby' names to full paths, such as '/opt/secret/bin/ruby'
127
114
128 Only necessary for items in script_magics where the default path will not
115 Only necessary for items in script_magics where the default path will not
129 find the right interpreter.
116 find the right interpreter.
130 """
117 """
131 )
118 )
132
119
133 def __init__(self, shell=None):
120 def __init__(self, shell=None):
134 Configurable.__init__(self, config=shell.config)
121 Configurable.__init__(self, config=shell.config)
135 self._generate_script_magics()
122 self._generate_script_magics()
136 Magics.__init__(self, shell=shell)
123 Magics.__init__(self, shell=shell)
137 self.job_manager = BackgroundJobManager()
124 self.job_manager = BackgroundJobManager()
138 self.bg_processes = []
125 self.bg_processes = []
139 atexit.register(self.kill_bg_processes)
126 atexit.register(self.kill_bg_processes)
140
127
141 def __del__(self):
128 def __del__(self):
142 self.kill_bg_processes()
129 self.kill_bg_processes()
143
130
144 def _generate_script_magics(self):
131 def _generate_script_magics(self):
145 cell_magics = self.magics['cell']
132 cell_magics = self.magics['cell']
146 for name in self.script_magics:
133 for name in self.script_magics:
147 cell_magics[name] = self._make_script_magic(name)
134 cell_magics[name] = self._make_script_magic(name)
148
135
149 def _make_script_magic(self, name):
136 def _make_script_magic(self, name):
150 """make a named magic, that calls %%script with a particular program"""
137 """make a named magic, that calls %%script with a particular program"""
151 # expand to explicit path if necessary:
138 # expand to explicit path if necessary:
152 script = self.script_paths.get(name, name)
139 script = self.script_paths.get(name, name)
153
140
154 @magic_arguments.magic_arguments()
141 @magic_arguments.magic_arguments()
155 @script_args
142 @script_args
156 def named_script_magic(line, cell):
143 def named_script_magic(line, cell):
157 # if line, add it as cl-flags
144 # if line, add it as cl-flags
158 if line:
145 if line:
159 line = "%s %s" % (script, line)
146 line = "%s %s" % (script, line)
160 else:
147 else:
161 line = script
148 line = script
162 return self.shebang(line, cell)
149 return self.shebang(line, cell)
163
150
164 # write a basic docstring:
151 # write a basic docstring:
165 named_script_magic.__doc__ = \
152 named_script_magic.__doc__ = \
166 """%%{name} script magic
153 """%%{name} script magic
167
154
168 Run cells with {script} in a subprocess.
155 Run cells with {script} in a subprocess.
169
156
170 This is a shortcut for `%%script {script}`
157 This is a shortcut for `%%script {script}`
171 """.format(**locals())
158 """.format(**locals())
172
159
173 return named_script_magic
160 return named_script_magic
174
161
175 @magic_arguments.magic_arguments()
162 @magic_arguments.magic_arguments()
176 @script_args
163 @script_args
177 @cell_magic("script")
164 @cell_magic("script")
178 def shebang(self, line, cell):
165 def shebang(self, line, cell):
179 """Run a cell via a shell command
166 """Run a cell via a shell command
180
167
181 The `%%script` line is like the #! line of script,
168 The `%%script` line is like the #! line of script,
182 specifying a program (bash, perl, ruby, etc.) with which to run.
169 specifying a program (bash, perl, ruby, etc.) with which to run.
183
170
184 The rest of the cell is run by that program.
171 The rest of the cell is run by that program.
185
172
186 Examples
173 Examples
187 --------
174 --------
188 ::
175 ::
189
176
190 In [1]: %%script bash
177 In [1]: %%script bash
191 ...: for i in 1 2 3; do
178 ...: for i in 1 2 3; do
192 ...: echo $i
179 ...: echo $i
193 ...: done
180 ...: done
194 1
181 1
195 2
182 2
196 3
183 3
197 """
184 """
198 argv = arg_split(line, posix = not sys.platform.startswith('win'))
185 argv = arg_split(line, posix = not sys.platform.startswith('win'))
199 args, cmd = self.shebang.parser.parse_known_args(argv)
186 args, cmd = self.shebang.parser.parse_known_args(argv)
200
187
201 p = Popen(cmd, stdout=PIPE, stderr=PIPE, stdin=PIPE)
188 p = Popen(cmd, stdout=PIPE, stderr=PIPE, stdin=PIPE)
202
189
203 cell = cell.encode('utf8', 'replace')
190 cell = cell.encode('utf8', 'replace')
204 if args.bg:
191 if args.bg:
205 self.bg_processes.append(p)
192 self.bg_processes.append(p)
206 self._gc_bg_processes()
193 self._gc_bg_processes()
207 if args.out:
194 if args.out:
208 self.shell.user_ns[args.out] = p.stdout
195 self.shell.user_ns[args.out] = p.stdout
209 if args.err:
196 if args.err:
210 self.shell.user_ns[args.err] = p.stderr
197 self.shell.user_ns[args.err] = p.stderr
211 self.job_manager.new(self._run_script, p, cell, daemon=True)
198 self.job_manager.new(self._run_script, p, cell, daemon=True)
212 if args.proc:
199 if args.proc:
213 self.shell.user_ns[args.proc] = p
200 self.shell.user_ns[args.proc] = p
214 return
201 return
215
202
216 try:
203 try:
217 out, err = p.communicate(cell)
204 out, err = p.communicate(cell)
218 except KeyboardInterrupt:
205 except KeyboardInterrupt:
219 try:
206 try:
220 p.send_signal(signal.SIGINT)
207 p.send_signal(signal.SIGINT)
221 time.sleep(0.1)
208 time.sleep(0.1)
222 if p.poll() is not None:
209 if p.poll() is not None:
223 print "Process is interrupted."
210 print "Process is interrupted."
224 return
211 return
225 p.terminate()
212 p.terminate()
226 time.sleep(0.1)
213 time.sleep(0.1)
227 if p.poll() is not None:
214 if p.poll() is not None:
228 print "Process is terminated."
215 print "Process is terminated."
229 return
216 return
230 p.kill()
217 p.kill()
231 print "Process is killed."
218 print "Process is killed."
232 except OSError:
219 except OSError:
233 pass
220 pass
234 except Exception as e:
221 except Exception as e:
235 print "Error while terminating subprocess (pid=%i): %s" \
222 print "Error while terminating subprocess (pid=%i): %s" \
236 % (p.pid, e)
223 % (p.pid, e)
237 return
224 return
238 out = py3compat.bytes_to_str(out)
225 out = py3compat.bytes_to_str(out)
239 err = py3compat.bytes_to_str(err)
226 err = py3compat.bytes_to_str(err)
240 if args.out:
227 if args.out:
241 self.shell.user_ns[args.out] = out
228 self.shell.user_ns[args.out] = out
242 else:
229 else:
243 sys.stdout.write(out)
230 sys.stdout.write(out)
244 sys.stdout.flush()
231 sys.stdout.flush()
245 if args.err:
232 if args.err:
246 self.shell.user_ns[args.err] = err
233 self.shell.user_ns[args.err] = err
247 else:
234 else:
248 sys.stderr.write(err)
235 sys.stderr.write(err)
249 sys.stderr.flush()
236 sys.stderr.flush()
250
237
251 def _run_script(self, p, cell):
238 def _run_script(self, p, cell):
252 """callback for running the script in the background"""
239 """callback for running the script in the background"""
253 p.stdin.write(cell)
240 p.stdin.write(cell)
254 p.stdin.close()
241 p.stdin.close()
255 p.wait()
242 p.wait()
256
243
257 @line_magic("killbgscripts")
244 @line_magic("killbgscripts")
258 def killbgscripts(self, _nouse_=''):
245 def killbgscripts(self, _nouse_=''):
259 """Kill all BG processes started by %%script and its family."""
246 """Kill all BG processes started by %%script and its family."""
260 self.kill_bg_processes()
247 self.kill_bg_processes()
261 print "All background processes were killed."
248 print "All background processes were killed."
262
249
263 def kill_bg_processes(self):
250 def kill_bg_processes(self):
264 """Kill all BG processes which are still running."""
251 """Kill all BG processes which are still running."""
265 for p in self.bg_processes:
252 for p in self.bg_processes:
266 if p.poll() is None:
253 if p.poll() is None:
267 try:
254 try:
268 p.send_signal(signal.SIGINT)
255 p.send_signal(signal.SIGINT)
269 except:
256 except:
270 pass
257 pass
271 time.sleep(0.1)
258 time.sleep(0.1)
272 for p in self.bg_processes:
259 for p in self.bg_processes:
273 if p.poll() is None:
260 if p.poll() is None:
274 try:
261 try:
275 p.terminate()
262 p.terminate()
276 except:
263 except:
277 pass
264 pass
278 time.sleep(0.1)
265 time.sleep(0.1)
279 for p in self.bg_processes:
266 for p in self.bg_processes:
280 if p.poll() is None:
267 if p.poll() is None:
281 try:
268 try:
282 p.kill()
269 p.kill()
283 except:
270 except:
284 pass
271 pass
285 self._gc_bg_processes()
272 self._gc_bg_processes()
286
273
287 def _gc_bg_processes(self):
274 def _gc_bg_processes(self):
288 self.bg_processes = [p for p in self.bg_processes if p.poll() is None]
275 self.bg_processes = [p for p in self.bg_processes if p.poll() is None]
General Comments 0
You need to be logged in to leave comments. Login now