##// END OF EJS Templates
Simplify parallelmagics by removing unnecessary plugin.
Fernando Perez -
Show More
@@ -1,330 +1,318 b''
1 # encoding: utf-8
1 # encoding: utf-8
2 """
2 """
3 =============
3 =============
4 parallelmagic
4 parallelmagic
5 =============
5 =============
6
6
7 Magic command interface for interactive parallel work.
7 Magic command interface for interactive parallel work.
8
8
9 Usage
9 Usage
10 =====
10 =====
11
11
12 ``%autopx``
12 ``%autopx``
13
13
14 @AUTOPX_DOC@
14 @AUTOPX_DOC@
15
15
16 ``%px``
16 ``%px``
17
17
18 @PX_DOC@
18 @PX_DOC@
19
19
20 ``%result``
20 ``%result``
21
21
22 @RESULT_DOC@
22 @RESULT_DOC@
23
23
24 """
24 """
25
25
26 #-----------------------------------------------------------------------------
26 #-----------------------------------------------------------------------------
27 # Copyright (C) 2008-2011 The IPython Development Team
27 # Copyright (C) 2008 The IPython Development Team
28 #
28 #
29 # Distributed under the terms of the BSD License. The full license is in
29 # Distributed under the terms of the BSD License. The full license is in
30 # the file COPYING, distributed as part of this software.
30 # the file COPYING, distributed as part of this software.
31 #-----------------------------------------------------------------------------
31 #-----------------------------------------------------------------------------
32
32
33 #-----------------------------------------------------------------------------
33 #-----------------------------------------------------------------------------
34 # Imports
34 # Imports
35 #-----------------------------------------------------------------------------
35 #-----------------------------------------------------------------------------
36
36
37 import ast
37 import ast
38 import re
38 import re
39
39
40 from IPython.core.magic import Magics, register_magics, line_magic
40 from IPython.core.magic import Magics, register_magics, line_magic
41 from IPython.core.plugin import Plugin
42 from IPython.testing.skipdoctest import skip_doctest
41 from IPython.testing.skipdoctest import skip_doctest
43 from IPython.utils.traitlets import Bool, Any, Instance
42 from IPython.utils.traitlets import Instance
44
43
45 #-----------------------------------------------------------------------------
44 #-----------------------------------------------------------------------------
46 # Definitions of magic functions for use with IPython
45 # Definitions of magic functions for use with IPython
47 #-----------------------------------------------------------------------------
46 #-----------------------------------------------------------------------------
48
47
49 NO_ACTIVE_VIEW = """
48 NO_ACTIVE_VIEW = """
50 Use activate() on a DirectView object to activate it for magics.
49 Use activate() on a DirectView object to activate it for magics.
51 """
50 """
52
51
53
52
54 class ParallelTricks(Plugin):
55 """A component to manage the %result, %px and %autopx magics."""
56
57 active_view = Instance('IPython.parallel.client.view.DirectView')
58 verbose = Bool(False, config=True)
59 shell = Instance('IPython.core.interactiveshell.InteractiveShellABC')
60
61 def __init__(self, shell=None, config=None):
62 super(ParallelTricks, self).__init__(shell=shell, config=config)
63 self.shell.register_magics(ParallelMagics)
64
65
66 @register_magics
53 @register_magics
67 class ParallelMagics(Magics):
54 class ParallelMagics(Magics):
68 """A set of magics useful when controlling a parallel IPython cluster.
55 """A set of magics useful when controlling a parallel IPython cluster.
69 """
56 """
57 active_view = Instance('IPython.parallel.client.view.DirectView')
58
70 def __init__(self, shell):
59 def __init__(self, shell):
71 super(ParallelMagics, self).__init__(shell)
60 super(ParallelMagics, self).__init__(shell)
72 # A flag showing if autopx is activated or not
61 # A flag showing if autopx is activated or not
73 self.autopx = False
62 self.autopx = False
74
63
75 @skip_doctest
64 @skip_doctest
76 @line_magic
65 @line_magic
77 def result(self, parameter_s=''):
66 def result(self, parameter_s=''):
78 """Print the result of command i on all engines..
67 """Print the result of command i on all engines..
79
68
80 To use this a :class:`DirectView` instance must be created
69 To use this a :class:`DirectView` instance must be created
81 and then activated by calling its :meth:`activate` method.
70 and then activated by calling its :meth:`activate` method.
82
71
83 Then you can do the following::
72 Then you can do the following::
84
73
85 In [23]: %result
74 In [23]: %result
86 Out[23]:
75 Out[23]:
87 <Results List>
76 <Results List>
88 [0] In [6]: a = 10
77 [0] In [6]: a = 10
89 [1] In [6]: a = 10
78 [1] In [6]: a = 10
90
79
91 In [22]: %result 6
80 In [22]: %result 6
92 Out[22]:
81 Out[22]:
93 <Results List>
82 <Results List>
94 [0] In [6]: a = 10
83 [0] In [6]: a = 10
95 [1] In [6]: a = 10
84 [1] In [6]: a = 10
96 """
85 """
97 if self.active_view is None:
86 if self.active_view is None:
98 print NO_ACTIVE_VIEW
87 print NO_ACTIVE_VIEW
99 return
88 return
100
89
101 try:
90 try:
102 index = int(parameter_s)
91 index = int(parameter_s)
103 except:
92 except:
104 index = None
93 index = None
105 result = self.active_view.get_result(index)
94 result = self.active_view.get_result(index)
106 return result
95 return result
107
96
108 @skip_doctest
97 @skip_doctest
109 @line_magic
98 @line_magic
110 def px(self, parameter_s=''):
99 def px(self, parameter_s=''):
111 """Executes the given python command in parallel.
100 """Executes the given python command in parallel.
112
101
113 To use this a :class:`DirectView` instance must be created
102 To use this a :class:`DirectView` instance must be created
114 and then activated by calling its :meth:`activate` method.
103 and then activated by calling its :meth:`activate` method.
115
104
116 Then you can do the following::
105 Then you can do the following::
117
106
118 In [24]: %px a = 5
107 In [24]: %px a = 5
119 Parallel execution on engine(s): all
108 Parallel execution on engine(s): all
120 Out[24]:
109 Out[24]:
121 <Results List>
110 <Results List>
122 [0] In [7]: a = 5
111 [0] In [7]: a = 5
123 [1] In [7]: a = 5
112 [1] In [7]: a = 5
124 """
113 """
125
114
126 if self.active_view is None:
115 if self.active_view is None:
127 print NO_ACTIVE_VIEW
116 print NO_ACTIVE_VIEW
128 return
117 return
129 print "Parallel execution on engine(s): %s" % self.active_view.targets
118 print "Parallel execution on engine(s): %s" % self.active_view.targets
130 result = self.active_view.execute(parameter_s, block=False)
119 result = self.active_view.execute(parameter_s, block=False)
131 if self.active_view.block:
120 if self.active_view.block:
132 result.get()
121 result.get()
133 self._maybe_display_output(result)
122 self._maybe_display_output(result)
134
123
135 @skip_doctest
124 @skip_doctest
136 @line_magic
125 @line_magic
137 def autopx(self, parameter_s=''):
126 def autopx(self, parameter_s=''):
138 """Toggles auto parallel mode.
127 """Toggles auto parallel mode.
139
128
140 To use this a :class:`DirectView` instance must be created
129 To use this a :class:`DirectView` instance must be created
141 and then activated by calling its :meth:`activate` method. Once this
130 and then activated by calling its :meth:`activate` method. Once this
142 is called, all commands typed at the command line are send to
131 is called, all commands typed at the command line are send to
143 the engines to be executed in parallel. To control which engine
132 the engines to be executed in parallel. To control which engine
144 are used, set the ``targets`` attributed of the multiengine client
133 are used, set the ``targets`` attributed of the multiengine client
145 before entering ``%autopx`` mode.
134 before entering ``%autopx`` mode.
146
135
147 Then you can do the following::
136 Then you can do the following::
148
137
149 In [25]: %autopx
138 In [25]: %autopx
150 %autopx to enabled
139 %autopx to enabled
151
140
152 In [26]: a = 10
141 In [26]: a = 10
153 Parallel execution on engine(s): [0,1,2,3]
142 Parallel execution on engine(s): [0,1,2,3]
154 In [27]: print a
143 In [27]: print a
155 Parallel execution on engine(s): [0,1,2,3]
144 Parallel execution on engine(s): [0,1,2,3]
156 [stdout:0] 10
145 [stdout:0] 10
157 [stdout:1] 10
146 [stdout:1] 10
158 [stdout:2] 10
147 [stdout:2] 10
159 [stdout:3] 10
148 [stdout:3] 10
160
149
161
150
162 In [27]: %autopx
151 In [27]: %autopx
163 %autopx disabled
152 %autopx disabled
164 """
153 """
165 if self.autopx:
154 if self.autopx:
166 self._disable_autopx()
155 self._disable_autopx()
167 else:
156 else:
168 self._enable_autopx()
157 self._enable_autopx()
169
158
170 def _enable_autopx(self):
159 def _enable_autopx(self):
171 """Enable %autopx mode by saving the original run_cell and installing
160 """Enable %autopx mode by saving the original run_cell and installing
172 pxrun_cell.
161 pxrun_cell.
173 """
162 """
174 if self.active_view is None:
163 if self.active_view is None:
175 print NO_ACTIVE_VIEW
164 print NO_ACTIVE_VIEW
176 return
165 return
177
166
178 # override run_cell and run_code
167 # override run_cell and run_code
179 self._original_run_cell = self.shell.run_cell
168 self._original_run_cell = self.shell.run_cell
180 self.shell.run_cell = self.pxrun_cell
169 self.shell.run_cell = self.pxrun_cell
181 self._original_run_code = self.shell.run_code
170 self._original_run_code = self.shell.run_code
182 self.shell.run_code = self.pxrun_code
171 self.shell.run_code = self.pxrun_code
183
172
184 self.autopx = True
173 self.autopx = True
185 print "%autopx enabled"
174 print "%autopx enabled"
186
175
187 def _disable_autopx(self):
176 def _disable_autopx(self):
188 """Disable %autopx by restoring the original InteractiveShell.run_cell.
177 """Disable %autopx by restoring the original InteractiveShell.run_cell.
189 """
178 """
190 if self.autopx:
179 if self.autopx:
191 self.shell.run_cell = self._original_run_cell
180 self.shell.run_cell = self._original_run_cell
192 self.shell.run_code = self._original_run_code
181 self.shell.run_code = self._original_run_code
193 self.autopx = False
182 self.autopx = False
194 print "%autopx disabled"
183 print "%autopx disabled"
195
184
196 def _maybe_display_output(self, result):
185 def _maybe_display_output(self, result):
197 """Maybe display the output of a parallel result.
186 """Maybe display the output of a parallel result.
198
187
199 If self.active_view.block is True, wait for the result
188 If self.active_view.block is True, wait for the result
200 and display the result. Otherwise, this is a noop.
189 and display the result. Otherwise, this is a noop.
201 """
190 """
202 if isinstance(result.stdout, basestring):
191 if isinstance(result.stdout, basestring):
203 # single result
192 # single result
204 stdouts = [result.stdout.rstrip()]
193 stdouts = [result.stdout.rstrip()]
205 else:
194 else:
206 stdouts = [s.rstrip() for s in result.stdout]
195 stdouts = [s.rstrip() for s in result.stdout]
207
196
208 targets = self.active_view.targets
197 targets = self.active_view.targets
209 if isinstance(targets, int):
198 if isinstance(targets, int):
210 targets = [targets]
199 targets = [targets]
211 elif targets == 'all':
200 elif targets == 'all':
212 targets = self.active_view.client.ids
201 targets = self.active_view.client.ids
213
202
214 if any(stdouts):
203 if any(stdouts):
215 for eid,stdout in zip(targets, stdouts):
204 for eid,stdout in zip(targets, stdouts):
216 print '[stdout:%i]'%eid, stdout
205 print '[stdout:%i]'%eid, stdout
217
206
218
207
219 def pxrun_cell(self, raw_cell, store_history=False, silent=False):
208 def pxrun_cell(self, raw_cell, store_history=False, silent=False):
220 """drop-in replacement for InteractiveShell.run_cell.
209 """drop-in replacement for InteractiveShell.run_cell.
221
210
222 This executes code remotely, instead of in the local namespace.
211 This executes code remotely, instead of in the local namespace.
223
212
224 See InteractiveShell.run_cell for details.
213 See InteractiveShell.run_cell for details.
225 """
214 """
226
215
227 if (not raw_cell) or raw_cell.isspace():
216 if (not raw_cell) or raw_cell.isspace():
228 return
217 return
229
218
230 ipself = self.shell
219 ipself = self.shell
231
220
232 with ipself.builtin_trap:
221 with ipself.builtin_trap:
233 cell = ipself.prefilter_manager.prefilter_lines(raw_cell)
222 cell = ipself.prefilter_manager.prefilter_lines(raw_cell)
234
223
235 # Store raw and processed history
224 # Store raw and processed history
236 if store_history:
225 if store_history:
237 ipself.history_manager.store_inputs(ipself.execution_count,
226 ipself.history_manager.store_inputs(ipself.execution_count,
238 cell, raw_cell)
227 cell, raw_cell)
239
228
240 # ipself.logger.log(cell, raw_cell)
229 # ipself.logger.log(cell, raw_cell)
241
230
242 cell_name = ipself.compile.cache(cell, ipself.execution_count)
231 cell_name = ipself.compile.cache(cell, ipself.execution_count)
243
232
244 try:
233 try:
245 code_ast = ast.parse(cell, filename=cell_name)
234 ast.parse(cell, filename=cell_name)
246 except (OverflowError, SyntaxError, ValueError, TypeError, MemoryError):
235 except (OverflowError, SyntaxError, ValueError, TypeError,
236 MemoryError):
247 # Case 1
237 # Case 1
248 ipself.showsyntaxerror()
238 ipself.showsyntaxerror()
249 ipself.execution_count += 1
239 ipself.execution_count += 1
250 return None
240 return None
251 except NameError:
241 except NameError:
252 # ignore name errors, because we don't know the remote keys
242 # ignore name errors, because we don't know the remote keys
253 pass
243 pass
254
244
255 if store_history:
245 if store_history:
256 # Write output to the database. Does nothing unless
246 # Write output to the database. Does nothing unless
257 # history output logging is enabled.
247 # history output logging is enabled.
258 ipself.history_manager.store_output(ipself.execution_count)
248 ipself.history_manager.store_output(ipself.execution_count)
259 # Each cell is a *single* input, regardless of how many lines it has
249 # Each cell is a *single* input, regardless of how many lines it has
260 ipself.execution_count += 1
250 ipself.execution_count += 1
261 if re.search(r'get_ipython\(\)\.magic\(u?["\']%?autopx', cell):
251 if re.search(r'get_ipython\(\)\.magic\(u?["\']%?autopx', cell):
262 self._disable_autopx()
252 self._disable_autopx()
263 return False
253 return False
264 else:
254 else:
265 try:
255 try:
266 result = self.active_view.execute(cell, silent=False, block=False)
256 result = self.active_view.execute(cell, silent=False, block=False)
267 except:
257 except:
268 ipself.showtraceback()
258 ipself.showtraceback()
269 return True
259 return True
270 else:
260 else:
271 if self.active_view.block:
261 if self.active_view.block:
272 try:
262 try:
273 result.get()
263 result.get()
274 except:
264 except:
275 self.shell.showtraceback()
265 self.shell.showtraceback()
276 return True
266 return True
277 else:
267 else:
278 self._maybe_display_output(result)
268 self._maybe_display_output(result)
279 return False
269 return False
280
270
281 def pxrun_code(self, code_obj):
271 def pxrun_code(self, code_obj):
282 """drop-in replacement for InteractiveShell.run_code.
272 """drop-in replacement for InteractiveShell.run_code.
283
273
284 This executes code remotely, instead of in the local namespace.
274 This executes code remotely, instead of in the local namespace.
285
275
286 See InteractiveShell.run_code for details.
276 See InteractiveShell.run_code for details.
287 """
277 """
288 ipself = self.shell
278 ipself = self.shell
289 # check code object for the autopx magic
279 # check code object for the autopx magic
290 if 'get_ipython' in code_obj.co_names and 'magic' in code_obj.co_names \
280 if 'get_ipython' in code_obj.co_names and 'magic' in code_obj.co_names \
291 and any( [ isinstance(c, basestring) and 'autopx' in c
281 and any( [ isinstance(c, basestring) and 'autopx' in c
292 for c in code_obj.co_consts ]):
282 for c in code_obj.co_consts ]):
293 self._disable_autopx()
283 self._disable_autopx()
294 return False
284 return False
295 else:
285 else:
296 try:
286 try:
297 result = self.active_view.execute(code_obj, block=False)
287 result = self.active_view.execute(code_obj, block=False)
298 except:
288 except:
299 ipself.showtraceback()
289 ipself.showtraceback()
300 return True
290 return True
301 else:
291 else:
302 if self.active_view.block:
292 if self.active_view.block:
303 try:
293 try:
304 result.get()
294 result.get()
305 except:
295 except:
306 self.shell.showtraceback()
296 self.shell.showtraceback()
307 return True
297 return True
308 else:
298 else:
309 self._maybe_display_output(result)
299 self._maybe_display_output(result)
310 return False
300 return False
311
301
312
302
313 __doc__ = __doc__.replace('@AUTOPX_DOC@',
303 __doc__ = __doc__.replace('@AUTOPX_DOC@',
314 " " + ParallelMagics.autopx.__doc__)
304 " " + ParallelMagics.autopx.__doc__)
315 __doc__ = __doc__.replace('@PX_DOC@',
305 __doc__ = __doc__.replace('@PX_DOC@',
316 " " + ParallelMagics.px.__doc__)
306 " " + ParallelMagics.px.__doc__)
317 __doc__ = __doc__.replace('@RESULT_DOC@',
307 __doc__ = __doc__.replace('@RESULT_DOC@',
318 " " + ParallelMagics.result.__doc__)
308 " " + ParallelMagics.result.__doc__)
319
309
320
321 _loaded = False
310 _loaded = False
322
311
323
312
324 def load_ipython_extension(ip):
313 def load_ipython_extension(ip):
325 """Load the extension in IPython."""
314 """Load the extension in IPython."""
326 global _loaded
315 global _loaded
327 if not _loaded:
316 if not _loaded:
328 plugin = ParallelTricks(shell=ip, config=ip.config)
317 ip.register_magics(ParallelMagics)
329 ip.plugin_manager.register_plugin('parallelmagic', plugin)
330 _loaded = True
318 _loaded = True
General Comments 0
You need to be logged in to leave comments. Login now