##// END OF EJS Templates
Fix invalid attribute access in parallelmagic
Fernando Perez -
Show More
@@ -1,330 +1,330 b''
1 # encoding: utf-8
1 # encoding: utf-8
2 """
2 """
3 =============
3 =============
4 parallelmagic
4 parallelmagic
5 =============
5 =============
6
6
7 Magic command interface for interactive parallel work.
7 Magic command interface for interactive parallel work.
8
8
9 Usage
9 Usage
10 =====
10 =====
11
11
12 ``%autopx``
12 ``%autopx``
13
13
14 @AUTOPX_DOC@
14 @AUTOPX_DOC@
15
15
16 ``%px``
16 ``%px``
17
17
18 @PX_DOC@
18 @PX_DOC@
19
19
20 ``%result``
20 ``%result``
21
21
22 @RESULT_DOC@
22 @RESULT_DOC@
23
23
24 """
24 """
25
25
26 #-----------------------------------------------------------------------------
26 #-----------------------------------------------------------------------------
27 # Copyright (C) 2008-2011 The IPython Development Team
27 # Copyright (C) 2008-2011 The IPython Development Team
28 #
28 #
29 # Distributed under the terms of the BSD License. The full license is in
29 # Distributed under the terms of the BSD License. The full license is in
30 # the file COPYING, distributed as part of this software.
30 # the file COPYING, distributed as part of this software.
31 #-----------------------------------------------------------------------------
31 #-----------------------------------------------------------------------------
32
32
33 #-----------------------------------------------------------------------------
33 #-----------------------------------------------------------------------------
34 # Imports
34 # Imports
35 #-----------------------------------------------------------------------------
35 #-----------------------------------------------------------------------------
36
36
37 import ast
37 import ast
38 import re
38 import re
39
39
40 from IPython.core.magic import Magics, register_magics, line_magic
40 from IPython.core.magic import Magics, register_magics, line_magic
41 from IPython.core.plugin import Plugin
41 from IPython.core.plugin import Plugin
42 from IPython.testing.skipdoctest import skip_doctest
42 from IPython.testing.skipdoctest import skip_doctest
43 from IPython.utils.traitlets import Bool, Any, Instance
43 from IPython.utils.traitlets import Bool, Any, Instance
44
44
45 #-----------------------------------------------------------------------------
45 #-----------------------------------------------------------------------------
46 # Definitions of magic functions for use with IPython
46 # Definitions of magic functions for use with IPython
47 #-----------------------------------------------------------------------------
47 #-----------------------------------------------------------------------------
48
48
49 NO_ACTIVE_VIEW = """
49 NO_ACTIVE_VIEW = """
50 Use activate() on a DirectView object to activate it for magics.
50 Use activate() on a DirectView object to activate it for magics.
51 """
51 """
52
52
53
53
54 class ParallelTricks(Plugin):
54 class ParallelTricks(Plugin):
55 """A component to manage the %result, %px and %autopx magics."""
55 """A component to manage the %result, %px and %autopx magics."""
56
56
57 active_view = Instance('IPython.parallel.client.view.DirectView')
57 active_view = Instance('IPython.parallel.client.view.DirectView')
58 verbose = Bool(False, config=True)
58 verbose = Bool(False, config=True)
59 shell = Instance('IPython.core.interactiveshell.InteractiveShellABC')
59 shell = Instance('IPython.core.interactiveshell.InteractiveShellABC')
60
60
61 def __init__(self, shell=None, config=None):
61 def __init__(self, shell=None, config=None):
62 super(ParallelTricks, self).__init__(shell=shell, config=config)
62 super(ParallelTricks, self).__init__(shell=shell, config=config)
63 self.shell.register_magics(ParallelMagics)
63 self.shell.register_magics(ParallelMagics)
64
64
65
65
66 @register_magics
66 @register_magics
67 class ParallelMagics(Magics):
67 class ParallelMagics(Magics):
68 """A set of magics useful when controlling a parallel IPython cluster.
68 """A set of magics useful when controlling a parallel IPython cluster.
69 """
69 """
70 def __init__(self, shell):
70 def __init__(self, shell):
71 super(ParallelMagics, self).__init__(shell)
71 super(ParallelMagics, self).__init__(shell)
72 # A flag showing if autopx is activated or not
72 # A flag showing if autopx is activated or not
73 self.autopx = False
73 self.autopx = False
74
74
75 @skip_doctest
75 @skip_doctest
76 @line_magic
76 @line_magic
77 def result(self, parameter_s=''):
77 def result(self, parameter_s=''):
78 """Print the result of command i on all engines..
78 """Print the result of command i on all engines..
79
79
80 To use this a :class:`DirectView` instance must be created
80 To use this a :class:`DirectView` instance must be created
81 and then activated by calling its :meth:`activate` method.
81 and then activated by calling its :meth:`activate` method.
82
82
83 Then you can do the following::
83 Then you can do the following::
84
84
85 In [23]: %result
85 In [23]: %result
86 Out[23]:
86 Out[23]:
87 <Results List>
87 <Results List>
88 [0] In [6]: a = 10
88 [0] In [6]: a = 10
89 [1] In [6]: a = 10
89 [1] In [6]: a = 10
90
90
91 In [22]: %result 6
91 In [22]: %result 6
92 Out[22]:
92 Out[22]:
93 <Results List>
93 <Results List>
94 [0] In [6]: a = 10
94 [0] In [6]: a = 10
95 [1] In [6]: a = 10
95 [1] In [6]: a = 10
96 """
96 """
97 if self.active_view is None:
97 if self.active_view is None:
98 print NO_ACTIVE_VIEW
98 print NO_ACTIVE_VIEW
99 return
99 return
100
100
101 try:
101 try:
102 index = int(parameter_s)
102 index = int(parameter_s)
103 except:
103 except:
104 index = None
104 index = None
105 result = self.active_view.get_result(index)
105 result = self.active_view.get_result(index)
106 return result
106 return result
107
107
108 @skip_doctest
108 @skip_doctest
109 @line_magic
109 @line_magic
110 def px(self, parameter_s=''):
110 def px(self, parameter_s=''):
111 """Executes the given python command in parallel.
111 """Executes the given python command in parallel.
112
112
113 To use this a :class:`DirectView` instance must be created
113 To use this a :class:`DirectView` instance must be created
114 and then activated by calling its :meth:`activate` method.
114 and then activated by calling its :meth:`activate` method.
115
115
116 Then you can do the following::
116 Then you can do the following::
117
117
118 In [24]: %px a = 5
118 In [24]: %px a = 5
119 Parallel execution on engine(s): all
119 Parallel execution on engine(s): all
120 Out[24]:
120 Out[24]:
121 <Results List>
121 <Results List>
122 [0] In [7]: a = 5
122 [0] In [7]: a = 5
123 [1] In [7]: a = 5
123 [1] In [7]: a = 5
124 """
124 """
125
125
126 if self.active_view is None:
126 if self.active_view is None:
127 print NO_ACTIVE_VIEW
127 print NO_ACTIVE_VIEW
128 return
128 return
129 print "Parallel execution on engine(s): %s" % self.active_view.targets
129 print "Parallel execution on engine(s): %s" % self.active_view.targets
130 result = self.active_view.execute(parameter_s, block=False)
130 result = self.active_view.execute(parameter_s, block=False)
131 if self.active_view.block:
131 if self.active_view.block:
132 result.get()
132 result.get()
133 self._maybe_display_output(result)
133 self._maybe_display_output(result)
134
134
135 @skip_doctest
135 @skip_doctest
136 @line_magic
136 @line_magic
137 def autopx(self, parameter_s=''):
137 def autopx(self, parameter_s=''):
138 """Toggles auto parallel mode.
138 """Toggles auto parallel mode.
139
139
140 To use this a :class:`DirectView` instance must be created
140 To use this a :class:`DirectView` instance must be created
141 and then activated by calling its :meth:`activate` method. Once this
141 and then activated by calling its :meth:`activate` method. Once this
142 is called, all commands typed at the command line are send to
142 is called, all commands typed at the command line are send to
143 the engines to be executed in parallel. To control which engine
143 the engines to be executed in parallel. To control which engine
144 are used, set the ``targets`` attributed of the multiengine client
144 are used, set the ``targets`` attributed of the multiengine client
145 before entering ``%autopx`` mode.
145 before entering ``%autopx`` mode.
146
146
147 Then you can do the following::
147 Then you can do the following::
148
148
149 In [25]: %autopx
149 In [25]: %autopx
150 %autopx to enabled
150 %autopx to enabled
151
151
152 In [26]: a = 10
152 In [26]: a = 10
153 Parallel execution on engine(s): [0,1,2,3]
153 Parallel execution on engine(s): [0,1,2,3]
154 In [27]: print a
154 In [27]: print a
155 Parallel execution on engine(s): [0,1,2,3]
155 Parallel execution on engine(s): [0,1,2,3]
156 [stdout:0] 10
156 [stdout:0] 10
157 [stdout:1] 10
157 [stdout:1] 10
158 [stdout:2] 10
158 [stdout:2] 10
159 [stdout:3] 10
159 [stdout:3] 10
160
160
161
161
162 In [27]: %autopx
162 In [27]: %autopx
163 %autopx disabled
163 %autopx disabled
164 """
164 """
165 if self.autopx:
165 if self.autopx:
166 self._disable_autopx()
166 self._disable_autopx()
167 else:
167 else:
168 self._enable_autopx()
168 self._enable_autopx()
169
169
170 def _enable_autopx(self):
170 def _enable_autopx(self):
171 """Enable %autopx mode by saving the original run_cell and installing
171 """Enable %autopx mode by saving the original run_cell and installing
172 pxrun_cell.
172 pxrun_cell.
173 """
173 """
174 if self.active_view is None:
174 if self.active_view is None:
175 print NO_ACTIVE_VIEW
175 print NO_ACTIVE_VIEW
176 return
176 return
177
177
178 # override run_cell and run_code
178 # override run_cell and run_code
179 self._original_run_cell = self.shell.run_cell
179 self._original_run_cell = self.shell.run_cell
180 self.shell.run_cell = self.pxrun_cell
180 self.shell.run_cell = self.pxrun_cell
181 self._original_run_code = self.shell.run_code
181 self._original_run_code = self.shell.run_code
182 self.shell.run_code = self.pxrun_code
182 self.shell.run_code = self.pxrun_code
183
183
184 self.autopx = True
184 self.autopx = True
185 print "%autopx enabled"
185 print "%autopx enabled"
186
186
187 def _disable_autopx(self):
187 def _disable_autopx(self):
188 """Disable %autopx by restoring the original InteractiveShell.run_cell.
188 """Disable %autopx by restoring the original InteractiveShell.run_cell.
189 """
189 """
190 if self.autopx:
190 if self.autopx:
191 self.shell.run_cell = self._original_run_cell
191 self.shell.run_cell = self._original_run_cell
192 self.shell.run_code = self._original_run_code
192 self.shell.run_code = self._original_run_code
193 self.autopx = False
193 self.autopx = False
194 print "%autopx disabled"
194 print "%autopx disabled"
195
195
196 def _maybe_display_output(self, result):
196 def _maybe_display_output(self, result):
197 """Maybe display the output of a parallel result.
197 """Maybe display the output of a parallel result.
198
198
199 If self.active_view.block is True, wait for the result
199 If self.active_view.block is True, wait for the result
200 and display the result. Otherwise, this is a noop.
200 and display the result. Otherwise, this is a noop.
201 """
201 """
202 if isinstance(result.stdout, basestring):
202 if isinstance(result.stdout, basestring):
203 # single result
203 # single result
204 stdouts = [result.stdout.rstrip()]
204 stdouts = [result.stdout.rstrip()]
205 else:
205 else:
206 stdouts = [s.rstrip() for s in result.stdout]
206 stdouts = [s.rstrip() for s in result.stdout]
207
207
208 targets = self.active_view.targets
208 targets = self.active_view.targets
209 if isinstance(targets, int):
209 if isinstance(targets, int):
210 targets = [targets]
210 targets = [targets]
211 elif targets == 'all':
211 elif targets == 'all':
212 targets = self.active_view.client.ids
212 targets = self.active_view.client.ids
213
213
214 if any(stdouts):
214 if any(stdouts):
215 for eid,stdout in zip(targets, stdouts):
215 for eid,stdout in zip(targets, stdouts):
216 print '[stdout:%i]'%eid, stdout
216 print '[stdout:%i]'%eid, stdout
217
217
218
218
219 def pxrun_cell(self, raw_cell, store_history=False, silent=False):
219 def pxrun_cell(self, raw_cell, store_history=False, silent=False):
220 """drop-in replacement for InteractiveShell.run_cell.
220 """drop-in replacement for InteractiveShell.run_cell.
221
221
222 This executes code remotely, instead of in the local namespace.
222 This executes code remotely, instead of in the local namespace.
223
223
224 See InteractiveShell.run_cell for details.
224 See InteractiveShell.run_cell for details.
225 """
225 """
226
226
227 if (not raw_cell) or raw_cell.isspace():
227 if (not raw_cell) or raw_cell.isspace():
228 return
228 return
229
229
230 ipself = self.shell
230 ipself = self.shell
231
231
232 with ipself.builtin_trap:
232 with ipself.builtin_trap:
233 cell = ipself.prefilter_manager.prefilter_lines(raw_cell)
233 cell = ipself.prefilter_manager.prefilter_lines(raw_cell)
234
234
235 # Store raw and processed history
235 # Store raw and processed history
236 if store_history:
236 if store_history:
237 ipself.history_manager.store_inputs(ipself.execution_count,
237 ipself.history_manager.store_inputs(ipself.execution_count,
238 cell, raw_cell)
238 cell, raw_cell)
239
239
240 # ipself.logger.log(cell, raw_cell)
240 # ipself.logger.log(cell, raw_cell)
241
241
242 cell_name = ipself.compile.cache(cell, ipself.execution_count)
242 cell_name = ipself.compile.cache(cell, ipself.execution_count)
243
243
244 try:
244 try:
245 code_ast = ast.parse(cell, filename=cell_name)
245 code_ast = ast.parse(cell, filename=cell_name)
246 except (OverflowError, SyntaxError, ValueError, TypeError, MemoryError):
246 except (OverflowError, SyntaxError, ValueError, TypeError, MemoryError):
247 # Case 1
247 # Case 1
248 ipself.showsyntaxerror()
248 ipself.showsyntaxerror()
249 ipself.execution_count += 1
249 ipself.execution_count += 1
250 return None
250 return None
251 except NameError:
251 except NameError:
252 # ignore name errors, because we don't know the remote keys
252 # ignore name errors, because we don't know the remote keys
253 pass
253 pass
254
254
255 if store_history:
255 if store_history:
256 # Write output to the database. Does nothing unless
256 # Write output to the database. Does nothing unless
257 # history output logging is enabled.
257 # history output logging is enabled.
258 ipself.history_manager.store_output(ipself.execution_count)
258 ipself.history_manager.store_output(ipself.execution_count)
259 # Each cell is a *single* input, regardless of how many lines it has
259 # Each cell is a *single* input, regardless of how many lines it has
260 ipself.execution_count += 1
260 ipself.execution_count += 1
261 if re.search(r'get_ipython\(\)\.magic\(u?["\']%?autopx', cell):
261 if re.search(r'get_ipython\(\)\.magic\(u?["\']%?autopx', cell):
262 self._disable_autopx()
262 self._disable_autopx()
263 return False
263 return False
264 else:
264 else:
265 try:
265 try:
266 result = self.active_view.execute(cell, silent=False, block=False)
266 result = self.active_view.execute(cell, silent=False, block=False)
267 except:
267 except:
268 ipself.showtraceback()
268 ipself.showtraceback()
269 return True
269 return True
270 else:
270 else:
271 if self.active_view.block:
271 if self.active_view.block:
272 try:
272 try:
273 result.get()
273 result.get()
274 except:
274 except:
275 self.shell.showtraceback()
275 self.shell.showtraceback()
276 return True
276 return True
277 else:
277 else:
278 self._maybe_display_output(result)
278 self._maybe_display_output(result)
279 return False
279 return False
280
280
281 def pxrun_code(self, code_obj):
281 def pxrun_code(self, code_obj):
282 """drop-in replacement for InteractiveShell.run_code.
282 """drop-in replacement for InteractiveShell.run_code.
283
283
284 This executes code remotely, instead of in the local namespace.
284 This executes code remotely, instead of in the local namespace.
285
285
286 See InteractiveShell.run_code for details.
286 See InteractiveShell.run_code for details.
287 """
287 """
288 ipself = self.shell
288 ipself = self.shell
289 # check code object for the autopx magic
289 # check code object for the autopx magic
290 if 'get_ipython' in code_obj.co_names and 'magic' in code_obj.co_names \
290 if 'get_ipython' in code_obj.co_names and 'magic' in code_obj.co_names \
291 and any( [ isinstance(c, basestring) and 'autopx' in c
291 and any( [ isinstance(c, basestring) and 'autopx' in c
292 for c in code_obj.co_consts ]):
292 for c in code_obj.co_consts ]):
293 self._disable_autopx()
293 self._disable_autopx()
294 return False
294 return False
295 else:
295 else:
296 try:
296 try:
297 result = self.active_view.execute(code_obj, block=False)
297 result = self.active_view.execute(code_obj, block=False)
298 except:
298 except:
299 ipself.showtraceback()
299 ipself.showtraceback()
300 return True
300 return True
301 else:
301 else:
302 if self.active_view.block:
302 if self.active_view.block:
303 try:
303 try:
304 result.get()
304 result.get()
305 except:
305 except:
306 self.shell.showtraceback()
306 self.shell.showtraceback()
307 return True
307 return True
308 else:
308 else:
309 self._maybe_display_output(result)
309 self._maybe_display_output(result)
310 return False
310 return False
311
311
312
312
313 __doc__ = __doc__.replace('@AUTOPX_DOC@',
313 __doc__ = __doc__.replace('@AUTOPX_DOC@',
314 " " + ParallelTricks.magic_autopx.__doc__)
314 " " + ParallelMagics.autopx.__doc__)
315 __doc__ = __doc__.replace('@PX_DOC@',
315 __doc__ = __doc__.replace('@PX_DOC@',
316 " " + ParallelTricks.magic_px.__doc__)
316 " " + ParallelMagics.px.__doc__)
317 __doc__ = __doc__.replace('@RESULT_DOC@',
317 __doc__ = __doc__.replace('@RESULT_DOC@',
318 " " + ParallelTricks.magic_result.__doc__)
318 " " + ParallelMagics.result.__doc__)
319
319
320
320
321 _loaded = False
321 _loaded = False
322
322
323
323
324 def load_ipython_extension(ip):
324 def load_ipython_extension(ip):
325 """Load the extension in IPython."""
325 """Load the extension in IPython."""
326 global _loaded
326 global _loaded
327 if not _loaded:
327 if not _loaded:
328 plugin = ParallelTricks(shell=ip, config=ip.config)
328 plugin = ParallelTricks(shell=ip, config=ip.config)
329 ip.plugin_manager.register_plugin('parallelmagic', plugin)
329 ip.plugin_manager.register_plugin('parallelmagic', plugin)
330 _loaded = True
330 _loaded = True
General Comments 0
You need to be logged in to leave comments. Login now