##// END OF EJS Templates
Simplify parallelmagics by removing unnecessary plugin.
Fernando Perez -
Show More
@@ -1,330 +1,318 b''
1 1 # encoding: utf-8
2 2 """
3 3 =============
4 4 parallelmagic
5 5 =============
6 6
7 7 Magic command interface for interactive parallel work.
8 8
9 9 Usage
10 10 =====
11 11
12 12 ``%autopx``
13 13
14 14 @AUTOPX_DOC@
15 15
16 16 ``%px``
17 17
18 18 @PX_DOC@
19 19
20 20 ``%result``
21 21
22 22 @RESULT_DOC@
23 23
24 24 """
25 25
26 26 #-----------------------------------------------------------------------------
27 # Copyright (C) 2008-2011 The IPython Development Team
27 # Copyright (C) 2008 The IPython Development Team
28 28 #
29 29 # Distributed under the terms of the BSD License. The full license is in
30 30 # the file COPYING, distributed as part of this software.
31 31 #-----------------------------------------------------------------------------
32 32
33 33 #-----------------------------------------------------------------------------
34 34 # Imports
35 35 #-----------------------------------------------------------------------------
36 36
37 37 import ast
38 38 import re
39 39
40 40 from IPython.core.magic import Magics, register_magics, line_magic
41 from IPython.core.plugin import Plugin
42 41 from IPython.testing.skipdoctest import skip_doctest
43 from IPython.utils.traitlets import Bool, Any, Instance
42 from IPython.utils.traitlets import Instance
44 43
45 44 #-----------------------------------------------------------------------------
46 45 # Definitions of magic functions for use with IPython
47 46 #-----------------------------------------------------------------------------
48 47
49 48 NO_ACTIVE_VIEW = """
50 49 Use activate() on a DirectView object to activate it for magics.
51 50 """
52 51
53 52
54 class ParallelTricks(Plugin):
55 """A component to manage the %result, %px and %autopx magics."""
56
57 active_view = Instance('IPython.parallel.client.view.DirectView')
58 verbose = Bool(False, config=True)
59 shell = Instance('IPython.core.interactiveshell.InteractiveShellABC')
60
61 def __init__(self, shell=None, config=None):
62 super(ParallelTricks, self).__init__(shell=shell, config=config)
63 self.shell.register_magics(ParallelMagics)
64
65
66 53 @register_magics
67 54 class ParallelMagics(Magics):
68 55 """A set of magics useful when controlling a parallel IPython cluster.
69 56 """
57 active_view = Instance('IPython.parallel.client.view.DirectView')
58
70 59 def __init__(self, shell):
71 60 super(ParallelMagics, self).__init__(shell)
72 61 # A flag showing if autopx is activated or not
73 62 self.autopx = False
74 63
75 64 @skip_doctest
76 65 @line_magic
77 66 def result(self, parameter_s=''):
78 67 """Print the result of command i on all engines..
79 68
80 69 To use this a :class:`DirectView` instance must be created
81 70 and then activated by calling its :meth:`activate` method.
82 71
83 72 Then you can do the following::
84 73
85 74 In [23]: %result
86 75 Out[23]:
87 76 <Results List>
88 77 [0] In [6]: a = 10
89 78 [1] In [6]: a = 10
90 79
91 80 In [22]: %result 6
92 81 Out[22]:
93 82 <Results List>
94 83 [0] In [6]: a = 10
95 84 [1] In [6]: a = 10
96 85 """
97 86 if self.active_view is None:
98 87 print NO_ACTIVE_VIEW
99 88 return
100 89
101 90 try:
102 91 index = int(parameter_s)
103 92 except:
104 93 index = None
105 94 result = self.active_view.get_result(index)
106 95 return result
107 96
108 97 @skip_doctest
109 98 @line_magic
110 99 def px(self, parameter_s=''):
111 100 """Executes the given python command in parallel.
112 101
113 102 To use this a :class:`DirectView` instance must be created
114 103 and then activated by calling its :meth:`activate` method.
115 104
116 105 Then you can do the following::
117 106
118 107 In [24]: %px a = 5
119 108 Parallel execution on engine(s): all
120 109 Out[24]:
121 110 <Results List>
122 111 [0] In [7]: a = 5
123 112 [1] In [7]: a = 5
124 113 """
125 114
126 115 if self.active_view is None:
127 116 print NO_ACTIVE_VIEW
128 117 return
129 118 print "Parallel execution on engine(s): %s" % self.active_view.targets
130 119 result = self.active_view.execute(parameter_s, block=False)
131 120 if self.active_view.block:
132 121 result.get()
133 122 self._maybe_display_output(result)
134 123
135 124 @skip_doctest
136 125 @line_magic
137 126 def autopx(self, parameter_s=''):
138 127 """Toggles auto parallel mode.
139 128
140 129 To use this a :class:`DirectView` instance must be created
141 130 and then activated by calling its :meth:`activate` method. Once this
142 131 is called, all commands typed at the command line are send to
143 132 the engines to be executed in parallel. To control which engine
144 133 are used, set the ``targets`` attributed of the multiengine client
145 134 before entering ``%autopx`` mode.
146 135
147 136 Then you can do the following::
148 137
149 138 In [25]: %autopx
150 139 %autopx to enabled
151 140
152 141 In [26]: a = 10
153 142 Parallel execution on engine(s): [0,1,2,3]
154 143 In [27]: print a
155 144 Parallel execution on engine(s): [0,1,2,3]
156 145 [stdout:0] 10
157 146 [stdout:1] 10
158 147 [stdout:2] 10
159 148 [stdout:3] 10
160 149
161 150
162 151 In [27]: %autopx
163 152 %autopx disabled
164 153 """
165 154 if self.autopx:
166 155 self._disable_autopx()
167 156 else:
168 157 self._enable_autopx()
169 158
170 159 def _enable_autopx(self):
171 160 """Enable %autopx mode by saving the original run_cell and installing
172 161 pxrun_cell.
173 162 """
174 163 if self.active_view is None:
175 164 print NO_ACTIVE_VIEW
176 165 return
177 166
178 167 # override run_cell and run_code
179 168 self._original_run_cell = self.shell.run_cell
180 169 self.shell.run_cell = self.pxrun_cell
181 170 self._original_run_code = self.shell.run_code
182 171 self.shell.run_code = self.pxrun_code
183 172
184 173 self.autopx = True
185 174 print "%autopx enabled"
186 175
187 176 def _disable_autopx(self):
188 177 """Disable %autopx by restoring the original InteractiveShell.run_cell.
189 178 """
190 179 if self.autopx:
191 180 self.shell.run_cell = self._original_run_cell
192 181 self.shell.run_code = self._original_run_code
193 182 self.autopx = False
194 183 print "%autopx disabled"
195 184
196 185 def _maybe_display_output(self, result):
197 186 """Maybe display the output of a parallel result.
198 187
199 188 If self.active_view.block is True, wait for the result
200 189 and display the result. Otherwise, this is a noop.
201 190 """
202 191 if isinstance(result.stdout, basestring):
203 192 # single result
204 193 stdouts = [result.stdout.rstrip()]
205 194 else:
206 195 stdouts = [s.rstrip() for s in result.stdout]
207 196
208 197 targets = self.active_view.targets
209 198 if isinstance(targets, int):
210 199 targets = [targets]
211 200 elif targets == 'all':
212 201 targets = self.active_view.client.ids
213 202
214 203 if any(stdouts):
215 204 for eid,stdout in zip(targets, stdouts):
216 205 print '[stdout:%i]'%eid, stdout
217 206
218 207
219 208 def pxrun_cell(self, raw_cell, store_history=False, silent=False):
220 209 """drop-in replacement for InteractiveShell.run_cell.
221 210
222 211 This executes code remotely, instead of in the local namespace.
223 212
224 213 See InteractiveShell.run_cell for details.
225 214 """
226 215
227 216 if (not raw_cell) or raw_cell.isspace():
228 217 return
229 218
230 219 ipself = self.shell
231 220
232 221 with ipself.builtin_trap:
233 222 cell = ipself.prefilter_manager.prefilter_lines(raw_cell)
234 223
235 224 # Store raw and processed history
236 225 if store_history:
237 226 ipself.history_manager.store_inputs(ipself.execution_count,
238 227 cell, raw_cell)
239 228
240 229 # ipself.logger.log(cell, raw_cell)
241 230
242 231 cell_name = ipself.compile.cache(cell, ipself.execution_count)
243 232
244 233 try:
245 code_ast = ast.parse(cell, filename=cell_name)
246 except (OverflowError, SyntaxError, ValueError, TypeError, MemoryError):
234 ast.parse(cell, filename=cell_name)
235 except (OverflowError, SyntaxError, ValueError, TypeError,
236 MemoryError):
247 237 # Case 1
248 238 ipself.showsyntaxerror()
249 239 ipself.execution_count += 1
250 240 return None
251 241 except NameError:
252 242 # ignore name errors, because we don't know the remote keys
253 243 pass
254 244
255 245 if store_history:
256 246 # Write output to the database. Does nothing unless
257 247 # history output logging is enabled.
258 248 ipself.history_manager.store_output(ipself.execution_count)
259 249 # Each cell is a *single* input, regardless of how many lines it has
260 250 ipself.execution_count += 1
261 251 if re.search(r'get_ipython\(\)\.magic\(u?["\']%?autopx', cell):
262 252 self._disable_autopx()
263 253 return False
264 254 else:
265 255 try:
266 256 result = self.active_view.execute(cell, silent=False, block=False)
267 257 except:
268 258 ipself.showtraceback()
269 259 return True
270 260 else:
271 261 if self.active_view.block:
272 262 try:
273 263 result.get()
274 264 except:
275 265 self.shell.showtraceback()
276 266 return True
277 267 else:
278 268 self._maybe_display_output(result)
279 269 return False
280 270
281 271 def pxrun_code(self, code_obj):
282 272 """drop-in replacement for InteractiveShell.run_code.
283 273
284 274 This executes code remotely, instead of in the local namespace.
285 275
286 276 See InteractiveShell.run_code for details.
287 277 """
288 278 ipself = self.shell
289 279 # check code object for the autopx magic
290 280 if 'get_ipython' in code_obj.co_names and 'magic' in code_obj.co_names \
291 281 and any( [ isinstance(c, basestring) and 'autopx' in c
292 282 for c in code_obj.co_consts ]):
293 283 self._disable_autopx()
294 284 return False
295 285 else:
296 286 try:
297 287 result = self.active_view.execute(code_obj, block=False)
298 288 except:
299 289 ipself.showtraceback()
300 290 return True
301 291 else:
302 292 if self.active_view.block:
303 293 try:
304 294 result.get()
305 295 except:
306 296 self.shell.showtraceback()
307 297 return True
308 298 else:
309 299 self._maybe_display_output(result)
310 300 return False
311 301
312 302
313 303 __doc__ = __doc__.replace('@AUTOPX_DOC@',
314 304 " " + ParallelMagics.autopx.__doc__)
315 305 __doc__ = __doc__.replace('@PX_DOC@',
316 306 " " + ParallelMagics.px.__doc__)
317 307 __doc__ = __doc__.replace('@RESULT_DOC@',
318 308 " " + ParallelMagics.result.__doc__)
319 309
320
321 310 _loaded = False
322 311
323 312
324 313 def load_ipython_extension(ip):
325 314 """Load the extension in IPython."""
326 315 global _loaded
327 316 if not _loaded:
328 plugin = ParallelTricks(shell=ip, config=ip.config)
329 ip.plugin_manager.register_plugin('parallelmagic', plugin)
317 ip.register_magics(ParallelMagics)
330 318 _loaded = True
General Comments 0
You need to be logged in to leave comments. Login now