Show More
@@ -0,0 +1,390 | |||||
|
1 | # encoding: utf-8 | |||
|
2 | """ | |||
|
3 | ============= | |||
|
4 | parallelmagic | |||
|
5 | ============= | |||
|
6 | ||||
|
7 | Magic command interface for interactive parallel work. | |||
|
8 | ||||
|
9 | Usage | |||
|
10 | ===== | |||
|
11 | ||||
|
12 | ``%autopx`` | |||
|
13 | ||||
|
14 | {AUTOPX_DOC} | |||
|
15 | ||||
|
16 | ``%px`` | |||
|
17 | ||||
|
18 | {PX_DOC} | |||
|
19 | ||||
|
20 | ``%result`` | |||
|
21 | ||||
|
22 | {RESULT_DOC} | |||
|
23 | ||||
|
24 | """ | |||
|
25 | ||||
|
26 | #----------------------------------------------------------------------------- | |||
|
27 | # Copyright (C) 2008 The IPython Development Team | |||
|
28 | # | |||
|
29 | # Distributed under the terms of the BSD License. The full license is in | |||
|
30 | # the file COPYING, distributed as part of this software. | |||
|
31 | #----------------------------------------------------------------------------- | |||
|
32 | ||||
|
33 | #----------------------------------------------------------------------------- | |||
|
34 | # Imports | |||
|
35 | #----------------------------------------------------------------------------- | |||
|
36 | ||||
|
37 | import ast | |||
|
38 | import re | |||
|
39 | ||||
|
40 | from IPython.core.error import UsageError | |||
|
41 | from IPython.core.magic import Magics, magics_class, line_magic, cell_magic | |||
|
42 | from IPython.testing.skipdoctest import skip_doctest | |||
|
43 | ||||
|
44 | #----------------------------------------------------------------------------- | |||
|
45 | # Definitions of magic functions for use with IPython | |||
|
46 | #----------------------------------------------------------------------------- | |||
|
47 | ||||
|
48 | ||||
|
49 | NO_ACTIVE_VIEW = "Use activate() on a DirectView object to use it with magics." | |||
|
50 | NO_LAST_RESULT = "%result recalls last %px result, which has not yet been used." | |||
|
51 | ||||
|
52 | ||||
|
53 | @magics_class | |||
|
54 | class ParallelMagics(Magics): | |||
|
55 | """A set of magics useful when controlling a parallel IPython cluster. | |||
|
56 | """ | |||
|
57 | ||||
|
58 | # A flag showing if autopx is activated or not | |||
|
59 | _autopx = False | |||
|
60 | # the current view used by the magics: | |||
|
61 | active_view = None | |||
|
62 | # last result cache for %result | |||
|
63 | last_result = None | |||
|
64 | ||||
|
65 | @skip_doctest | |||
|
66 | @line_magic | |||
|
67 | def result(self, line=''): | |||
|
68 | """Print the result of the last asynchronous %px command. | |||
|
69 | ||||
|
70 | Usage: | |||
|
71 | ||||
|
72 | %result [-o] [-e] [--group-options=type|engine|order] | |||
|
73 | ||||
|
74 | Options: | |||
|
75 | ||||
|
76 | -o: collate outputs in order (same as group-outputs=order) | |||
|
77 | ||||
|
78 | -e: group outputs by engine (same as group-outputs=engine) | |||
|
79 | ||||
|
80 | --group-outputs=type [default behavior]: | |||
|
81 | each output type (stdout, stderr, displaypub) for all engines | |||
|
82 | displayed together. | |||
|
83 | ||||
|
84 | --group-outputs=order: | |||
|
85 | The same as 'type', but individual displaypub outputs (e.g. plots) | |||
|
86 | will be interleaved, so it will display all of the first plots, | |||
|
87 | then all of the second plots, etc. | |||
|
88 | ||||
|
89 | --group-outputs=engine: | |||
|
90 | All of an engine's output is displayed before moving on to the next. | |||
|
91 | ||||
|
92 | To use this a :class:`DirectView` instance must be created | |||
|
93 | and then activated by calling its :meth:`activate` method. | |||
|
94 | ||||
|
95 | This lets you recall the results of %px computations after | |||
|
96 | asynchronous submission (view.block=False). | |||
|
97 | ||||
|
98 | Then you can do the following:: | |||
|
99 | ||||
|
100 | In [23]: %px os.getpid() | |||
|
101 | Async parallel execution on engine(s): all | |||
|
102 | ||||
|
103 | In [24]: %result | |||
|
104 | [ 8] Out[10]: 60920 | |||
|
105 | [ 9] Out[10]: 60921 | |||
|
106 | [10] Out[10]: 60922 | |||
|
107 | [11] Out[10]: 60923 | |||
|
108 | """ | |||
|
109 | opts, _ = self.parse_options(line, 'oe', 'group-outputs=') | |||
|
110 | ||||
|
111 | if 'group-outputs' in opts: | |||
|
112 | groupby = opts['group-outputs'] | |||
|
113 | elif 'o' in opts: | |||
|
114 | groupby = 'order' | |||
|
115 | elif 'e' in opts: | |||
|
116 | groupby = 'engine' | |||
|
117 | else: | |||
|
118 | groupby = 'type' | |||
|
119 | ||||
|
120 | if self.active_view is None: | |||
|
121 | raise UsageError(NO_ACTIVE_VIEW) | |||
|
122 | ||||
|
123 | if self.last_result is None: | |||
|
124 | raise UsageError(NO_LAST_RESULT) | |||
|
125 | ||||
|
126 | self.last_result.get() | |||
|
127 | self.last_result.display_outputs(groupby=groupby) | |||
|
128 | ||||
|
129 | @skip_doctest | |||
|
130 | @line_magic | |||
|
131 | def px(self, parameter_s=''): | |||
|
132 | """Executes the given python command in parallel. | |||
|
133 | ||||
|
134 | To use this a :class:`DirectView` instance must be created | |||
|
135 | and then activated by calling its :meth:`activate` method. | |||
|
136 | ||||
|
137 | Then you can do the following:: | |||
|
138 | ||||
|
139 | In [24]: %px a = os.getpid() | |||
|
140 | Parallel execution on engine(s): all | |||
|
141 | ||||
|
142 | In [25]: %px print a | |||
|
143 | [stdout:0] 1234 | |||
|
144 | [stdout:1] 1235 | |||
|
145 | [stdout:2] 1236 | |||
|
146 | [stdout:3] 1237 | |||
|
147 | """ | |||
|
148 | return self.parallel_execute(parameter_s) | |||
|
149 | ||||
|
150 | def parallel_execute(self, cell, block=None, groupby='type', save_name=None): | |||
|
151 | """implementation used by %px and %%parallel""" | |||
|
152 | ||||
|
153 | if self.active_view is None: | |||
|
154 | raise UsageError(NO_ACTIVE_VIEW) | |||
|
155 | ||||
|
156 | # defaults: | |||
|
157 | block = self.active_view.block if block is None else block | |||
|
158 | ||||
|
159 | base = "Parallel" if block else "Async parallel" | |||
|
160 | ||||
|
161 | targets = self.active_view.targets | |||
|
162 | if isinstance(targets, list) and len(targets) > 10: | |||
|
163 | str_targets = str(targets[:4])[:-1] + ', ..., ' + str(targets[-4:])[1:] | |||
|
164 | else: | |||
|
165 | str_targets = str(targets) | |||
|
166 | print base + " execution on engine(s): %s" % str_targets | |||
|
167 | ||||
|
168 | result = self.active_view.execute(cell, silent=False, block=False) | |||
|
169 | self.last_result = result | |||
|
170 | ||||
|
171 | if save_name: | |||
|
172 | self.shell.user_ns[save_name] = result | |||
|
173 | ||||
|
174 | if block: | |||
|
175 | result.get() | |||
|
176 | result.display_outputs(groupby) | |||
|
177 | else: | |||
|
178 | # return AsyncResult only on non-blocking submission | |||
|
179 | return result | |||
|
180 | ||||
|
181 | @skip_doctest | |||
|
182 | @cell_magic('px') | |||
|
183 | def cell_px(self, line='', cell=None): | |||
|
184 | """Executes the given python command in parallel. | |||
|
185 | ||||
|
186 | Cell magic usage: | |||
|
187 | ||||
|
188 | %%px [-o] [-e] [--group-options=type|engine|order] [--[no]block] [--out name] | |||
|
189 | ||||
|
190 | Options: | |||
|
191 | ||||
|
192 | --out <name>: store the AsyncResult object for this computation | |||
|
193 | in the global namespace. | |||
|
194 | ||||
|
195 | -o: collate outputs in order (same as group-outputs=order) | |||
|
196 | ||||
|
197 | -e: group outputs by engine (same as group-outputs=engine) | |||
|
198 | ||||
|
199 | --group-outputs=type [default behavior]: | |||
|
200 | each output type (stdout, stderr, displaypub) for all engines | |||
|
201 | displayed together. | |||
|
202 | ||||
|
203 | --group-outputs=order: | |||
|
204 | The same as 'type', but individual displaypub outputs (e.g. plots) | |||
|
205 | will be interleaved, so it will display all of the first plots, | |||
|
206 | then all of the second plots, etc. | |||
|
207 | ||||
|
208 | --group-outputs=engine: | |||
|
209 | All of an engine's output is displayed before moving on to the next. | |||
|
210 | ||||
|
211 | --[no]block: | |||
|
212 | Whether or not to block for the execution to complete | |||
|
213 | (and display the results). If unspecified, the active view's | |||
|
214 | ||||
|
215 | ||||
|
216 | To use this a :class:`DirectView` instance must be created | |||
|
217 | and then activated by calling its :meth:`activate` method. | |||
|
218 | ||||
|
219 | Then you can do the following:: | |||
|
220 | ||||
|
221 | In [24]: %%px --noblock | |||
|
222 | ....: a = os.getpid() | |||
|
223 | Async parallel execution on engine(s): all | |||
|
224 | ||||
|
225 | In [25]: %%px | |||
|
226 | ....: print a | |||
|
227 | [stdout:0] 1234 | |||
|
228 | [stdout:1] 1235 | |||
|
229 | [stdout:2] 1236 | |||
|
230 | [stdout:3] 1237 | |||
|
231 | """ | |||
|
232 | ||||
|
233 | block = None | |||
|
234 | groupby = 'type' | |||
|
235 | # as a cell magic, we accept args | |||
|
236 | opts, _ = self.parse_options(line, 'oe', 'group-outputs=', 'out=', 'block', 'noblock') | |||
|
237 | ||||
|
238 | if 'group-outputs' in opts: | |||
|
239 | groupby = opts['group-outputs'] | |||
|
240 | elif 'o' in opts: | |||
|
241 | groupby = 'order' | |||
|
242 | elif 'e' in opts: | |||
|
243 | groupby = 'engine' | |||
|
244 | ||||
|
245 | if 'block' in opts: | |||
|
246 | block = True | |||
|
247 | elif 'noblock' in opts: | |||
|
248 | block = False | |||
|
249 | ||||
|
250 | save_name = opts.get('out') | |||
|
251 | ||||
|
252 | return self.parallel_execute(cell, block=block, groupby=groupby, save_name=save_name) | |||
|
253 | ||||
|
254 | @skip_doctest | |||
|
255 | @line_magic | |||
|
256 | def autopx(self, parameter_s=''): | |||
|
257 | """Toggles auto parallel mode. | |||
|
258 | ||||
|
259 | To use this a :class:`DirectView` instance must be created | |||
|
260 | and then activated by calling its :meth:`activate` method. Once this | |||
|
261 | is called, all commands typed at the command line are send to | |||
|
262 | the engines to be executed in parallel. To control which engine | |||
|
263 | are used, set the ``targets`` attributed of the multiengine client | |||
|
264 | before entering ``%autopx`` mode. | |||
|
265 | ||||
|
266 | Then you can do the following:: | |||
|
267 | ||||
|
268 | In [25]: %autopx | |||
|
269 | %autopx to enabled | |||
|
270 | ||||
|
271 | In [26]: a = 10 | |||
|
272 | Parallel execution on engine(s): [0,1,2,3] | |||
|
273 | In [27]: print a | |||
|
274 | Parallel execution on engine(s): [0,1,2,3] | |||
|
275 | [stdout:0] 10 | |||
|
276 | [stdout:1] 10 | |||
|
277 | [stdout:2] 10 | |||
|
278 | [stdout:3] 10 | |||
|
279 | ||||
|
280 | ||||
|
281 | In [27]: %autopx | |||
|
282 | %autopx disabled | |||
|
283 | """ | |||
|
284 | if self._autopx: | |||
|
285 | self._disable_autopx() | |||
|
286 | else: | |||
|
287 | self._enable_autopx() | |||
|
288 | ||||
|
289 | def _enable_autopx(self): | |||
|
290 | """Enable %autopx mode by saving the original run_cell and installing | |||
|
291 | pxrun_cell. | |||
|
292 | """ | |||
|
293 | if self.active_view is None: | |||
|
294 | raise UsageError(NO_ACTIVE_VIEW) | |||
|
295 | ||||
|
296 | # override run_cell | |||
|
297 | self._original_run_cell = self.shell.run_cell | |||
|
298 | self.shell.run_cell = self.pxrun_cell | |||
|
299 | ||||
|
300 | self._autopx = True | |||
|
301 | print "%autopx enabled" | |||
|
302 | ||||
|
303 | def _disable_autopx(self): | |||
|
304 | """Disable %autopx by restoring the original InteractiveShell.run_cell. | |||
|
305 | """ | |||
|
306 | if self._autopx: | |||
|
307 | self.shell.run_cell = self._original_run_cell | |||
|
308 | self._autopx = False | |||
|
309 | print "%autopx disabled" | |||
|
310 | ||||
|
311 | def pxrun_cell(self, raw_cell, store_history=False, silent=False): | |||
|
312 | """drop-in replacement for InteractiveShell.run_cell. | |||
|
313 | ||||
|
314 | This executes code remotely, instead of in the local namespace. | |||
|
315 | ||||
|
316 | See InteractiveShell.run_cell for details. | |||
|
317 | """ | |||
|
318 | ||||
|
319 | if (not raw_cell) or raw_cell.isspace(): | |||
|
320 | return | |||
|
321 | ||||
|
322 | ipself = self.shell | |||
|
323 | ||||
|
324 | with ipself.builtin_trap: | |||
|
325 | cell = ipself.prefilter_manager.prefilter_lines(raw_cell) | |||
|
326 | ||||
|
327 | # Store raw and processed history | |||
|
328 | if store_history: | |||
|
329 | ipself.history_manager.store_inputs(ipself.execution_count, | |||
|
330 | cell, raw_cell) | |||
|
331 | ||||
|
332 | # ipself.logger.log(cell, raw_cell) | |||
|
333 | ||||
|
334 | cell_name = ipself.compile.cache(cell, ipself.execution_count) | |||
|
335 | ||||
|
336 | try: | |||
|
337 | ast.parse(cell, filename=cell_name) | |||
|
338 | except (OverflowError, SyntaxError, ValueError, TypeError, | |||
|
339 | MemoryError): | |||
|
340 | # Case 1 | |||
|
341 | ipself.showsyntaxerror() | |||
|
342 | ipself.execution_count += 1 | |||
|
343 | return None | |||
|
344 | except NameError: | |||
|
345 | # ignore name errors, because we don't know the remote keys | |||
|
346 | pass | |||
|
347 | ||||
|
348 | if store_history: | |||
|
349 | # Write output to the database. Does nothing unless | |||
|
350 | # history output logging is enabled. | |||
|
351 | ipself.history_manager.store_output(ipself.execution_count) | |||
|
352 | # Each cell is a *single* input, regardless of how many lines it has | |||
|
353 | ipself.execution_count += 1 | |||
|
354 | if re.search(r'get_ipython\(\)\.magic\(u?["\']%?autopx', cell): | |||
|
355 | self._disable_autopx() | |||
|
356 | return False | |||
|
357 | else: | |||
|
358 | try: | |||
|
359 | result = self.active_view.execute(cell, silent=False, block=False) | |||
|
360 | except: | |||
|
361 | ipself.showtraceback() | |||
|
362 | return True | |||
|
363 | else: | |||
|
364 | if self.active_view.block: | |||
|
365 | try: | |||
|
366 | result.get() | |||
|
367 | except: | |||
|
368 | self.shell.showtraceback() | |||
|
369 | return True | |||
|
370 | else: | |||
|
371 | with ipself.builtin_trap: | |||
|
372 | result.display_outputs() | |||
|
373 | return False | |||
|
374 | ||||
|
375 | ||||
|
376 | __doc__ = __doc__.format( | |||
|
377 | AUTOPX_DOC = ' '*8 + ParallelMagics.autopx.__doc__, | |||
|
378 | PX_DOC = ' '*8 + ParallelMagics.px.__doc__, | |||
|
379 | RESULT_DOC = ' '*8 + ParallelMagics.result.__doc__ | |||
|
380 | ) | |||
|
381 | ||||
|
382 | _loaded = False | |||
|
383 | ||||
|
384 | ||||
|
385 | def load_ipython_extension(ip): | |||
|
386 | """Load the extension in IPython.""" | |||
|
387 | global _loaded | |||
|
388 | if not _loaded: | |||
|
389 | ip.register_magics(ParallelMagics) | |||
|
390 | _loaded = True |
@@ -1,390 +1,20 | |||||
1 | # encoding: utf-8 |
|
1 | # encoding: utf-8 | |
2 | """ |
|
2 | """ | |
3 | ============= |
|
3 | ============= | |
4 | parallelmagic |
|
4 | parallelmagic | |
5 | ============= |
|
5 | ============= | |
6 |
|
6 | |||
7 | Magic command interface for interactive parallel work. |
|
7 | Deprecated, parallel magics are no longer an extension. | |
8 |
|
||||
9 | Usage |
|
|||
10 | ===== |
|
|||
11 |
|
||||
12 | ``%autopx`` |
|
|||
13 |
|
||||
14 | {AUTOPX_DOC} |
|
|||
15 |
|
||||
16 | ``%px`` |
|
|||
17 |
|
||||
18 | {PX_DOC} |
|
|||
19 |
|
||||
20 | ``%result`` |
|
|||
21 |
|
||||
22 | {RESULT_DOC} |
|
|||
23 |
|
||||
24 | """ |
|
8 | """ | |
25 |
|
9 | |||
26 | #----------------------------------------------------------------------------- |
|
10 | #----------------------------------------------------------------------------- | |
27 | # Copyright (C) 2008 The IPython Development Team |
|
11 | # Copyright (C) 2008 The IPython Development Team | |
28 | # |
|
12 | # | |
29 | # Distributed under the terms of the BSD License. The full license is in |
|
13 | # Distributed under the terms of the BSD License. The full license is in | |
30 | # the file COPYING, distributed as part of this software. |
|
14 | # the file COPYING, distributed as part of this software. | |
31 | #----------------------------------------------------------------------------- |
|
15 | #----------------------------------------------------------------------------- | |
32 |
|
16 | |||
33 | #----------------------------------------------------------------------------- |
|
17 | from warnings import warn | |
34 | # Imports |
|
|||
35 | #----------------------------------------------------------------------------- |
|
|||
36 |
|
||||
37 | import ast |
|
|||
38 | import re |
|
|||
39 |
|
||||
40 | from IPython.core.error import UsageError |
|
|||
41 | from IPython.core.magic import Magics, magics_class, line_magic, cell_magic |
|
|||
42 | from IPython.testing.skipdoctest import skip_doctest |
|
|||
43 |
|
||||
44 | #----------------------------------------------------------------------------- |
|
|||
45 | # Definitions of magic functions for use with IPython |
|
|||
46 | #----------------------------------------------------------------------------- |
|
|||
47 |
|
||||
48 |
|
||||
49 | NO_ACTIVE_VIEW = "Use activate() on a DirectView object to use it with magics." |
|
|||
50 | NO_LAST_RESULT = "%result recalls last %px result, which has not yet been used." |
|
|||
51 |
|
||||
52 |
|
||||
53 | @magics_class |
|
|||
54 | class ParallelMagics(Magics): |
|
|||
55 | """A set of magics useful when controlling a parallel IPython cluster. |
|
|||
56 | """ |
|
|||
57 |
|
||||
58 | # A flag showing if autopx is activated or not |
|
|||
59 | _autopx = False |
|
|||
60 | # the current view used by the magics: |
|
|||
61 | active_view = None |
|
|||
62 | # last result cache for %result |
|
|||
63 | last_result = None |
|
|||
64 |
|
||||
65 | @skip_doctest |
|
|||
66 | @line_magic |
|
|||
67 | def result(self, line=''): |
|
|||
68 | """Print the result of the last asynchronous %px command. |
|
|||
69 |
|
||||
70 | Usage: |
|
|||
71 |
|
||||
72 | %result [-o] [-e] [--group-options=type|engine|order] |
|
|||
73 |
|
||||
74 | Options: |
|
|||
75 |
|
||||
76 | -o: collate outputs in order (same as group-outputs=order) |
|
|||
77 |
|
||||
78 | -e: group outputs by engine (same as group-outputs=engine) |
|
|||
79 |
|
||||
80 | --group-outputs=type [default behavior]: |
|
|||
81 | each output type (stdout, stderr, displaypub) for all engines |
|
|||
82 | displayed together. |
|
|||
83 |
|
||||
84 | --group-outputs=order: |
|
|||
85 | The same as 'type', but individual displaypub outputs (e.g. plots) |
|
|||
86 | will be interleaved, so it will display all of the first plots, |
|
|||
87 | then all of the second plots, etc. |
|
|||
88 |
|
||||
89 | --group-outputs=engine: |
|
|||
90 | All of an engine's output is displayed before moving on to the next. |
|
|||
91 |
|
||||
92 | To use this a :class:`DirectView` instance must be created |
|
|||
93 | and then activated by calling its :meth:`activate` method. |
|
|||
94 |
|
||||
95 | This lets you recall the results of %px computations after |
|
|||
96 | asynchronous submission (view.block=False). |
|
|||
97 |
|
||||
98 | Then you can do the following:: |
|
|||
99 |
|
||||
100 | In [23]: %px os.getpid() |
|
|||
101 | Async parallel execution on engine(s): all |
|
|||
102 |
|
||||
103 | In [24]: %result |
|
|||
104 | [ 8] Out[10]: 60920 |
|
|||
105 | [ 9] Out[10]: 60921 |
|
|||
106 | [10] Out[10]: 60922 |
|
|||
107 | [11] Out[10]: 60923 |
|
|||
108 | """ |
|
|||
109 | opts, _ = self.parse_options(line, 'oe', 'group-outputs=') |
|
|||
110 |
|
||||
111 | if 'group-outputs' in opts: |
|
|||
112 | groupby = opts['group-outputs'] |
|
|||
113 | elif 'o' in opts: |
|
|||
114 | groupby = 'order' |
|
|||
115 | elif 'e' in opts: |
|
|||
116 | groupby = 'engine' |
|
|||
117 | else: |
|
|||
118 | groupby = 'type' |
|
|||
119 |
|
||||
120 | if self.active_view is None: |
|
|||
121 | raise UsageError(NO_ACTIVE_VIEW) |
|
|||
122 |
|
||||
123 | if self.last_result is None: |
|
|||
124 | raise UsageError(NO_LAST_RESULT) |
|
|||
125 |
|
||||
126 | self.last_result.get() |
|
|||
127 | self.last_result.display_outputs(groupby=groupby) |
|
|||
128 |
|
||||
129 | @skip_doctest |
|
|||
130 | @line_magic |
|
|||
131 | def px(self, parameter_s=''): |
|
|||
132 | """Executes the given python command in parallel. |
|
|||
133 |
|
||||
134 | To use this a :class:`DirectView` instance must be created |
|
|||
135 | and then activated by calling its :meth:`activate` method. |
|
|||
136 |
|
||||
137 | Then you can do the following:: |
|
|||
138 |
|
||||
139 | In [24]: %px a = os.getpid() |
|
|||
140 | Parallel execution on engine(s): all |
|
|||
141 |
|
||||
142 | In [25]: %px print a |
|
|||
143 | [stdout:0] 1234 |
|
|||
144 | [stdout:1] 1235 |
|
|||
145 | [stdout:2] 1236 |
|
|||
146 | [stdout:3] 1237 |
|
|||
147 | """ |
|
|||
148 | return self.parallel_execute(parameter_s) |
|
|||
149 |
|
||||
150 | def parallel_execute(self, cell, block=None, groupby='type', save_name=None): |
|
|||
151 | """implementation used by %px and %%parallel""" |
|
|||
152 |
|
||||
153 | if self.active_view is None: |
|
|||
154 | raise UsageError(NO_ACTIVE_VIEW) |
|
|||
155 |
|
||||
156 | # defaults: |
|
|||
157 | block = self.active_view.block if block is None else block |
|
|||
158 |
|
||||
159 | base = "Parallel" if block else "Async parallel" |
|
|||
160 |
|
||||
161 | targets = self.active_view.targets |
|
|||
162 | if isinstance(targets, list) and len(targets) > 10: |
|
|||
163 | str_targets = str(targets[:4])[:-1] + ', ..., ' + str(targets[-4:])[1:] |
|
|||
164 | else: |
|
|||
165 | str_targets = str(targets) |
|
|||
166 | print base + " execution on engine(s): %s" % str_targets |
|
|||
167 |
|
||||
168 | result = self.active_view.execute(cell, silent=False, block=False) |
|
|||
169 | self.last_result = result |
|
|||
170 |
|
||||
171 | if save_name: |
|
|||
172 | self.shell.user_ns[save_name] = result |
|
|||
173 |
|
||||
174 | if block: |
|
|||
175 | result.get() |
|
|||
176 | result.display_outputs(groupby) |
|
|||
177 | else: |
|
|||
178 | # return AsyncResult only on non-blocking submission |
|
|||
179 | return result |
|
|||
180 |
|
||||
181 | @skip_doctest |
|
|||
182 | @cell_magic('px') |
|
|||
183 | def cell_px(self, line='', cell=None): |
|
|||
184 | """Executes the given python command in parallel. |
|
|||
185 |
|
||||
186 | Cell magic usage: |
|
|||
187 |
|
||||
188 | %%px [-o] [-e] [--group-options=type|engine|order] [--[no]block] [--out name] |
|
|||
189 |
|
||||
190 | Options: |
|
|||
191 |
|
||||
192 | --out <name>: store the AsyncResult object for this computation |
|
|||
193 | in the global namespace. |
|
|||
194 |
|
||||
195 | -o: collate outputs in order (same as group-outputs=order) |
|
|||
196 |
|
||||
197 | -e: group outputs by engine (same as group-outputs=engine) |
|
|||
198 |
|
||||
199 | --group-outputs=type [default behavior]: |
|
|||
200 | each output type (stdout, stderr, displaypub) for all engines |
|
|||
201 | displayed together. |
|
|||
202 |
|
||||
203 | --group-outputs=order: |
|
|||
204 | The same as 'type', but individual displaypub outputs (e.g. plots) |
|
|||
205 | will be interleaved, so it will display all of the first plots, |
|
|||
206 | then all of the second plots, etc. |
|
|||
207 |
|
||||
208 | --group-outputs=engine: |
|
|||
209 | All of an engine's output is displayed before moving on to the next. |
|
|||
210 |
|
||||
211 | --[no]block: |
|
|||
212 | Whether or not to block for the execution to complete |
|
|||
213 | (and display the results). If unspecified, the active view's |
|
|||
214 |
|
||||
215 |
|
||||
216 | To use this a :class:`DirectView` instance must be created |
|
|||
217 | and then activated by calling its :meth:`activate` method. |
|
|||
218 |
|
||||
219 | Then you can do the following:: |
|
|||
220 |
|
||||
221 | In [24]: %%px --noblock |
|
|||
222 | ....: a = os.getpid() |
|
|||
223 | Async parallel execution on engine(s): all |
|
|||
224 |
|
||||
225 | In [25]: %%px |
|
|||
226 | ....: print a |
|
|||
227 | [stdout:0] 1234 |
|
|||
228 | [stdout:1] 1235 |
|
|||
229 | [stdout:2] 1236 |
|
|||
230 | [stdout:3] 1237 |
|
|||
231 | """ |
|
|||
232 |
|
||||
233 | block = None |
|
|||
234 | groupby = 'type' |
|
|||
235 | # as a cell magic, we accept args |
|
|||
236 | opts, _ = self.parse_options(line, 'oe', 'group-outputs=', 'out=', 'block', 'noblock') |
|
|||
237 |
|
||||
238 | if 'group-outputs' in opts: |
|
|||
239 | groupby = opts['group-outputs'] |
|
|||
240 | elif 'o' in opts: |
|
|||
241 | groupby = 'order' |
|
|||
242 | elif 'e' in opts: |
|
|||
243 | groupby = 'engine' |
|
|||
244 |
|
||||
245 | if 'block' in opts: |
|
|||
246 | block = True |
|
|||
247 | elif 'noblock' in opts: |
|
|||
248 | block = False |
|
|||
249 |
|
||||
250 | save_name = opts.get('out') |
|
|||
251 |
|
||||
252 | return self.parallel_execute(cell, block=block, groupby=groupby, save_name=save_name) |
|
|||
253 |
|
||||
254 | @skip_doctest |
|
|||
255 | @line_magic |
|
|||
256 | def autopx(self, parameter_s=''): |
|
|||
257 | """Toggles auto parallel mode. |
|
|||
258 |
|
||||
259 | To use this a :class:`DirectView` instance must be created |
|
|||
260 | and then activated by calling its :meth:`activate` method. Once this |
|
|||
261 | is called, all commands typed at the command line are send to |
|
|||
262 | the engines to be executed in parallel. To control which engine |
|
|||
263 | are used, set the ``targets`` attributed of the multiengine client |
|
|||
264 | before entering ``%autopx`` mode. |
|
|||
265 |
|
||||
266 | Then you can do the following:: |
|
|||
267 |
|
||||
268 | In [25]: %autopx |
|
|||
269 | %autopx to enabled |
|
|||
270 |
|
||||
271 | In [26]: a = 10 |
|
|||
272 | Parallel execution on engine(s): [0,1,2,3] |
|
|||
273 | In [27]: print a |
|
|||
274 | Parallel execution on engine(s): [0,1,2,3] |
|
|||
275 | [stdout:0] 10 |
|
|||
276 | [stdout:1] 10 |
|
|||
277 | [stdout:2] 10 |
|
|||
278 | [stdout:3] 10 |
|
|||
279 |
|
||||
280 |
|
||||
281 | In [27]: %autopx |
|
|||
282 | %autopx disabled |
|
|||
283 | """ |
|
|||
284 | if self._autopx: |
|
|||
285 | self._disable_autopx() |
|
|||
286 | else: |
|
|||
287 | self._enable_autopx() |
|
|||
288 |
|
||||
289 | def _enable_autopx(self): |
|
|||
290 | """Enable %autopx mode by saving the original run_cell and installing |
|
|||
291 | pxrun_cell. |
|
|||
292 | """ |
|
|||
293 | if self.active_view is None: |
|
|||
294 | raise UsageError(NO_ACTIVE_VIEW) |
|
|||
295 |
|
||||
296 | # override run_cell |
|
|||
297 | self._original_run_cell = self.shell.run_cell |
|
|||
298 | self.shell.run_cell = self.pxrun_cell |
|
|||
299 |
|
||||
300 | self._autopx = True |
|
|||
301 | print "%autopx enabled" |
|
|||
302 |
|
||||
303 | def _disable_autopx(self): |
|
|||
304 | """Disable %autopx by restoring the original InteractiveShell.run_cell. |
|
|||
305 | """ |
|
|||
306 | if self._autopx: |
|
|||
307 | self.shell.run_cell = self._original_run_cell |
|
|||
308 | self._autopx = False |
|
|||
309 | print "%autopx disabled" |
|
|||
310 |
|
||||
311 | def pxrun_cell(self, raw_cell, store_history=False, silent=False): |
|
|||
312 | """drop-in replacement for InteractiveShell.run_cell. |
|
|||
313 |
|
||||
314 | This executes code remotely, instead of in the local namespace. |
|
|||
315 |
|
||||
316 | See InteractiveShell.run_cell for details. |
|
|||
317 | """ |
|
|||
318 |
|
||||
319 | if (not raw_cell) or raw_cell.isspace(): |
|
|||
320 | return |
|
|||
321 |
|
||||
322 | ipself = self.shell |
|
|||
323 |
|
||||
324 | with ipself.builtin_trap: |
|
|||
325 | cell = ipself.prefilter_manager.prefilter_lines(raw_cell) |
|
|||
326 |
|
||||
327 | # Store raw and processed history |
|
|||
328 | if store_history: |
|
|||
329 | ipself.history_manager.store_inputs(ipself.execution_count, |
|
|||
330 | cell, raw_cell) |
|
|||
331 |
|
||||
332 | # ipself.logger.log(cell, raw_cell) |
|
|||
333 |
|
||||
334 | cell_name = ipself.compile.cache(cell, ipself.execution_count) |
|
|||
335 |
|
||||
336 | try: |
|
|||
337 | ast.parse(cell, filename=cell_name) |
|
|||
338 | except (OverflowError, SyntaxError, ValueError, TypeError, |
|
|||
339 | MemoryError): |
|
|||
340 | # Case 1 |
|
|||
341 | ipself.showsyntaxerror() |
|
|||
342 | ipself.execution_count += 1 |
|
|||
343 | return None |
|
|||
344 | except NameError: |
|
|||
345 | # ignore name errors, because we don't know the remote keys |
|
|||
346 | pass |
|
|||
347 |
|
||||
348 | if store_history: |
|
|||
349 | # Write output to the database. Does nothing unless |
|
|||
350 | # history output logging is enabled. |
|
|||
351 | ipself.history_manager.store_output(ipself.execution_count) |
|
|||
352 | # Each cell is a *single* input, regardless of how many lines it has |
|
|||
353 | ipself.execution_count += 1 |
|
|||
354 | if re.search(r'get_ipython\(\)\.magic\(u?["\']%?autopx', cell): |
|
|||
355 | self._disable_autopx() |
|
|||
356 | return False |
|
|||
357 | else: |
|
|||
358 | try: |
|
|||
359 | result = self.active_view.execute(cell, silent=False, block=False) |
|
|||
360 | except: |
|
|||
361 | ipself.showtraceback() |
|
|||
362 | return True |
|
|||
363 | else: |
|
|||
364 | if self.active_view.block: |
|
|||
365 | try: |
|
|||
366 | result.get() |
|
|||
367 | except: |
|
|||
368 | self.shell.showtraceback() |
|
|||
369 | return True |
|
|||
370 | else: |
|
|||
371 | with ipself.builtin_trap: |
|
|||
372 | result.display_outputs() |
|
|||
373 | return False |
|
|||
374 |
|
||||
375 |
|
||||
376 | __doc__ = __doc__.format( |
|
|||
377 | AUTOPX_DOC = ' '*8 + ParallelMagics.autopx.__doc__, |
|
|||
378 | PX_DOC = ' '*8 + ParallelMagics.px.__doc__, |
|
|||
379 | RESULT_DOC = ' '*8 + ParallelMagics.result.__doc__ |
|
|||
380 | ) |
|
|||
381 |
|
||||
382 | _loaded = False |
|
|||
383 |
|
||||
384 |
|
18 | |||
385 | def load_ipython_extension(ip): |
|
19 | def load_ipython_extension(ip): | |
386 | """Load the extension in IPython.""" |
|
20 | warn("Parallel Magics are no longer defined in an extension", DeprecationWarning) | |
387 | global _loaded |
|
|||
388 | if not _loaded: |
|
|||
389 | ip.register_magics(ParallelMagics) |
|
|||
390 | _loaded = True |
|
General Comments 0
You need to be logged in to leave comments.
Login now