Show More
@@ -0,0 +1,390 | |||
|
1 | # encoding: utf-8 | |
|
2 | """ | |
|
3 | ============= | |
|
4 | parallelmagic | |
|
5 | ============= | |
|
6 | ||
|
7 | Magic command interface for interactive parallel work. | |
|
8 | ||
|
9 | Usage | |
|
10 | ===== | |
|
11 | ||
|
12 | ``%autopx`` | |
|
13 | ||
|
14 | {AUTOPX_DOC} | |
|
15 | ||
|
16 | ``%px`` | |
|
17 | ||
|
18 | {PX_DOC} | |
|
19 | ||
|
20 | ``%result`` | |
|
21 | ||
|
22 | {RESULT_DOC} | |
|
23 | ||
|
24 | """ | |
|
25 | ||
|
26 | #----------------------------------------------------------------------------- | |
|
27 | # Copyright (C) 2008 The IPython Development Team | |
|
28 | # | |
|
29 | # Distributed under the terms of the BSD License. The full license is in | |
|
30 | # the file COPYING, distributed as part of this software. | |
|
31 | #----------------------------------------------------------------------------- | |
|
32 | ||
|
33 | #----------------------------------------------------------------------------- | |
|
34 | # Imports | |
|
35 | #----------------------------------------------------------------------------- | |
|
36 | ||
|
37 | import ast | |
|
38 | import re | |
|
39 | ||
|
40 | from IPython.core.error import UsageError | |
|
41 | from IPython.core.magic import Magics, magics_class, line_magic, cell_magic | |
|
42 | from IPython.testing.skipdoctest import skip_doctest | |
|
43 | ||
|
44 | #----------------------------------------------------------------------------- | |
|
45 | # Definitions of magic functions for use with IPython | |
|
46 | #----------------------------------------------------------------------------- | |
|
47 | ||
|
48 | ||
|
49 | NO_ACTIVE_VIEW = "Use activate() on a DirectView object to use it with magics." | |
|
50 | NO_LAST_RESULT = "%result recalls last %px result, which has not yet been used." | |
|
51 | ||
|
52 | ||
|
53 | @magics_class | |
|
54 | class ParallelMagics(Magics): | |
|
55 | """A set of magics useful when controlling a parallel IPython cluster. | |
|
56 | """ | |
|
57 | ||
|
58 | # A flag showing if autopx is activated or not | |
|
59 | _autopx = False | |
|
60 | # the current view used by the magics: | |
|
61 | active_view = None | |
|
62 | # last result cache for %result | |
|
63 | last_result = None | |
|
64 | ||
|
65 | @skip_doctest | |
|
66 | @line_magic | |
|
67 | def result(self, line=''): | |
|
68 | """Print the result of the last asynchronous %px command. | |
|
69 | ||
|
70 | Usage: | |
|
71 | ||
|
72 | %result [-o] [-e] [--group-options=type|engine|order] | |
|
73 | ||
|
74 | Options: | |
|
75 | ||
|
76 | -o: collate outputs in order (same as group-outputs=order) | |
|
77 | ||
|
78 | -e: group outputs by engine (same as group-outputs=engine) | |
|
79 | ||
|
80 | --group-outputs=type [default behavior]: | |
|
81 | each output type (stdout, stderr, displaypub) for all engines | |
|
82 | displayed together. | |
|
83 | ||
|
84 | --group-outputs=order: | |
|
85 | The same as 'type', but individual displaypub outputs (e.g. plots) | |
|
86 | will be interleaved, so it will display all of the first plots, | |
|
87 | then all of the second plots, etc. | |
|
88 | ||
|
89 | --group-outputs=engine: | |
|
90 | All of an engine's output is displayed before moving on to the next. | |
|
91 | ||
|
92 | To use this a :class:`DirectView` instance must be created | |
|
93 | and then activated by calling its :meth:`activate` method. | |
|
94 | ||
|
95 | This lets you recall the results of %px computations after | |
|
96 | asynchronous submission (view.block=False). | |
|
97 | ||
|
98 | Then you can do the following:: | |
|
99 | ||
|
100 | In [23]: %px os.getpid() | |
|
101 | Async parallel execution on engine(s): all | |
|
102 | ||
|
103 | In [24]: %result | |
|
104 | [ 8] Out[10]: 60920 | |
|
105 | [ 9] Out[10]: 60921 | |
|
106 | [10] Out[10]: 60922 | |
|
107 | [11] Out[10]: 60923 | |
|
108 | """ | |
|
109 | opts, _ = self.parse_options(line, 'oe', 'group-outputs=') | |
|
110 | ||
|
111 | if 'group-outputs' in opts: | |
|
112 | groupby = opts['group-outputs'] | |
|
113 | elif 'o' in opts: | |
|
114 | groupby = 'order' | |
|
115 | elif 'e' in opts: | |
|
116 | groupby = 'engine' | |
|
117 | else: | |
|
118 | groupby = 'type' | |
|
119 | ||
|
120 | if self.active_view is None: | |
|
121 | raise UsageError(NO_ACTIVE_VIEW) | |
|
122 | ||
|
123 | if self.last_result is None: | |
|
124 | raise UsageError(NO_LAST_RESULT) | |
|
125 | ||
|
126 | self.last_result.get() | |
|
127 | self.last_result.display_outputs(groupby=groupby) | |
|
128 | ||
|
129 | @skip_doctest | |
|
130 | @line_magic | |
|
131 | def px(self, parameter_s=''): | |
|
132 | """Executes the given python command in parallel. | |
|
133 | ||
|
134 | To use this a :class:`DirectView` instance must be created | |
|
135 | and then activated by calling its :meth:`activate` method. | |
|
136 | ||
|
137 | Then you can do the following:: | |
|
138 | ||
|
139 | In [24]: %px a = os.getpid() | |
|
140 | Parallel execution on engine(s): all | |
|
141 | ||
|
142 | In [25]: %px print a | |
|
143 | [stdout:0] 1234 | |
|
144 | [stdout:1] 1235 | |
|
145 | [stdout:2] 1236 | |
|
146 | [stdout:3] 1237 | |
|
147 | """ | |
|
148 | return self.parallel_execute(parameter_s) | |
|
149 | ||
|
150 | def parallel_execute(self, cell, block=None, groupby='type', save_name=None): | |
|
151 | """implementation used by %px and %%parallel""" | |
|
152 | ||
|
153 | if self.active_view is None: | |
|
154 | raise UsageError(NO_ACTIVE_VIEW) | |
|
155 | ||
|
156 | # defaults: | |
|
157 | block = self.active_view.block if block is None else block | |
|
158 | ||
|
159 | base = "Parallel" if block else "Async parallel" | |
|
160 | ||
|
161 | targets = self.active_view.targets | |
|
162 | if isinstance(targets, list) and len(targets) > 10: | |
|
163 | str_targets = str(targets[:4])[:-1] + ', ..., ' + str(targets[-4:])[1:] | |
|
164 | else: | |
|
165 | str_targets = str(targets) | |
|
166 | print base + " execution on engine(s): %s" % str_targets | |
|
167 | ||
|
168 | result = self.active_view.execute(cell, silent=False, block=False) | |
|
169 | self.last_result = result | |
|
170 | ||
|
171 | if save_name: | |
|
172 | self.shell.user_ns[save_name] = result | |
|
173 | ||
|
174 | if block: | |
|
175 | result.get() | |
|
176 | result.display_outputs(groupby) | |
|
177 | else: | |
|
178 | # return AsyncResult only on non-blocking submission | |
|
179 | return result | |
|
180 | ||
|
181 | @skip_doctest | |
|
182 | @cell_magic('px') | |
|
183 | def cell_px(self, line='', cell=None): | |
|
184 | """Executes the given python command in parallel. | |
|
185 | ||
|
186 | Cell magic usage: | |
|
187 | ||
|
188 | %%px [-o] [-e] [--group-options=type|engine|order] [--[no]block] [--out name] | |
|
189 | ||
|
190 | Options: | |
|
191 | ||
|
192 | --out <name>: store the AsyncResult object for this computation | |
|
193 | in the global namespace. | |
|
194 | ||
|
195 | -o: collate outputs in order (same as group-outputs=order) | |
|
196 | ||
|
197 | -e: group outputs by engine (same as group-outputs=engine) | |
|
198 | ||
|
199 | --group-outputs=type [default behavior]: | |
|
200 | each output type (stdout, stderr, displaypub) for all engines | |
|
201 | displayed together. | |
|
202 | ||
|
203 | --group-outputs=order: | |
|
204 | The same as 'type', but individual displaypub outputs (e.g. plots) | |
|
205 | will be interleaved, so it will display all of the first plots, | |
|
206 | then all of the second plots, etc. | |
|
207 | ||
|
208 | --group-outputs=engine: | |
|
209 | All of an engine's output is displayed before moving on to the next. | |
|
210 | ||
|
211 | --[no]block: | |
|
212 | Whether or not to block for the execution to complete | |
|
213 | (and display the results). If unspecified, the active view's | |
|
214 | ||
|
215 | ||
|
216 | To use this a :class:`DirectView` instance must be created | |
|
217 | and then activated by calling its :meth:`activate` method. | |
|
218 | ||
|
219 | Then you can do the following:: | |
|
220 | ||
|
221 | In [24]: %%px --noblock | |
|
222 | ....: a = os.getpid() | |
|
223 | Async parallel execution on engine(s): all | |
|
224 | ||
|
225 | In [25]: %%px | |
|
226 | ....: print a | |
|
227 | [stdout:0] 1234 | |
|
228 | [stdout:1] 1235 | |
|
229 | [stdout:2] 1236 | |
|
230 | [stdout:3] 1237 | |
|
231 | """ | |
|
232 | ||
|
233 | block = None | |
|
234 | groupby = 'type' | |
|
235 | # as a cell magic, we accept args | |
|
236 | opts, _ = self.parse_options(line, 'oe', 'group-outputs=', 'out=', 'block', 'noblock') | |
|
237 | ||
|
238 | if 'group-outputs' in opts: | |
|
239 | groupby = opts['group-outputs'] | |
|
240 | elif 'o' in opts: | |
|
241 | groupby = 'order' | |
|
242 | elif 'e' in opts: | |
|
243 | groupby = 'engine' | |
|
244 | ||
|
245 | if 'block' in opts: | |
|
246 | block = True | |
|
247 | elif 'noblock' in opts: | |
|
248 | block = False | |
|
249 | ||
|
250 | save_name = opts.get('out') | |
|
251 | ||
|
252 | return self.parallel_execute(cell, block=block, groupby=groupby, save_name=save_name) | |
|
253 | ||
|
254 | @skip_doctest | |
|
255 | @line_magic | |
|
256 | def autopx(self, parameter_s=''): | |
|
257 | """Toggles auto parallel mode. | |
|
258 | ||
|
259 | To use this a :class:`DirectView` instance must be created | |
|
260 | and then activated by calling its :meth:`activate` method. Once this | |
|
261 | is called, all commands typed at the command line are send to | |
|
262 | the engines to be executed in parallel. To control which engine | |
|
263 | are used, set the ``targets`` attributed of the multiengine client | |
|
264 | before entering ``%autopx`` mode. | |
|
265 | ||
|
266 | Then you can do the following:: | |
|
267 | ||
|
268 | In [25]: %autopx | |
|
269 | %autopx to enabled | |
|
270 | ||
|
271 | In [26]: a = 10 | |
|
272 | Parallel execution on engine(s): [0,1,2,3] | |
|
273 | In [27]: print a | |
|
274 | Parallel execution on engine(s): [0,1,2,3] | |
|
275 | [stdout:0] 10 | |
|
276 | [stdout:1] 10 | |
|
277 | [stdout:2] 10 | |
|
278 | [stdout:3] 10 | |
|
279 | ||
|
280 | ||
|
281 | In [27]: %autopx | |
|
282 | %autopx disabled | |
|
283 | """ | |
|
284 | if self._autopx: | |
|
285 | self._disable_autopx() | |
|
286 | else: | |
|
287 | self._enable_autopx() | |
|
288 | ||
|
289 | def _enable_autopx(self): | |
|
290 | """Enable %autopx mode by saving the original run_cell and installing | |
|
291 | pxrun_cell. | |
|
292 | """ | |
|
293 | if self.active_view is None: | |
|
294 | raise UsageError(NO_ACTIVE_VIEW) | |
|
295 | ||
|
296 | # override run_cell | |
|
297 | self._original_run_cell = self.shell.run_cell | |
|
298 | self.shell.run_cell = self.pxrun_cell | |
|
299 | ||
|
300 | self._autopx = True | |
|
301 | print "%autopx enabled" | |
|
302 | ||
|
303 | def _disable_autopx(self): | |
|
304 | """Disable %autopx by restoring the original InteractiveShell.run_cell. | |
|
305 | """ | |
|
306 | if self._autopx: | |
|
307 | self.shell.run_cell = self._original_run_cell | |
|
308 | self._autopx = False | |
|
309 | print "%autopx disabled" | |
|
310 | ||
|
311 | def pxrun_cell(self, raw_cell, store_history=False, silent=False): | |
|
312 | """drop-in replacement for InteractiveShell.run_cell. | |
|
313 | ||
|
314 | This executes code remotely, instead of in the local namespace. | |
|
315 | ||
|
316 | See InteractiveShell.run_cell for details. | |
|
317 | """ | |
|
318 | ||
|
319 | if (not raw_cell) or raw_cell.isspace(): | |
|
320 | return | |
|
321 | ||
|
322 | ipself = self.shell | |
|
323 | ||
|
324 | with ipself.builtin_trap: | |
|
325 | cell = ipself.prefilter_manager.prefilter_lines(raw_cell) | |
|
326 | ||
|
327 | # Store raw and processed history | |
|
328 | if store_history: | |
|
329 | ipself.history_manager.store_inputs(ipself.execution_count, | |
|
330 | cell, raw_cell) | |
|
331 | ||
|
332 | # ipself.logger.log(cell, raw_cell) | |
|
333 | ||
|
334 | cell_name = ipself.compile.cache(cell, ipself.execution_count) | |
|
335 | ||
|
336 | try: | |
|
337 | ast.parse(cell, filename=cell_name) | |
|
338 | except (OverflowError, SyntaxError, ValueError, TypeError, | |
|
339 | MemoryError): | |
|
340 | # Case 1 | |
|
341 | ipself.showsyntaxerror() | |
|
342 | ipself.execution_count += 1 | |
|
343 | return None | |
|
344 | except NameError: | |
|
345 | # ignore name errors, because we don't know the remote keys | |
|
346 | pass | |
|
347 | ||
|
348 | if store_history: | |
|
349 | # Write output to the database. Does nothing unless | |
|
350 | # history output logging is enabled. | |
|
351 | ipself.history_manager.store_output(ipself.execution_count) | |
|
352 | # Each cell is a *single* input, regardless of how many lines it has | |
|
353 | ipself.execution_count += 1 | |
|
354 | if re.search(r'get_ipython\(\)\.magic\(u?["\']%?autopx', cell): | |
|
355 | self._disable_autopx() | |
|
356 | return False | |
|
357 | else: | |
|
358 | try: | |
|
359 | result = self.active_view.execute(cell, silent=False, block=False) | |
|
360 | except: | |
|
361 | ipself.showtraceback() | |
|
362 | return True | |
|
363 | else: | |
|
364 | if self.active_view.block: | |
|
365 | try: | |
|
366 | result.get() | |
|
367 | except: | |
|
368 | self.shell.showtraceback() | |
|
369 | return True | |
|
370 | else: | |
|
371 | with ipself.builtin_trap: | |
|
372 | result.display_outputs() | |
|
373 | return False | |
|
374 | ||
|
375 | ||
|
376 | __doc__ = __doc__.format( | |
|
377 | AUTOPX_DOC = ' '*8 + ParallelMagics.autopx.__doc__, | |
|
378 | PX_DOC = ' '*8 + ParallelMagics.px.__doc__, | |
|
379 | RESULT_DOC = ' '*8 + ParallelMagics.result.__doc__ | |
|
380 | ) | |
|
381 | ||
|
382 | _loaded = False | |
|
383 | ||
|
384 | ||
|
385 | def load_ipython_extension(ip): | |
|
386 | """Load the extension in IPython.""" | |
|
387 | global _loaded | |
|
388 | if not _loaded: | |
|
389 | ip.register_magics(ParallelMagics) | |
|
390 | _loaded = True |
@@ -1,390 +1,20 | |||
|
1 | 1 | # encoding: utf-8 |
|
2 | 2 | """ |
|
3 | 3 | ============= |
|
4 | 4 | parallelmagic |
|
5 | 5 | ============= |
|
6 | 6 | |
|
7 | Magic command interface for interactive parallel work. | |
|
8 | ||
|
9 | Usage | |
|
10 | ===== | |
|
11 | ||
|
12 | ``%autopx`` | |
|
13 | ||
|
14 | {AUTOPX_DOC} | |
|
15 | ||
|
16 | ``%px`` | |
|
17 | ||
|
18 | {PX_DOC} | |
|
19 | ||
|
20 | ``%result`` | |
|
21 | ||
|
22 | {RESULT_DOC} | |
|
23 | ||
|
7 | Deprecated, parallel magics are no longer an extension. | |
|
24 | 8 | """ |
|
25 | 9 | |
|
26 | 10 | #----------------------------------------------------------------------------- |
|
27 | 11 | # Copyright (C) 2008 The IPython Development Team |
|
28 | 12 | # |
|
29 | 13 | # Distributed under the terms of the BSD License. The full license is in |
|
30 | 14 | # the file COPYING, distributed as part of this software. |
|
31 | 15 | #----------------------------------------------------------------------------- |
|
32 | 16 | |
|
33 | #----------------------------------------------------------------------------- | |
|
34 | # Imports | |
|
35 | #----------------------------------------------------------------------------- | |
|
36 | ||
|
37 | import ast | |
|
38 | import re | |
|
39 | ||
|
40 | from IPython.core.error import UsageError | |
|
41 | from IPython.core.magic import Magics, magics_class, line_magic, cell_magic | |
|
42 | from IPython.testing.skipdoctest import skip_doctest | |
|
43 | ||
|
44 | #----------------------------------------------------------------------------- | |
|
45 | # Definitions of magic functions for use with IPython | |
|
46 | #----------------------------------------------------------------------------- | |
|
47 | ||
|
48 | ||
|
49 | NO_ACTIVE_VIEW = "Use activate() on a DirectView object to use it with magics." | |
|
50 | NO_LAST_RESULT = "%result recalls last %px result, which has not yet been used." | |
|
51 | ||
|
52 | ||
|
53 | @magics_class | |
|
54 | class ParallelMagics(Magics): | |
|
55 | """A set of magics useful when controlling a parallel IPython cluster. | |
|
56 | """ | |
|
57 | ||
|
58 | # A flag showing if autopx is activated or not | |
|
59 | _autopx = False | |
|
60 | # the current view used by the magics: | |
|
61 | active_view = None | |
|
62 | # last result cache for %result | |
|
63 | last_result = None | |
|
64 | ||
|
65 | @skip_doctest | |
|
66 | @line_magic | |
|
67 | def result(self, line=''): | |
|
68 | """Print the result of the last asynchronous %px command. | |
|
69 | ||
|
70 | Usage: | |
|
71 | ||
|
72 | %result [-o] [-e] [--group-options=type|engine|order] | |
|
73 | ||
|
74 | Options: | |
|
75 | ||
|
76 | -o: collate outputs in order (same as group-outputs=order) | |
|
77 | ||
|
78 | -e: group outputs by engine (same as group-outputs=engine) | |
|
79 | ||
|
80 | --group-outputs=type [default behavior]: | |
|
81 | each output type (stdout, stderr, displaypub) for all engines | |
|
82 | displayed together. | |
|
83 | ||
|
84 | --group-outputs=order: | |
|
85 | The same as 'type', but individual displaypub outputs (e.g. plots) | |
|
86 | will be interleaved, so it will display all of the first plots, | |
|
87 | then all of the second plots, etc. | |
|
88 | ||
|
89 | --group-outputs=engine: | |
|
90 | All of an engine's output is displayed before moving on to the next. | |
|
91 | ||
|
92 | To use this a :class:`DirectView` instance must be created | |
|
93 | and then activated by calling its :meth:`activate` method. | |
|
94 | ||
|
95 | This lets you recall the results of %px computations after | |
|
96 | asynchronous submission (view.block=False). | |
|
97 | ||
|
98 | Then you can do the following:: | |
|
99 | ||
|
100 | In [23]: %px os.getpid() | |
|
101 | Async parallel execution on engine(s): all | |
|
102 | ||
|
103 | In [24]: %result | |
|
104 | [ 8] Out[10]: 60920 | |
|
105 | [ 9] Out[10]: 60921 | |
|
106 | [10] Out[10]: 60922 | |
|
107 | [11] Out[10]: 60923 | |
|
108 | """ | |
|
109 | opts, _ = self.parse_options(line, 'oe', 'group-outputs=') | |
|
110 | ||
|
111 | if 'group-outputs' in opts: | |
|
112 | groupby = opts['group-outputs'] | |
|
113 | elif 'o' in opts: | |
|
114 | groupby = 'order' | |
|
115 | elif 'e' in opts: | |
|
116 | groupby = 'engine' | |
|
117 | else: | |
|
118 | groupby = 'type' | |
|
119 | ||
|
120 | if self.active_view is None: | |
|
121 | raise UsageError(NO_ACTIVE_VIEW) | |
|
122 | ||
|
123 | if self.last_result is None: | |
|
124 | raise UsageError(NO_LAST_RESULT) | |
|
125 | ||
|
126 | self.last_result.get() | |
|
127 | self.last_result.display_outputs(groupby=groupby) | |
|
128 | ||
|
129 | @skip_doctest | |
|
130 | @line_magic | |
|
131 | def px(self, parameter_s=''): | |
|
132 | """Executes the given python command in parallel. | |
|
133 | ||
|
134 | To use this a :class:`DirectView` instance must be created | |
|
135 | and then activated by calling its :meth:`activate` method. | |
|
136 | ||
|
137 | Then you can do the following:: | |
|
138 | ||
|
139 | In [24]: %px a = os.getpid() | |
|
140 | Parallel execution on engine(s): all | |
|
141 | ||
|
142 | In [25]: %px print a | |
|
143 | [stdout:0] 1234 | |
|
144 | [stdout:1] 1235 | |
|
145 | [stdout:2] 1236 | |
|
146 | [stdout:3] 1237 | |
|
147 | """ | |
|
148 | return self.parallel_execute(parameter_s) | |
|
149 | ||
|
150 | def parallel_execute(self, cell, block=None, groupby='type', save_name=None): | |
|
151 | """implementation used by %px and %%parallel""" | |
|
152 | ||
|
153 | if self.active_view is None: | |
|
154 | raise UsageError(NO_ACTIVE_VIEW) | |
|
155 | ||
|
156 | # defaults: | |
|
157 | block = self.active_view.block if block is None else block | |
|
158 | ||
|
159 | base = "Parallel" if block else "Async parallel" | |
|
160 | ||
|
161 | targets = self.active_view.targets | |
|
162 | if isinstance(targets, list) and len(targets) > 10: | |
|
163 | str_targets = str(targets[:4])[:-1] + ', ..., ' + str(targets[-4:])[1:] | |
|
164 | else: | |
|
165 | str_targets = str(targets) | |
|
166 | print base + " execution on engine(s): %s" % str_targets | |
|
167 | ||
|
168 | result = self.active_view.execute(cell, silent=False, block=False) | |
|
169 | self.last_result = result | |
|
170 | ||
|
171 | if save_name: | |
|
172 | self.shell.user_ns[save_name] = result | |
|
173 | ||
|
174 | if block: | |
|
175 | result.get() | |
|
176 | result.display_outputs(groupby) | |
|
177 | else: | |
|
178 | # return AsyncResult only on non-blocking submission | |
|
179 | return result | |
|
180 | ||
|
181 | @skip_doctest | |
|
182 | @cell_magic('px') | |
|
183 | def cell_px(self, line='', cell=None): | |
|
184 | """Executes the given python command in parallel. | |
|
185 | ||
|
186 | Cell magic usage: | |
|
187 | ||
|
188 | %%px [-o] [-e] [--group-options=type|engine|order] [--[no]block] [--out name] | |
|
189 | ||
|
190 | Options: | |
|
191 | ||
|
192 | --out <name>: store the AsyncResult object for this computation | |
|
193 | in the global namespace. | |
|
194 | ||
|
195 | -o: collate outputs in order (same as group-outputs=order) | |
|
196 | ||
|
197 | -e: group outputs by engine (same as group-outputs=engine) | |
|
198 | ||
|
199 | --group-outputs=type [default behavior]: | |
|
200 | each output type (stdout, stderr, displaypub) for all engines | |
|
201 | displayed together. | |
|
202 | ||
|
203 | --group-outputs=order: | |
|
204 | The same as 'type', but individual displaypub outputs (e.g. plots) | |
|
205 | will be interleaved, so it will display all of the first plots, | |
|
206 | then all of the second plots, etc. | |
|
207 | ||
|
208 | --group-outputs=engine: | |
|
209 | All of an engine's output is displayed before moving on to the next. | |
|
210 | ||
|
211 | --[no]block: | |
|
212 | Whether or not to block for the execution to complete | |
|
213 | (and display the results). If unspecified, the active view's | |
|
214 | ||
|
215 | ||
|
216 | To use this a :class:`DirectView` instance must be created | |
|
217 | and then activated by calling its :meth:`activate` method. | |
|
218 | ||
|
219 | Then you can do the following:: | |
|
220 | ||
|
221 | In [24]: %%px --noblock | |
|
222 | ....: a = os.getpid() | |
|
223 | Async parallel execution on engine(s): all | |
|
224 | ||
|
225 | In [25]: %%px | |
|
226 | ....: print a | |
|
227 | [stdout:0] 1234 | |
|
228 | [stdout:1] 1235 | |
|
229 | [stdout:2] 1236 | |
|
230 | [stdout:3] 1237 | |
|
231 | """ | |
|
232 | ||
|
233 | block = None | |
|
234 | groupby = 'type' | |
|
235 | # as a cell magic, we accept args | |
|
236 | opts, _ = self.parse_options(line, 'oe', 'group-outputs=', 'out=', 'block', 'noblock') | |
|
237 | ||
|
238 | if 'group-outputs' in opts: | |
|
239 | groupby = opts['group-outputs'] | |
|
240 | elif 'o' in opts: | |
|
241 | groupby = 'order' | |
|
242 | elif 'e' in opts: | |
|
243 | groupby = 'engine' | |
|
244 | ||
|
245 | if 'block' in opts: | |
|
246 | block = True | |
|
247 | elif 'noblock' in opts: | |
|
248 | block = False | |
|
249 | ||
|
250 | save_name = opts.get('out') | |
|
251 | ||
|
252 | return self.parallel_execute(cell, block=block, groupby=groupby, save_name=save_name) | |
|
253 | ||
|
254 | @skip_doctest | |
|
255 | @line_magic | |
|
256 | def autopx(self, parameter_s=''): | |
|
257 | """Toggles auto parallel mode. | |
|
258 | ||
|
259 | To use this a :class:`DirectView` instance must be created | |
|
260 | and then activated by calling its :meth:`activate` method. Once this | |
|
261 | is called, all commands typed at the command line are send to | |
|
262 | the engines to be executed in parallel. To control which engine | |
|
263 | are used, set the ``targets`` attributed of the multiengine client | |
|
264 | before entering ``%autopx`` mode. | |
|
265 | ||
|
266 | Then you can do the following:: | |
|
267 | ||
|
268 | In [25]: %autopx | |
|
269 | %autopx to enabled | |
|
270 | ||
|
271 | In [26]: a = 10 | |
|
272 | Parallel execution on engine(s): [0,1,2,3] | |
|
273 | In [27]: print a | |
|
274 | Parallel execution on engine(s): [0,1,2,3] | |
|
275 | [stdout:0] 10 | |
|
276 | [stdout:1] 10 | |
|
277 | [stdout:2] 10 | |
|
278 | [stdout:3] 10 | |
|
279 | ||
|
280 | ||
|
281 | In [27]: %autopx | |
|
282 | %autopx disabled | |
|
283 | """ | |
|
284 | if self._autopx: | |
|
285 | self._disable_autopx() | |
|
286 | else: | |
|
287 | self._enable_autopx() | |
|
288 | ||
|
289 | def _enable_autopx(self): | |
|
290 | """Enable %autopx mode by saving the original run_cell and installing | |
|
291 | pxrun_cell. | |
|
292 | """ | |
|
293 | if self.active_view is None: | |
|
294 | raise UsageError(NO_ACTIVE_VIEW) | |
|
295 | ||
|
296 | # override run_cell | |
|
297 | self._original_run_cell = self.shell.run_cell | |
|
298 | self.shell.run_cell = self.pxrun_cell | |
|
299 | ||
|
300 | self._autopx = True | |
|
301 | print "%autopx enabled" | |
|
302 | ||
|
303 | def _disable_autopx(self): | |
|
304 | """Disable %autopx by restoring the original InteractiveShell.run_cell. | |
|
305 | """ | |
|
306 | if self._autopx: | |
|
307 | self.shell.run_cell = self._original_run_cell | |
|
308 | self._autopx = False | |
|
309 | print "%autopx disabled" | |
|
310 | ||
|
311 | def pxrun_cell(self, raw_cell, store_history=False, silent=False): | |
|
312 | """drop-in replacement for InteractiveShell.run_cell. | |
|
313 | ||
|
314 | This executes code remotely, instead of in the local namespace. | |
|
315 | ||
|
316 | See InteractiveShell.run_cell for details. | |
|
317 | """ | |
|
318 | ||
|
319 | if (not raw_cell) or raw_cell.isspace(): | |
|
320 | return | |
|
321 | ||
|
322 | ipself = self.shell | |
|
323 | ||
|
324 | with ipself.builtin_trap: | |
|
325 | cell = ipself.prefilter_manager.prefilter_lines(raw_cell) | |
|
326 | ||
|
327 | # Store raw and processed history | |
|
328 | if store_history: | |
|
329 | ipself.history_manager.store_inputs(ipself.execution_count, | |
|
330 | cell, raw_cell) | |
|
331 | ||
|
332 | # ipself.logger.log(cell, raw_cell) | |
|
333 | ||
|
334 | cell_name = ipself.compile.cache(cell, ipself.execution_count) | |
|
335 | ||
|
336 | try: | |
|
337 | ast.parse(cell, filename=cell_name) | |
|
338 | except (OverflowError, SyntaxError, ValueError, TypeError, | |
|
339 | MemoryError): | |
|
340 | # Case 1 | |
|
341 | ipself.showsyntaxerror() | |
|
342 | ipself.execution_count += 1 | |
|
343 | return None | |
|
344 | except NameError: | |
|
345 | # ignore name errors, because we don't know the remote keys | |
|
346 | pass | |
|
347 | ||
|
348 | if store_history: | |
|
349 | # Write output to the database. Does nothing unless | |
|
350 | # history output logging is enabled. | |
|
351 | ipself.history_manager.store_output(ipself.execution_count) | |
|
352 | # Each cell is a *single* input, regardless of how many lines it has | |
|
353 | ipself.execution_count += 1 | |
|
354 | if re.search(r'get_ipython\(\)\.magic\(u?["\']%?autopx', cell): | |
|
355 | self._disable_autopx() | |
|
356 | return False | |
|
357 | else: | |
|
358 | try: | |
|
359 | result = self.active_view.execute(cell, silent=False, block=False) | |
|
360 | except: | |
|
361 | ipself.showtraceback() | |
|
362 | return True | |
|
363 | else: | |
|
364 | if self.active_view.block: | |
|
365 | try: | |
|
366 | result.get() | |
|
367 | except: | |
|
368 | self.shell.showtraceback() | |
|
369 | return True | |
|
370 | else: | |
|
371 | with ipself.builtin_trap: | |
|
372 | result.display_outputs() | |
|
373 | return False | |
|
374 | ||
|
375 | ||
|
376 | __doc__ = __doc__.format( | |
|
377 | AUTOPX_DOC = ' '*8 + ParallelMagics.autopx.__doc__, | |
|
378 | PX_DOC = ' '*8 + ParallelMagics.px.__doc__, | |
|
379 | RESULT_DOC = ' '*8 + ParallelMagics.result.__doc__ | |
|
380 | ) | |
|
381 | ||
|
382 | _loaded = False | |
|
383 | ||
|
17 | from warnings import warn | |
|
384 | 18 | |
|
385 | 19 | def load_ipython_extension(ip): |
|
386 | """Load the extension in IPython.""" | |
|
387 | global _loaded | |
|
388 | if not _loaded: | |
|
389 | ip.register_magics(ParallelMagics) | |
|
390 | _loaded = True | |
|
20 | warn("Parallel Magics are no longer defined in an extension", DeprecationWarning) |
General Comments 0
You need to be logged in to leave comments.
Login now