Show More
@@ -0,0 +1,340 b'' | |||||
|
1 | # -*- coding: utf-8 -*- | |||
|
2 | """Test Parallel magics | |||
|
3 | ||||
|
4 | Authors: | |||
|
5 | ||||
|
6 | * Min RK | |||
|
7 | """ | |||
|
8 | #------------------------------------------------------------------------------- | |||
|
9 | # Copyright (C) 2011 The IPython Development Team | |||
|
10 | # | |||
|
11 | # Distributed under the terms of the BSD License. The full license is in | |||
|
12 | # the file COPYING, distributed as part of this software. | |||
|
13 | #------------------------------------------------------------------------------- | |||
|
14 | ||||
|
15 | #------------------------------------------------------------------------------- | |||
|
16 | # Imports | |||
|
17 | #------------------------------------------------------------------------------- | |||
|
18 | ||||
|
19 | import sys | |||
|
20 | import time | |||
|
21 | ||||
|
22 | import zmq | |||
|
23 | from nose import SkipTest | |||
|
24 | ||||
|
25 | from IPython.testing import decorators as dec | |||
|
26 | from IPython.testing.ipunittest import ParametricTestCase | |||
|
27 | ||||
|
28 | from IPython import parallel as pmod | |||
|
29 | from IPython.parallel import error | |||
|
30 | from IPython.parallel import AsyncResult | |||
|
31 | from IPython.parallel.util import interactive | |||
|
32 | ||||
|
33 | from IPython.parallel.tests import add_engines | |||
|
34 | ||||
|
35 | from .clienttest import ClusterTestCase, capture_output, generate_output | |||
|
36 | ||||
|
37 | def setup(): | |||
|
38 | add_engines(3, total=True) | |||
|
39 | ||||
|
40 | class TestParallelMagics(ClusterTestCase, ParametricTestCase): | |||
|
41 | ||||
|
42 | def test_px_blocking(self): | |||
|
43 | ip = get_ipython() | |||
|
44 | v = self.client[-1:] | |||
|
45 | v.activate() | |||
|
46 | v.block=True | |||
|
47 | ||||
|
48 | ip.magic('px a=5') | |||
|
49 | self.assertEquals(v['a'], [5]) | |||
|
50 | ip.magic('px a=10') | |||
|
51 | self.assertEquals(v['a'], [10]) | |||
|
52 | # just 'print a' works ~99% of the time, but this ensures that | |||
|
53 | # the stdout message has arrived when the result is finished: | |||
|
54 | with capture_output() as io: | |||
|
55 | ip.magic( | |||
|
56 | 'px import sys,time;print(a);sys.stdout.flush();time.sleep(0.2)' | |||
|
57 | ) | |||
|
58 | out = io.stdout | |||
|
59 | self.assertTrue('[stdout:' in out, out) | |||
|
60 | self.assertTrue(out.rstrip().endswith('10')) | |||
|
61 | self.assertRaisesRemote(ZeroDivisionError, ip.magic, 'px 1/0') | |||
|
62 | ||||
|
63 | def test_cellpx_block_args(self): | |||
|
64 | """%%px --[no]block flags work""" | |||
|
65 | ip = get_ipython() | |||
|
66 | v = self.client[-1:] | |||
|
67 | v.activate() | |||
|
68 | v.block=False | |||
|
69 | ||||
|
70 | for block in (True, False): | |||
|
71 | v.block = block | |||
|
72 | ||||
|
73 | with capture_output() as io: | |||
|
74 | ip.run_cell_magic("px", "", "1") | |||
|
75 | if block: | |||
|
76 | self.assertTrue(io.stdout.startswith("Parallel"), io.stdout) | |||
|
77 | else: | |||
|
78 | self.assertTrue(io.stdout.startswith("Async"), io.stdout) | |||
|
79 | ||||
|
80 | with capture_output() as io: | |||
|
81 | ip.run_cell_magic("px", "--block", "1") | |||
|
82 | self.assertTrue(io.stdout.startswith("Parallel"), io.stdout) | |||
|
83 | ||||
|
84 | with capture_output() as io: | |||
|
85 | ip.run_cell_magic("px", "--noblock", "1") | |||
|
86 | self.assertTrue(io.stdout.startswith("Async"), io.stdout) | |||
|
87 | ||||
|
88 | def test_cellpx_groupby_engine(self): | |||
|
89 | """%%px --group-outputs=engine""" | |||
|
90 | ip = get_ipython() | |||
|
91 | v = self.client[:] | |||
|
92 | v.block = True | |||
|
93 | v.activate() | |||
|
94 | ||||
|
95 | v['generate_output'] = generate_output | |||
|
96 | ||||
|
97 | with capture_output() as io: | |||
|
98 | ip.run_cell_magic('px', '--group-outputs=engine', 'generate_output()') | |||
|
99 | ||||
|
100 | lines = io.stdout.strip().splitlines()[1:] | |||
|
101 | expected = [ | |||
|
102 | ('[stdout:', '] stdout'), | |||
|
103 | 'stdout2', | |||
|
104 | 'IPython.core.display.HTML', | |||
|
105 | 'IPython.core.display.Math', | |||
|
106 | ('] Out[', 'IPython.core.display.Math') | |||
|
107 | ] * len(v) | |||
|
108 | ||||
|
109 | self.assertEquals(len(lines), len(expected), io.stdout) | |||
|
110 | for line,expect in zip(lines, expected): | |||
|
111 | if isinstance(expect, str): | |||
|
112 | expect = [expect] | |||
|
113 | for ex in expect: | |||
|
114 | self.assertTrue(ex in line, "Expected %r in %r" % (ex, line)) | |||
|
115 | ||||
|
116 | expected = [ | |||
|
117 | ('[stderr:', '] stderr'), | |||
|
118 | 'stderr2', | |||
|
119 | ] * len(v) | |||
|
120 | ||||
|
121 | lines = io.stderr.strip().splitlines() | |||
|
122 | self.assertEquals(len(lines), len(expected), io.stderr) | |||
|
123 | for line,expect in zip(lines, expected): | |||
|
124 | if isinstance(expect, str): | |||
|
125 | expect = [expect] | |||
|
126 | for ex in expect: | |||
|
127 | self.assertTrue(ex in line, "Expected %r in %r" % (ex, line)) | |||
|
128 | ||||
|
129 | ||||
|
130 | def test_cellpx_groupby_order(self): | |||
|
131 | """%%px --group-outputs=order""" | |||
|
132 | ip = get_ipython() | |||
|
133 | v = self.client[:] | |||
|
134 | v.block = True | |||
|
135 | v.activate() | |||
|
136 | ||||
|
137 | v['generate_output'] = generate_output | |||
|
138 | ||||
|
139 | with capture_output() as io: | |||
|
140 | ip.run_cell_magic('px', '--group-outputs=order', 'generate_output()') | |||
|
141 | ||||
|
142 | lines = io.stdout.strip().splitlines()[1:] | |||
|
143 | expected = [] | |||
|
144 | expected.extend([ | |||
|
145 | ('[stdout:', '] stdout'), | |||
|
146 | 'stdout2', | |||
|
147 | ] * len(v)) | |||
|
148 | expected.extend([ | |||
|
149 | 'IPython.core.display.HTML', | |||
|
150 | ] * len(v)) | |||
|
151 | expected.extend([ | |||
|
152 | 'IPython.core.display.Math', | |||
|
153 | ] * len(v)) | |||
|
154 | expected.extend([ | |||
|
155 | ('] Out[', 'IPython.core.display.Math') | |||
|
156 | ] * len(v)) | |||
|
157 | ||||
|
158 | self.assertEquals(len(lines), len(expected), io.stdout) | |||
|
159 | for line,expect in zip(lines, expected): | |||
|
160 | if isinstance(expect, str): | |||
|
161 | expect = [expect] | |||
|
162 | for ex in expect: | |||
|
163 | self.assertTrue(ex in line, "Expected %r in %r" % (ex, line)) | |||
|
164 | ||||
|
165 | expected = [ | |||
|
166 | ('[stderr:', '] stderr'), | |||
|
167 | 'stderr2', | |||
|
168 | ] * len(v) | |||
|
169 | ||||
|
170 | lines = io.stderr.strip().splitlines() | |||
|
171 | self.assertEquals(len(lines), len(expected), io.stderr) | |||
|
172 | for line,expect in zip(lines, expected): | |||
|
173 | if isinstance(expect, str): | |||
|
174 | expect = [expect] | |||
|
175 | for ex in expect: | |||
|
176 | self.assertTrue(ex in line, "Expected %r in %r" % (ex, line)) | |||
|
177 | ||||
|
178 | def test_cellpx_groupby_atype(self): | |||
|
179 | """%%px --group-outputs=type""" | |||
|
180 | ip = get_ipython() | |||
|
181 | v = self.client[:] | |||
|
182 | v.block = True | |||
|
183 | v.activate() | |||
|
184 | ||||
|
185 | v['generate_output'] = generate_output | |||
|
186 | ||||
|
187 | with capture_output() as io: | |||
|
188 | ip.run_cell_magic('px', '--group-outputs=type', 'generate_output()') | |||
|
189 | ||||
|
190 | lines = io.stdout.strip().splitlines()[1:] | |||
|
191 | ||||
|
192 | expected = [] | |||
|
193 | expected.extend([ | |||
|
194 | ('[stdout:', '] stdout'), | |||
|
195 | 'stdout2', | |||
|
196 | ] * len(v)) | |||
|
197 | expected.extend([ | |||
|
198 | 'IPython.core.display.HTML', | |||
|
199 | 'IPython.core.display.Math', | |||
|
200 | ] * len(v)) | |||
|
201 | expected.extend([ | |||
|
202 | ('] Out[', 'IPython.core.display.Math') | |||
|
203 | ] * len(v)) | |||
|
204 | ||||
|
205 | self.assertEquals(len(lines), len(expected), io.stdout) | |||
|
206 | for line,expect in zip(lines, expected): | |||
|
207 | if isinstance(expect, str): | |||
|
208 | expect = [expect] | |||
|
209 | for ex in expect: | |||
|
210 | self.assertTrue(ex in line, "Expected %r in %r" % (ex, line)) | |||
|
211 | ||||
|
212 | expected = [ | |||
|
213 | ('[stderr:', '] stderr'), | |||
|
214 | 'stderr2', | |||
|
215 | ] * len(v) | |||
|
216 | ||||
|
217 | lines = io.stderr.strip().splitlines() | |||
|
218 | self.assertEquals(len(lines), len(expected), io.stderr) | |||
|
219 | for line,expect in zip(lines, expected): | |||
|
220 | if isinstance(expect, str): | |||
|
221 | expect = [expect] | |||
|
222 | for ex in expect: | |||
|
223 | self.assertTrue(ex in line, "Expected %r in %r" % (ex, line)) | |||
|
224 | ||||
|
225 | ||||
|
226 | def test_px_nonblocking(self): | |||
|
227 | ip = get_ipython() | |||
|
228 | v = self.client[-1:] | |||
|
229 | v.activate() | |||
|
230 | v.block=False | |||
|
231 | ||||
|
232 | ip.magic('px a=5') | |||
|
233 | self.assertEquals(v['a'], [5]) | |||
|
234 | ip.magic('px a=10') | |||
|
235 | self.assertEquals(v['a'], [10]) | |||
|
236 | with capture_output() as io: | |||
|
237 | ip.magic('px print a') | |||
|
238 | self.assertFalse('[stdout:' in io.stdout) | |||
|
239 | ar = ip.magic('px 1/0') | |||
|
240 | self.assertRaisesRemote(ZeroDivisionError, ar.get) | |||
|
241 | ||||
|
242 | def test_autopx_blocking(self): | |||
|
243 | ip = get_ipython() | |||
|
244 | v = self.client[-1] | |||
|
245 | v.activate() | |||
|
246 | v.block=True | |||
|
247 | ||||
|
248 | with capture_output() as io: | |||
|
249 | ip.magic('autopx') | |||
|
250 | ip.run_cell('\n'.join(('a=5','b=12345','c=0'))) | |||
|
251 | ip.run_cell('b*=2') | |||
|
252 | ip.run_cell('print (b)') | |||
|
253 | ip.run_cell('b') | |||
|
254 | ip.run_cell("b/c") | |||
|
255 | ip.magic('autopx') | |||
|
256 | ||||
|
257 | output = io.stdout.strip() | |||
|
258 | ||||
|
259 | self.assertTrue(output.startswith('%autopx enabled'), output) | |||
|
260 | self.assertTrue(output.endswith('%autopx disabled'), output) | |||
|
261 | self.assertTrue('RemoteError: ZeroDivisionError' in output, output) | |||
|
262 | self.assertTrue('] Out[' in output, output) | |||
|
263 | self.assertTrue(': 24690' in output, output) | |||
|
264 | ar = v.get_result(-1) | |||
|
265 | self.assertEquals(v['a'], 5) | |||
|
266 | self.assertEquals(v['b'], 24690) | |||
|
267 | self.assertRaisesRemote(ZeroDivisionError, ar.get) | |||
|
268 | ||||
|
269 | def test_autopx_nonblocking(self): | |||
|
270 | ip = get_ipython() | |||
|
271 | v = self.client[-1] | |||
|
272 | v.activate() | |||
|
273 | v.block=False | |||
|
274 | ||||
|
275 | with capture_output() as io: | |||
|
276 | ip.magic('autopx') | |||
|
277 | ip.run_cell('\n'.join(('a=5','b=10','c=0'))) | |||
|
278 | ip.run_cell('print (b)') | |||
|
279 | ip.run_cell('import time; time.sleep(0.1)') | |||
|
280 | ip.run_cell("b/c") | |||
|
281 | ip.run_cell('b*=2') | |||
|
282 | ip.magic('autopx') | |||
|
283 | ||||
|
284 | output = io.stdout.strip() | |||
|
285 | ||||
|
286 | self.assertTrue(output.startswith('%autopx enabled')) | |||
|
287 | self.assertTrue(output.endswith('%autopx disabled')) | |||
|
288 | self.assertFalse('ZeroDivisionError' in output) | |||
|
289 | ar = v.get_result(-2) | |||
|
290 | self.assertRaisesRemote(ZeroDivisionError, ar.get) | |||
|
291 | # prevent TaskAborted on pulls, due to ZeroDivisionError | |||
|
292 | time.sleep(0.5) | |||
|
293 | self.assertEquals(v['a'], 5) | |||
|
294 | # b*=2 will not fire, due to abort | |||
|
295 | self.assertEquals(v['b'], 10) | |||
|
296 | ||||
|
297 | def test_result(self): | |||
|
298 | ip = get_ipython() | |||
|
299 | v = self.client[-1] | |||
|
300 | v.activate() | |||
|
301 | data = dict(a=111,b=222) | |||
|
302 | v.push(data, block=True) | |||
|
303 | ||||
|
304 | ip.magic('px a') | |||
|
305 | ip.magic('px b') | |||
|
306 | for idx, name in [ | |||
|
307 | ('', 'b'), | |||
|
308 | ('-1', 'b'), | |||
|
309 | ('2', 'b'), | |||
|
310 | ('1', 'a'), | |||
|
311 | ('-2', 'a'), | |||
|
312 | ]: | |||
|
313 | with capture_output() as io: | |||
|
314 | ip.magic('result ' + idx) | |||
|
315 | output = io.stdout.strip() | |||
|
316 | msg = "expected %s output to include %s, but got: %s" % \ | |||
|
317 | ('%result '+idx, str(data[name]), output) | |||
|
318 | self.assertTrue(str(data[name]) in output, msg) | |||
|
319 | ||||
|
320 | @dec.skipif_not_matplotlib | |||
|
321 | def test_px_pylab(self): | |||
|
322 | """%pylab works on engines""" | |||
|
323 | ip = get_ipython() | |||
|
324 | v = self.client[-1] | |||
|
325 | v.block = True | |||
|
326 | v.activate() | |||
|
327 | ||||
|
328 | with capture_output() as io: | |||
|
329 | ip.magic("px %pylab inline") | |||
|
330 | ||||
|
331 | self.assertTrue("Welcome to pylab" in io.stdout, io.stdout) | |||
|
332 | self.assertTrue("backend_inline" in io.stdout, io.stdout) | |||
|
333 | ||||
|
334 | with capture_output() as io: | |||
|
335 | ip.magic("px plot(rand(100))") | |||
|
336 | ||||
|
337 | self.assertTrue('] Out[' in io.stdout, io.stdout) | |||
|
338 | self.assertTrue('matplotlib.lines' in io.stdout, io.stdout) | |||
|
339 | ||||
|
340 |
@@ -366,133 +366,6 b' class TestView(ClusterTestCase, ParametricTestCase):' | |||||
366 |
|
366 | |||
367 | self.assertEquals(view.apply_sync(findall, '\w+', 'hello world'), 'hello world'.split()) |
|
367 | self.assertEquals(view.apply_sync(findall, '\w+', 'hello world'), 'hello world'.split()) | |
368 |
|
368 | |||
369 | # parallel magic tests |
|
|||
370 |
|
||||
371 | def test_magic_px_blocking(self): |
|
|||
372 | ip = get_ipython() |
|
|||
373 | v = self.client[-1:] |
|
|||
374 | v.activate() |
|
|||
375 | v.block=True |
|
|||
376 |
|
||||
377 | ip.magic('px a=5') |
|
|||
378 | self.assertEquals(v['a'], [5]) |
|
|||
379 | ip.magic('px a=10') |
|
|||
380 | self.assertEquals(v['a'], [10]) |
|
|||
381 | sio = StringIO() |
|
|||
382 | savestdout = sys.stdout |
|
|||
383 | sys.stdout = sio |
|
|||
384 | # just 'print a' worst ~99% of the time, but this ensures that |
|
|||
385 | # the stdout message has arrived when the result is finished: |
|
|||
386 | ip.magic('px import sys,time;print (a); sys.stdout.flush();time.sleep(0.2)') |
|
|||
387 | sys.stdout = savestdout |
|
|||
388 | buf = sio.getvalue() |
|
|||
389 | self.assertTrue('[stdout:' in buf, buf) |
|
|||
390 | self.assertTrue(buf.rstrip().endswith('10')) |
|
|||
391 | self.assertRaisesRemote(ZeroDivisionError, ip.magic, 'px 1/0') |
|
|||
392 |
|
||||
393 | def test_magic_px_nonblocking(self): |
|
|||
394 | ip = get_ipython() |
|
|||
395 | v = self.client[-1:] |
|
|||
396 | v.activate() |
|
|||
397 | v.block=False |
|
|||
398 |
|
||||
399 | ip.magic('px a=5') |
|
|||
400 | self.assertEquals(v['a'], [5]) |
|
|||
401 | ip.magic('px a=10') |
|
|||
402 | self.assertEquals(v['a'], [10]) |
|
|||
403 | sio = StringIO() |
|
|||
404 | savestdout = sys.stdout |
|
|||
405 | sys.stdout = sio |
|
|||
406 | ip.magic('px print a') |
|
|||
407 | sys.stdout = savestdout |
|
|||
408 | buf = sio.getvalue() |
|
|||
409 | self.assertFalse('[stdout:' in buf) |
|
|||
410 | ip.magic('px 1/0') |
|
|||
411 | ar = v.get_result(-1) |
|
|||
412 | self.assertRaisesRemote(ZeroDivisionError, ar.get) |
|
|||
413 |
|
||||
414 | def test_magic_autopx_blocking(self): |
|
|||
415 | ip = get_ipython() |
|
|||
416 | v = self.client[-1] |
|
|||
417 | v.activate() |
|
|||
418 | v.block=True |
|
|||
419 |
|
||||
420 | sio = StringIO() |
|
|||
421 | savestdout = sys.stdout |
|
|||
422 | sys.stdout = sio |
|
|||
423 | ip.magic('autopx') |
|
|||
424 | ip.run_cell('\n'.join(('a=5','b=10','c=0'))) |
|
|||
425 | ip.run_cell('b*=2') |
|
|||
426 | ip.run_cell('print (b)') |
|
|||
427 | ip.run_cell("b/c") |
|
|||
428 | ip.magic('autopx') |
|
|||
429 | sys.stdout = savestdout |
|
|||
430 | output = sio.getvalue().strip() |
|
|||
431 | self.assertTrue(output.startswith('%autopx enabled')) |
|
|||
432 | self.assertTrue(output.endswith('%autopx disabled')) |
|
|||
433 | self.assertTrue('RemoteError: ZeroDivisionError' in output) |
|
|||
434 | ar = v.get_result(-1) |
|
|||
435 | self.assertEquals(v['a'], 5) |
|
|||
436 | self.assertEquals(v['b'], 20) |
|
|||
437 | self.assertRaisesRemote(ZeroDivisionError, ar.get) |
|
|||
438 |
|
||||
439 | def test_magic_autopx_nonblocking(self): |
|
|||
440 | ip = get_ipython() |
|
|||
441 | v = self.client[-1] |
|
|||
442 | v.activate() |
|
|||
443 | v.block=False |
|
|||
444 |
|
||||
445 | sio = StringIO() |
|
|||
446 | savestdout = sys.stdout |
|
|||
447 | sys.stdout = sio |
|
|||
448 | ip.magic('autopx') |
|
|||
449 | ip.run_cell('\n'.join(('a=5','b=10','c=0'))) |
|
|||
450 | ip.run_cell('print (b)') |
|
|||
451 | ip.run_cell('import time; time.sleep(0.1)') |
|
|||
452 | ip.run_cell("b/c") |
|
|||
453 | ip.run_cell('b*=2') |
|
|||
454 | ip.magic('autopx') |
|
|||
455 | sys.stdout = savestdout |
|
|||
456 | output = sio.getvalue().strip() |
|
|||
457 | self.assertTrue(output.startswith('%autopx enabled')) |
|
|||
458 | self.assertTrue(output.endswith('%autopx disabled')) |
|
|||
459 | self.assertFalse('ZeroDivisionError' in output) |
|
|||
460 | ar = v.get_result(-2) |
|
|||
461 | self.assertRaisesRemote(ZeroDivisionError, ar.get) |
|
|||
462 | # prevent TaskAborted on pulls, due to ZeroDivisionError |
|
|||
463 | time.sleep(0.5) |
|
|||
464 | self.assertEquals(v['a'], 5) |
|
|||
465 | # b*=2 will not fire, due to abort |
|
|||
466 | self.assertEquals(v['b'], 10) |
|
|||
467 |
|
||||
468 | def test_magic_result(self): |
|
|||
469 | ip = get_ipython() |
|
|||
470 | v = self.client[-1] |
|
|||
471 | v.activate() |
|
|||
472 | data = dict(a=111,b=222) |
|
|||
473 | v.push(data, block=True) |
|
|||
474 |
|
||||
475 | ip.magic('px a') |
|
|||
476 | ip.magic('px b') |
|
|||
477 | for idx, name in [ |
|
|||
478 | ('', 'b'), |
|
|||
479 | ('-1', 'b'), |
|
|||
480 | ('2', 'b'), |
|
|||
481 | ('1', 'a'), |
|
|||
482 | ('-2', 'a'), |
|
|||
483 | ]: |
|
|||
484 | sio = StringIO() |
|
|||
485 | savestdout = sys.stdout |
|
|||
486 | sys.stdout = sio |
|
|||
487 | ip.magic('result ' + idx) |
|
|||
488 | sys.stdout.flush() |
|
|||
489 | sys.stdout = savestdout |
|
|||
490 | output = sio.getvalue().strip() |
|
|||
491 | msg = "expected %s output to include %s, but got: %s" % \ |
|
|||
492 | ('%result '+idx, str(data[name]), output) |
|
|||
493 | self.assertTrue(str(data[name]) in output, msg) |
|
|||
494 |
|
||||
495 |
|
||||
496 | def test_unicode_execute(self): |
|
369 | def test_unicode_execute(self): | |
497 | """test executing unicode strings""" |
|
370 | """test executing unicode strings""" | |
498 | v = self.client[-1] |
|
371 | v = self.client[-1] |
General Comments 0
You need to be logged in to leave comments.
Login now