test_magics.py
379 lines
| 12.3 KiB
| text/x-python
|
PythonLexer
MinRK
|
r7055 | # -*- coding: utf-8 -*- | ||
"""Test Parallel magics | ||||
Authors: | ||||
* Min RK | ||||
""" | ||||
#------------------------------------------------------------------------------- | ||||
# Copyright (C) 2011 The IPython Development Team | ||||
# | ||||
# Distributed under the terms of the BSD License. The full license is in | ||||
# the file COPYING, distributed as part of this software. | ||||
#------------------------------------------------------------------------------- | ||||
#------------------------------------------------------------------------------- | ||||
# Imports | ||||
#------------------------------------------------------------------------------- | ||||
MinRK
|
r7239 | import re | ||
MinRK
|
r7055 | import sys | ||
import time | ||||
import zmq | ||||
from nose import SkipTest | ||||
from IPython.testing import decorators as dec | ||||
MinRK
|
r7324 | from IPython.utils.io import capture_output | ||
MinRK
|
r7055 | |||
from IPython import parallel as pmod | ||||
from IPython.parallel import error | ||||
from IPython.parallel import AsyncResult | ||||
from IPython.parallel.util import interactive | ||||
from IPython.parallel.tests import add_engines | ||||
MinRK
|
r7324 | from .clienttest import ClusterTestCase, generate_output | ||
MinRK
|
r7055 | |||
def setup(): | ||||
add_engines(3, total=True) | ||||
Thomas Kluyver
|
r12372 | class TestParallelMagics(ClusterTestCase): | ||
MinRK
|
r7055 | |||
def test_px_blocking(self): | ||||
ip = get_ipython() | ||||
v = self.client[-1:] | ||||
v.activate() | ||||
v.block=True | ||||
ip.magic('px a=5') | ||||
Bradley M. Froehle
|
r7874 | self.assertEqual(v['a'], [5]) | ||
MinRK
|
r7055 | ip.magic('px a=10') | ||
Bradley M. Froehle
|
r7874 | self.assertEqual(v['a'], [10]) | ||
MinRK
|
r7055 | # just 'print a' works ~99% of the time, but this ensures that | ||
# the stdout message has arrived when the result is finished: | ||||
with capture_output() as io: | ||||
ip.magic( | ||||
'px import sys,time;print(a);sys.stdout.flush();time.sleep(0.2)' | ||||
) | ||||
MinRK
|
r12289 | self.assertIn('[stdout:', io.stdout) | ||
self.assertNotIn('\n\n', io.stdout) | ||||
assert io.stdout.rstrip().endswith('10') | ||||
MinRK
|
r7055 | self.assertRaisesRemote(ZeroDivisionError, ip.magic, 'px 1/0') | ||
MinRK
|
r7239 | def _check_generated_stderr(self, stderr, n): | ||
expected = [ | ||||
r'\[stderr:\d+\]', | ||||
'^stderr$', | ||||
'^stderr2$', | ||||
] * n | ||||
MinRK
|
r12290 | self.assertNotIn('\n\n', stderr) | ||
MinRK
|
r7239 | lines = stderr.splitlines() | ||
Bradley M. Froehle
|
r7874 | self.assertEqual(len(lines), len(expected), stderr) | ||
MinRK
|
r7239 | for line,expect in zip(lines, expected): | ||
if isinstance(expect, str): | ||||
expect = [expect] | ||||
for ex in expect: | ||||
MinRK
|
r12290 | assert re.search(ex, line) is not None, "Expected %r in %r" % (ex, line) | ||
MinRK
|
r7239 | |||
MinRK
|
r7055 | def test_cellpx_block_args(self): | ||
"""%%px --[no]block flags work""" | ||||
ip = get_ipython() | ||||
v = self.client[-1:] | ||||
v.activate() | ||||
v.block=False | ||||
for block in (True, False): | ||||
v.block = block | ||||
MinRK
|
r7501 | ip.magic("pxconfig --verbose") | ||
MinRK
|
r12289 | with capture_output(display=False) as io: | ||
MinRK
|
r7055 | ip.run_cell_magic("px", "", "1") | ||
if block: | ||||
MinRK
|
r12289 | assert io.stdout.startswith("Parallel"), io.stdout | ||
MinRK
|
r7055 | else: | ||
MinRK
|
r12289 | assert io.stdout.startswith("Async"), io.stdout | ||
MinRK
|
r7055 | |||
MinRK
|
r12289 | with capture_output(display=False) as io: | ||
MinRK
|
r7055 | ip.run_cell_magic("px", "--block", "1") | ||
MinRK
|
r12289 | assert io.stdout.startswith("Parallel"), io.stdout | ||
MinRK
|
r7055 | |||
MinRK
|
r12289 | with capture_output(display=False) as io: | ||
MinRK
|
r7055 | ip.run_cell_magic("px", "--noblock", "1") | ||
MinRK
|
r12289 | assert io.stdout.startswith("Async"), io.stdout | ||
MinRK
|
r7055 | |||
def test_cellpx_groupby_engine(self): | ||||
"""%%px --group-outputs=engine""" | ||||
ip = get_ipython() | ||||
v = self.client[:] | ||||
v.block = True | ||||
v.activate() | ||||
v['generate_output'] = generate_output | ||||
MinRK
|
r12289 | with capture_output(display=False) as io: | ||
MinRK
|
r7055 | ip.run_cell_magic('px', '--group-outputs=engine', 'generate_output()') | ||
MinRK
|
r12289 | self.assertNotIn('\n\n', io.stdout) | ||
MinRK
|
r7501 | lines = io.stdout.splitlines() | ||
MinRK
|
r7055 | expected = [ | ||
MinRK
|
r7239 | r'\[stdout:\d+\]', | ||
'stdout', | ||||
MinRK
|
r7055 | 'stdout2', | ||
MinRK
|
r7239 | r'\[output:\d+\]', | ||
r'IPython\.core\.display\.HTML', | ||||
r'IPython\.core\.display\.Math', | ||||
r'Out\[\d+:\d+\]:.*IPython\.core\.display\.Math', | ||||
MinRK
|
r7055 | ] * len(v) | ||
Bradley M. Froehle
|
r7874 | self.assertEqual(len(lines), len(expected), io.stdout) | ||
MinRK
|
r7055 | for line,expect in zip(lines, expected): | ||
if isinstance(expect, str): | ||||
expect = [expect] | ||||
for ex in expect: | ||||
MinRK
|
r12290 | assert re.search(ex, line) is not None, "Expected %r in %r" % (ex, line) | ||
MinRK
|
r7055 | |||
MinRK
|
r7239 | self._check_generated_stderr(io.stderr, len(v)) | ||
MinRK
|
r7055 | |||
def test_cellpx_groupby_order(self): | ||||
"""%%px --group-outputs=order""" | ||||
ip = get_ipython() | ||||
v = self.client[:] | ||||
v.block = True | ||||
v.activate() | ||||
v['generate_output'] = generate_output | ||||
MinRK
|
r12289 | with capture_output(display=False) as io: | ||
MinRK
|
r7055 | ip.run_cell_magic('px', '--group-outputs=order', 'generate_output()') | ||
MinRK
|
r12289 | self.assertNotIn('\n\n', io.stdout) | ||
MinRK
|
r7501 | lines = io.stdout.splitlines() | ||
MinRK
|
r7055 | expected = [] | ||
expected.extend([ | ||||
MinRK
|
r7239 | r'\[stdout:\d+\]', | ||
'stdout', | ||||
MinRK
|
r7055 | 'stdout2', | ||
] * len(v)) | ||||
expected.extend([ | ||||
MinRK
|
r7239 | r'\[output:\d+\]', | ||
MinRK
|
r7055 | 'IPython.core.display.HTML', | ||
] * len(v)) | ||||
expected.extend([ | ||||
MinRK
|
r7239 | r'\[output:\d+\]', | ||
MinRK
|
r7055 | 'IPython.core.display.Math', | ||
] * len(v)) | ||||
expected.extend([ | ||||
MinRK
|
r7239 | r'Out\[\d+:\d+\]:.*IPython\.core\.display\.Math' | ||
MinRK
|
r7055 | ] * len(v)) | ||
Bradley M. Froehle
|
r7874 | self.assertEqual(len(lines), len(expected), io.stdout) | ||
MinRK
|
r7055 | for line,expect in zip(lines, expected): | ||
if isinstance(expect, str): | ||||
expect = [expect] | ||||
for ex in expect: | ||||
MinRK
|
r12290 | assert re.search(ex, line) is not None, "Expected %r in %r" % (ex, line) | ||
MinRK
|
r7055 | |||
MinRK
|
r7239 | self._check_generated_stderr(io.stderr, len(v)) | ||
MinRK
|
r7055 | |||
MinRK
|
r7239 | def test_cellpx_groupby_type(self): | ||
MinRK
|
r7055 | """%%px --group-outputs=type""" | ||
ip = get_ipython() | ||||
v = self.client[:] | ||||
v.block = True | ||||
v.activate() | ||||
v['generate_output'] = generate_output | ||||
MinRK
|
r12289 | with capture_output(display=False) as io: | ||
MinRK
|
r7055 | ip.run_cell_magic('px', '--group-outputs=type', 'generate_output()') | ||
MinRK
|
r12289 | self.assertNotIn('\n\n', io.stdout) | ||
MinRK
|
r7501 | lines = io.stdout.splitlines() | ||
MinRK
|
r7055 | |||
expected = [] | ||||
expected.extend([ | ||||
MinRK
|
r7239 | r'\[stdout:\d+\]', | ||
'stdout', | ||||
MinRK
|
r7055 | 'stdout2', | ||
] * len(v)) | ||||
expected.extend([ | ||||
MinRK
|
r7239 | r'\[output:\d+\]', | ||
r'IPython\.core\.display\.HTML', | ||||
r'IPython\.core\.display\.Math', | ||||
MinRK
|
r7055 | ] * len(v)) | ||
expected.extend([ | ||||
MinRK
|
r7239 | (r'Out\[\d+:\d+\]', r'IPython\.core\.display\.Math') | ||
MinRK
|
r7055 | ] * len(v)) | ||
Bradley M. Froehle
|
r7874 | self.assertEqual(len(lines), len(expected), io.stdout) | ||
MinRK
|
r7055 | for line,expect in zip(lines, expected): | ||
if isinstance(expect, str): | ||||
expect = [expect] | ||||
for ex in expect: | ||||
MinRK
|
r12290 | assert re.search(ex, line) is not None, "Expected %r in %r" % (ex, line) | ||
MinRK
|
r7055 | |||
MinRK
|
r7239 | self._check_generated_stderr(io.stderr, len(v)) | ||
MinRK
|
r7055 | |||
def test_px_nonblocking(self): | ||||
ip = get_ipython() | ||||
v = self.client[-1:] | ||||
v.activate() | ||||
v.block=False | ||||
ip.magic('px a=5') | ||||
Bradley M. Froehle
|
r7874 | self.assertEqual(v['a'], [5]) | ||
MinRK
|
r7055 | ip.magic('px a=10') | ||
Bradley M. Froehle
|
r7874 | self.assertEqual(v['a'], [10]) | ||
MinRK
|
r7501 | ip.magic('pxconfig --verbose') | ||
MinRK
|
r7055 | with capture_output() as io: | ||
MinRK
|
r7058 | ar = ip.magic('px print (a)') | ||
MinRK
|
r12290 | self.assertIsInstance(ar, AsyncResult) | ||
self.assertIn('Async', io.stdout) | ||||
self.assertNotIn('[stdout:', io.stdout) | ||||
self.assertNotIn('\n\n', io.stdout) | ||||
MinRK
|
r7239 | |||
MinRK
|
r7055 | ar = ip.magic('px 1/0') | ||
self.assertRaisesRemote(ZeroDivisionError, ar.get) | ||||
def test_autopx_blocking(self): | ||||
ip = get_ipython() | ||||
v = self.client[-1] | ||||
v.activate() | ||||
v.block=True | ||||
MinRK
|
r12289 | with capture_output(display=False) as io: | ||
MinRK
|
r7055 | ip.magic('autopx') | ||
ip.run_cell('\n'.join(('a=5','b=12345','c=0'))) | ||||
ip.run_cell('b*=2') | ||||
ip.run_cell('print (b)') | ||||
ip.run_cell('b') | ||||
ip.run_cell("b/c") | ||||
ip.magic('autopx') | ||||
MinRK
|
r7239 | output = io.stdout | ||
MinRK
|
r7055 | |||
MinRK
|
r12290 | assert output.startswith('%autopx enabled'), output | ||
assert output.rstrip().endswith('%autopx disabled'), output | ||||
self.assertIn('ZeroDivisionError', output) | ||||
self.assertIn('\nOut[', output) | ||||
self.assertIn(': 24690', output) | ||||
MinRK
|
r7055 | ar = v.get_result(-1) | ||
Bradley M. Froehle
|
r7874 | self.assertEqual(v['a'], 5) | ||
self.assertEqual(v['b'], 24690) | ||||
MinRK
|
r7055 | self.assertRaisesRemote(ZeroDivisionError, ar.get) | ||
def test_autopx_nonblocking(self): | ||||
ip = get_ipython() | ||||
v = self.client[-1] | ||||
v.activate() | ||||
v.block=False | ||||
with capture_output() as io: | ||||
ip.magic('autopx') | ||||
ip.run_cell('\n'.join(('a=5','b=10','c=0'))) | ||||
ip.run_cell('print (b)') | ||||
ip.run_cell('import time; time.sleep(0.1)') | ||||
ip.run_cell("b/c") | ||||
ip.run_cell('b*=2') | ||||
ip.magic('autopx') | ||||
MinRK
|
r7239 | output = io.stdout.rstrip() | ||
MinRK
|
r7055 | |||
MinRK
|
r12290 | assert output.startswith('%autopx enabled'), output | ||
assert output.endswith('%autopx disabled'), output | ||||
self.assertNotIn('ZeroDivisionError', output) | ||||
MinRK
|
r7055 | ar = v.get_result(-2) | ||
self.assertRaisesRemote(ZeroDivisionError, ar.get) | ||||
# prevent TaskAborted on pulls, due to ZeroDivisionError | ||||
time.sleep(0.5) | ||||
Bradley M. Froehle
|
r7874 | self.assertEqual(v['a'], 5) | ||
MinRK
|
r7055 | # b*=2 will not fire, due to abort | ||
Bradley M. Froehle
|
r7874 | self.assertEqual(v['b'], 10) | ||
MinRK
|
r7055 | |||
def test_result(self): | ||||
ip = get_ipython() | ||||
v = self.client[-1] | ||||
v.activate() | ||||
data = dict(a=111,b=222) | ||||
v.push(data, block=True) | ||||
MinRK
|
r7476 | for name in ('a', 'b'): | ||
ip.magic('px ' + name) | ||||
MinRK
|
r12289 | with capture_output(display=False) as io: | ||
MinRK
|
r7476 | ip.magic('pxresult') | ||
MinRK
|
r12289 | self.assertIn(str(data[name]), io.stdout) | ||
MinRK
|
r7055 | |||
@dec.skipif_not_matplotlib | ||||
def test_px_pylab(self): | ||||
"""%pylab works on engines""" | ||||
ip = get_ipython() | ||||
v = self.client[-1] | ||||
v.block = True | ||||
v.activate() | ||||
with capture_output() as io: | ||||
ip.magic("px %pylab inline") | ||||
MinRK
|
r12289 | self.assertIn("Populating the interactive namespace from numpy and matplotlib", io.stdout) | ||
MinRK
|
r7055 | |||
MinRK
|
r12289 | with capture_output(display=False) as io: | ||
MinRK
|
r7055 | ip.magic("px plot(rand(100))") | ||
MinRK
|
r12289 | self.assertIn('Out[', io.stdout) | ||
self.assertIn('matplotlib.lines', io.stdout) | ||||
MinRK
|
r7476 | |||
def test_pxconfig(self): | ||||
ip = get_ipython() | ||||
rc = self.client | ||||
v = rc.activate(-1, '_tst') | ||||
Bradley M. Froehle
|
r7874 | self.assertEqual(v.targets, rc.ids[-1]) | ||
MinRK
|
r7476 | ip.magic("%pxconfig_tst -t :") | ||
Bradley M. Froehle
|
r7874 | self.assertEqual(v.targets, rc.ids) | ||
MinRK
|
r7485 | ip.magic("%pxconfig_tst -t ::2") | ||
Bradley M. Froehle
|
r7874 | self.assertEqual(v.targets, rc.ids[::2]) | ||
MinRK
|
r7485 | ip.magic("%pxconfig_tst -t 1::2") | ||
Bradley M. Froehle
|
r7874 | self.assertEqual(v.targets, rc.ids[1::2]) | ||
MinRK
|
r7485 | ip.magic("%pxconfig_tst -t 1") | ||
Bradley M. Froehle
|
r7874 | self.assertEqual(v.targets, 1) | ||
MinRK
|
r7476 | ip.magic("%pxconfig_tst --block") | ||
Bradley M. Froehle
|
r7874 | self.assertEqual(v.block, True) | ||
MinRK
|
r7476 | ip.magic("%pxconfig_tst --noblock") | ||
Bradley M. Froehle
|
r7874 | self.assertEqual(v.block, False) | ||
MinRK
|
r7484 | |||
def test_cellpx_targets(self): | ||||
"""%%px --targets doesn't change defaults""" | ||||
ip = get_ipython() | ||||
rc = self.client | ||||
view = rc.activate(rc.ids) | ||||
Bradley M. Froehle
|
r7874 | self.assertEqual(view.targets, rc.ids) | ||
MinRK
|
r7501 | ip.magic('pxconfig --verbose') | ||
MinRK
|
r7484 | for cell in ("pass", "1/0"): | ||
MinRK
|
r12289 | with capture_output(display=False) as io: | ||
MinRK
|
r7484 | try: | ||
ip.run_cell_magic("px", "--targets all", cell) | ||||
except pmod.RemoteError: | ||||
pass | ||||
MinRK
|
r12289 | self.assertIn('engine(s): all', io.stdout) | ||
Bradley M. Froehle
|
r7874 | self.assertEqual(view.targets, rc.ids) | ||
MinRK
|
r7484 | |||
def test_cellpx_block(self): | ||||
"""%%px --block doesn't change default""" | ||||
ip = get_ipython() | ||||
rc = self.client | ||||
view = rc.activate(rc.ids) | ||||
view.block = False | ||||
Bradley M. Froehle
|
r7874 | self.assertEqual(view.targets, rc.ids) | ||
MinRK
|
r7501 | ip.magic('pxconfig --verbose') | ||
MinRK
|
r7484 | for cell in ("pass", "1/0"): | ||
MinRK
|
r12289 | with capture_output(display=False) as io: | ||
MinRK
|
r7484 | try: | ||
ip.run_cell_magic("px", "--block", cell) | ||||
except pmod.RemoteError: | ||||
pass | ||||
MinRK
|
r12289 | self.assertNotIn('Async', io.stdout) | ||
self.assertEqual(view.block, False) | ||||
MinRK
|
r7484 | |||
MinRK
|
r7055 | |||