diff --git a/IPython/parallel/client/client.py b/IPython/parallel/client/client.py index 37f233f..c8e4753 100644 --- a/IPython/parallel/client/client.py +++ b/IPython/parallel/client/client.py @@ -32,6 +32,7 @@ import zmq from IPython.config.configurable import MultipleInstanceError from IPython.core.application import BaseIPythonApplication +from IPython.core.profiledir import ProfileDir, ProfileDirError from IPython.utils.coloransi import TermColors from IPython.utils.jsonutil import rekey @@ -50,7 +51,6 @@ from IPython.parallel import util from IPython.zmq.session import Session, Message from .asyncresult import AsyncResult, AsyncHubResult -from IPython.core.profiledir import ProfileDir, ProfileDirError from .view import DirectView, LoadBalancedView if sys.version_info[0] >= 3: @@ -480,6 +480,18 @@ class Client(HasTraits): self._queue_handlers = {'execute_reply' : self._handle_execute_reply, 'apply_reply' : self._handle_apply_reply} self._connect(sshserver, ssh_kwargs, timeout) + + # last step: setup magics, if we are in IPython: + + try: + ip = get_ipython() + except NameError: + return + else: + if 'px' not in ip.magics_manager.magics: + # in IPython but we are the first Client. + # activate a default view for parallel magics. + self.activate() def __del__(self): """cleanup sockets, but _not_ context.""" @@ -905,6 +917,29 @@ class Client(HasTraits): # always copy: return list(self._ids) + def activate(self, targets='all', suffix=''): + """Create a DirectView and register it with IPython magics + + Defines the magics `%px, %autopx, %pxresult, %%px` + + Parameters + ---------- + + targets: int, list of ints, or 'all' + The engines on which the view's magics will run + suffix: str [default: ''] + The suffix, if any, for the magics. This allows you to have + multiple views associated with parallel magics at the same time. + + e.g. ``rc.activate(targets=0, suffix='0')`` will give you + the magics ``%px0``, ``%pxresult0``, etc. for running magics just + on engine 0. + """ + view = self.direct_view(targets) + view.block = True + view.activate(suffix) + return view + def close(self): if self._closed: return diff --git a/IPython/parallel/client/magics.py b/IPython/parallel/client/magics.py index a6885bb..5c3ee10 100644 --- a/IPython/parallel/client/magics.py +++ b/IPython/parallel/client/magics.py @@ -17,10 +17,14 @@ Usage {PX_DOC} -``%result`` +``%pxresult`` {RESULT_DOC} +``%pxconfig`` + +{CONFIG_DOC} + """ #----------------------------------------------------------------------------- @@ -38,7 +42,8 @@ import ast import re from IPython.core.error import UsageError -from IPython.core.magic import Magics, magics_class, line_magic, cell_magic +from IPython.core.magic import Magics +from IPython.core import magic_arguments from IPython.testing.skipdoctest import skip_doctest #----------------------------------------------------------------------------- @@ -46,95 +51,164 @@ from IPython.testing.skipdoctest import skip_doctest #----------------------------------------------------------------------------- -NO_ACTIVE_VIEW = "Use activate() on a DirectView object to use it with magics." -NO_LAST_RESULT = "%result recalls last %px result, which has not yet been used." +NO_LAST_RESULT = "%pxresult recalls last %px result, which has not yet been used." +def exec_args(f): + """decorator for adding block/targets args for execution + + applied to %pxconfig and %%px + """ + args = [ + magic_arguments.argument('-b', '--block', action="store_const", + const=True, dest='block', + help="use blocking (sync) execution" + ), + magic_arguments.argument('-a', '--noblock', action="store_const", + const=False, dest='block', + help="use non-blocking (async) execution" + ), + magic_arguments.argument('-t', '--targets', type=str, + help="specify the targets on which to execute" + ), + ] + for a in args: + f = a(f) + return f + +def output_args(f): + """decorator for output-formatting args + + applied to %pxresult and %%px + """ + args = [ + magic_arguments.argument('-r', action="store_const", dest='groupby', + const='order', + help="collate outputs in order (same as group-outputs=order)" + ), + magic_arguments.argument('-e', action="store_const", dest='groupby', + const='engine', + help="group outputs by engine (same as group-outputs=engine)" + ), + magic_arguments.argument('--group-outputs', dest='groupby', type=str, + choices=['engine', 'order', 'type'], default='type', + help="""Group the outputs in a particular way. + + Choices are: + + type: group outputs of all engines by type (stdout, stderr, displaypub, etc.). + + engine: display all output for each engine together. + + order: like type, but individual displaypub output from each engine is collated. + For example, if multiple plots are generated by each engine, the first + figure of each engine will be displayed, then the second of each, etc. + """ + ), + magic_arguments.argument('-o', '--out', dest='save_name', type=str, + help="""store the AsyncResult object for this computation + in the global namespace under this name. + """ + ), + ] + for a in args: + f = a(f) + return f -@magics_class class ParallelMagics(Magics): """A set of magics useful when controlling a parallel IPython cluster. """ + # magic-related + magics = None + registered = True + + # suffix for magics + suffix = '' # A flag showing if autopx is activated or not _autopx = False # the current view used by the magics: - active_view = None - # last result cache for %result + view = None + # last result cache for %pxresult last_result = None - @skip_doctest - @line_magic - def result(self, line=''): - """Print the result of the last asynchronous %px command. - - Usage: + def __init__(self, shell, view, suffix=''): + self.view = view + self.suffix = suffix - %result [-o] [-e] [--group-options=type|engine|order] + # register magics + self.magics = dict(cell={},line={}) + line_magics = self.magics['line'] - Options: + px = 'px' + suffix + if not suffix: + # keep %result for legacy compatibility + line_magics['result'] = self.result - -o: collate outputs in order (same as group-outputs=order) + line_magics['pxresult' + suffix] = self.result + line_magics[px] = self.px + line_magics['pxconfig' + suffix] = self.pxconfig + line_magics['auto' + px] = self.autopx - -e: group outputs by engine (same as group-outputs=engine) + self.magics['cell'][px] = self.cell_px - --group-outputs=type [default behavior]: - each output type (stdout, stderr, displaypub) for all engines - displayed together. - - --group-outputs=order: - The same as 'type', but individual displaypub outputs (e.g. plots) - will be interleaved, so it will display all of the first plots, - then all of the second plots, etc. - - --group-outputs=engine: - All of an engine's output is displayed before moving on to the next. - - To use this a :class:`DirectView` instance must be created - and then activated by calling its :meth:`activate` method. + super(ParallelMagics, self).__init__(shell=shell) + + def _eval_target_str(self, ts): + if ':' in ts: + targets = eval("self.view.client.ids[%s]" % ts) + elif 'all' in ts: + targets = 'all' + else: + targets = eval(ts) + return targets + + @magic_arguments.magic_arguments() + @exec_args + def pxconfig(self, line): + """configure default targets/blocking for %px magics""" + args = magic_arguments.parse_argstring(self.pxconfig, line) + if args.targets: + self.view.targets = self._eval_target_str(args.targets) + if args.block is not None: + self.view.block = args.block + + @magic_arguments.magic_arguments() + @output_args + @skip_doctest + def result(self, line=''): + """Print the result of the last asynchronous %px command. This lets you recall the results of %px computations after - asynchronous submission (view.block=False). + asynchronous submission (block=False). - Then you can do the following:: + Examples + -------- + :: In [23]: %px os.getpid() Async parallel execution on engine(s): all - In [24]: %result + In [24]: %pxresult [ 8] Out[10]: 60920 [ 9] Out[10]: 60921 [10] Out[10]: 60922 [11] Out[10]: 60923 """ - opts, _ = self.parse_options(line, 'oe', 'group-outputs=') - - if 'group-outputs' in opts: - groupby = opts['group-outputs'] - elif 'o' in opts: - groupby = 'order' - elif 'e' in opts: - groupby = 'engine' - else: - groupby = 'type' - - if self.active_view is None: - raise UsageError(NO_ACTIVE_VIEW) + args = magic_arguments.parse_argstring(self.result, line) if self.last_result is None: raise UsageError(NO_LAST_RESULT) self.last_result.get() - self.last_result.display_outputs(groupby=groupby) + self.last_result.display_outputs(groupby=args.groupby) @skip_doctest - @line_magic - def px(self, parameter_s=''): + def px(self, line=''): """Executes the given python command in parallel. - To use this a :class:`DirectView` instance must be created - and then activated by calling its :meth:`activate` method. - - Then you can do the following:: + Examples + -------- + :: In [24]: %px a = os.getpid() Parallel execution on engine(s): all @@ -145,27 +219,24 @@ class ParallelMagics(Magics): [stdout:2] 1236 [stdout:3] 1237 """ - return self.parallel_execute(parameter_s) + return self.parallel_execute(line) def parallel_execute(self, cell, block=None, groupby='type', save_name=None): """implementation used by %px and %%parallel""" - if self.active_view is None: - raise UsageError(NO_ACTIVE_VIEW) - # defaults: - block = self.active_view.block if block is None else block + block = self.view.block if block is None else block base = "Parallel" if block else "Async parallel" - targets = self.active_view.targets + targets = self.view.targets if isinstance(targets, list) and len(targets) > 10: str_targets = str(targets[:4])[:-1] + ', ..., ' + str(targets[-4:])[1:] else: str_targets = str(targets) print base + " execution on engine(s): %s" % str_targets - result = self.active_view.execute(cell, silent=False, block=False) + result = self.view.execute(cell, silent=False, block=False) self.last_result = result if save_name: @@ -178,45 +249,16 @@ class ParallelMagics(Magics): # return AsyncResult only on non-blocking submission return result + @magic_arguments.magic_arguments() + @exec_args + @output_args @skip_doctest - @cell_magic('px') def cell_px(self, line='', cell=None): - """Executes the given python command in parallel. - - Cell magic usage: - - %%px [-o] [-e] [--group-options=type|engine|order] [--[no]block] [--out name] - - Options: - - --out : store the AsyncResult object for this computation - in the global namespace. - - -o: collate outputs in order (same as group-outputs=order) - - -e: group outputs by engine (same as group-outputs=engine) - - --group-outputs=type [default behavior]: - each output type (stdout, stderr, displaypub) for all engines - displayed together. - - --group-outputs=order: - The same as 'type', but individual displaypub outputs (e.g. plots) - will be interleaved, so it will display all of the first plots, - then all of the second plots, etc. - - --group-outputs=engine: - All of an engine's output is displayed before moving on to the next. - - --[no]block: - Whether or not to block for the execution to complete - (and display the results). If unspecified, the active view's + """Executes the cell in parallel. - - To use this a :class:`DirectView` instance must be created - and then activated by calling its :meth:`activate` method. - - Then you can do the following:: + Examples + -------- + :: In [24]: %%px --noblock ....: a = os.getpid() @@ -230,38 +272,29 @@ class ParallelMagics(Magics): [stdout:3] 1237 """ - block = None - groupby = 'type' - # as a cell magic, we accept args - opts, _ = self.parse_options(line, 'oe', 'group-outputs=', 'out=', 'block', 'noblock') - - if 'group-outputs' in opts: - groupby = opts['group-outputs'] - elif 'o' in opts: - groupby = 'order' - elif 'e' in opts: - groupby = 'engine' - - if 'block' in opts: - block = True - elif 'noblock' in opts: - block = False - - save_name = opts.get('out') - - return self.parallel_execute(cell, block=block, groupby=groupby, save_name=save_name) - + args = magic_arguments.parse_argstring(self.cell_px, line) + + if args.targets: + save_targets = self.view.targets + self.view.targets = self._eval_target_str(args.targets) + try: + return self.parallel_execute(cell, block=args.block, + groupby=args.groupby, + save_name=args.save_name, + ) + finally: + if args.targets: + self.view.targets = save_targets + @skip_doctest - @line_magic - def autopx(self, parameter_s=''): + def autopx(self, line=''): """Toggles auto parallel mode. - To use this a :class:`DirectView` instance must be created - and then activated by calling its :meth:`activate` method. Once this - is called, all commands typed at the command line are send to - the engines to be executed in parallel. To control which engine - are used, set the ``targets`` attributed of the multiengine client - before entering ``%autopx`` mode. + Once this is called, all commands typed at the command line are send to + the engines to be executed in parallel. To control which engine are + used, the ``targets`` attribute of the view before + entering ``%autopx`` mode. + Then you can do the following:: @@ -290,9 +323,6 @@ class ParallelMagics(Magics): """Enable %autopx mode by saving the original run_cell and installing pxrun_cell. """ - if self.active_view is None: - raise UsageError(NO_ACTIVE_VIEW) - # override run_cell self._original_run_cell = self.shell.run_cell self.shell.run_cell = self.pxrun_cell @@ -356,12 +386,12 @@ class ParallelMagics(Magics): return False else: try: - result = self.active_view.execute(cell, silent=False, block=False) + result = self.view.execute(cell, silent=False, block=False) except: ipself.showtraceback() return True else: - if self.active_view.block: + if self.view.block: try: result.get() except: @@ -376,15 +406,6 @@ class ParallelMagics(Magics): __doc__ = __doc__.format( AUTOPX_DOC = ' '*8 + ParallelMagics.autopx.__doc__, PX_DOC = ' '*8 + ParallelMagics.px.__doc__, - RESULT_DOC = ' '*8 + ParallelMagics.result.__doc__ + RESULT_DOC = ' '*8 + ParallelMagics.result.__doc__, + CONFIG_DOC = ' '*8 + ParallelMagics.pxconfig.__doc__, ) - -_loaded = False - - -def load_ipython_extension(ip): - """Load the extension in IPython.""" - global _loaded - if not _loaded: - ip.register_magics(ParallelMagics) - _loaded = True diff --git a/IPython/parallel/client/view.py b/IPython/parallel/client/view.py index 7707516..c4f3d12 100644 --- a/IPython/parallel/client/view.py +++ b/IPython/parallel/client/view.py @@ -792,33 +792,37 @@ class DirectView(View): return self.client.kill(targets=targets, block=block) #---------------------------------------- - # activate for %px,%autopx magics + # activate for %px, %autopx, etc. magics #---------------------------------------- - def activate(self): - """Make this `View` active for parallel magic commands. - IPython has a magic command syntax to work with `MultiEngineClient` objects. - In a given IPython session there is a single active one. While - there can be many `Views` created and used by the user, - there is only one active one. The active `View` is used whenever - the magic commands %px and %autopx are used. - - The activate() method is called on a given `View` to make it - active. Once this has been done, the magic commands can be used. + def activate(self, suffix=''): + """Activate IPython magics associated with this View + + Defines the magics `%px, %autopx, %pxresult, %%px, %pxconfig` + + Parameters + ---------- + + suffix: str [default: ''] + The suffix, if any, for the magics. This allows you to have + multiple views associated with parallel magics at the same time. + + e.g. ``rc[::2].activate(suffix='_even')`` will give you + the magics ``%px_even``, ``%pxresult_even``, etc. for running magics + on the even engines. """ - + + from IPython.parallel.client.magics import ParallelMagics + try: # This is injected into __builtins__. ip = get_ipython() except NameError: - print "The IPython parallel magics (%result, %px, %autopx) only work within IPython." - else: - pmagic = ip.magics_manager.registry.get('ParallelMagics') - if pmagic is None: - ip.magic('load_ext parallelmagic') - pmagic = ip.magics_manager.registry.get('ParallelMagics') - - pmagic.active_view = self + print "The IPython parallel magics (%px, etc.) only work within IPython." + return + + M = ParallelMagics(ip, self, suffix) + ip.magics_manager.register(M) @skip_doctest diff --git a/IPython/parallel/tests/test_client.py b/IPython/parallel/tests/test_client.py index 263c0a4..1cbc1bb 100644 --- a/IPython/parallel/tests/test_client.py +++ b/IPython/parallel/tests/test_client.py @@ -24,6 +24,7 @@ from tempfile import mktemp import zmq +from IPython import parallel from IPython.parallel.client import client as clientmod from IPython.parallel import error from IPython.parallel import AsyncResult, AsyncHubResult @@ -420,3 +421,16 @@ class TestClient(ClusterTestCase): "Shouldn't be spinning, but got wall_time=%f" % ar.wall_time ) + def test_activate(self): + ip = get_ipython() + magics = ip.magics_manager.magics + self.assertTrue('px' in magics['line']) + self.assertTrue('px' in magics['cell']) + v0 = self.client.activate(-1, '0') + self.assertTrue('px0' in magics['line']) + self.assertTrue('px0' in magics['cell']) + self.assertEquals(v0.targets, self.client.ids[-1]) + v0 = self.client.activate('all', 'all') + self.assertTrue('pxall' in magics['line']) + self.assertTrue('pxall' in magics['cell']) + self.assertEquals(v0.targets, 'all') diff --git a/IPython/parallel/tests/test_magics.py b/IPython/parallel/tests/test_magics.py index 73be3c0..fa4b5b6 100644 --- a/IPython/parallel/tests/test_magics.py +++ b/IPython/parallel/tests/test_magics.py @@ -301,20 +301,13 @@ class TestParallelMagics(ClusterTestCase, ParametricTestCase): data = dict(a=111,b=222) v.push(data, block=True) - ip.magic('px a') - ip.magic('px b') - for idx, name in [ - ('', 'b'), - ('-1', 'b'), - ('2', 'b'), - ('1', 'a'), - ('-2', 'a'), - ]: + for name in ('a', 'b'): + ip.magic('px ' + name) with capture_output() as io: - ip.magic('result ' + idx) + ip.magic('pxresult') output = io.stdout msg = "expected %s output to include %s, but got: %s" % \ - ('%result '+idx, str(data[name]), output) + ('%pxresult', str(data[name]), output) self.assertTrue(str(data[name]) in output, msg) @dec.skipif_not_matplotlib @@ -336,5 +329,17 @@ class TestParallelMagics(ClusterTestCase, ParametricTestCase): self.assertTrue('Out[' in io.stdout, io.stdout) self.assertTrue('matplotlib.lines' in io.stdout, io.stdout) + + def test_pxconfig(self): + ip = get_ipython() + rc = self.client + v = rc.activate(-1, '_tst') + self.assertEquals(v.targets, rc.ids[-1]) + ip.magic("%pxconfig_tst -t :") + self.assertEquals(v.targets, rc.ids) + ip.magic("%pxconfig_tst --block") + self.assertEquals(v.block, True) + ip.magic("%pxconfig_tst --noblock") + self.assertEquals(v.block, False)