diff --git a/IPython/config/profile/cluster/ipython_config.py b/IPython/config/profile/cluster/ipython_config.py index e28144e..07df14f 100644 --- a/IPython/config/profile/cluster/ipython_config.py +++ b/IPython/config/profile/cluster/ipython_config.py @@ -17,9 +17,3 @@ if hasattr(app, 'exec_lines'): else: app.exec_lines = [lines] -# Load the parallelmagic extension to enable %result, %px, %autopx magics. -if hasattr(app, 'extensions'): - app.extensions.append('parallelmagic') -else: - app.extensions = ['parallelmagic'] - diff --git a/IPython/core/interactiveshell.py b/IPython/core/interactiveshell.py index 28d1e04..93d7d32 100644 --- a/IPython/core/interactiveshell.py +++ b/IPython/core/interactiveshell.py @@ -1697,6 +1697,11 @@ class InteractiveShell(SingletonConfigurable): care of calling it if needed, so unless you are explicitly catching a SyntaxError exception, don't try to analyze the stack manually and simply call this method.""" + try: + from IPython.parallel.error import RemoteError + except ImportError: + class RemoteError(Exception): pass + try: try: @@ -1711,6 +1716,10 @@ class InteractiveShell(SingletonConfigurable): self.showsyntaxerror(filename) elif etype is UsageError: self.write_err("UsageError: %s" % value) + elif issubclass(etype, RemoteError): + # IPython.parallel remote exceptions. + # Draw the remote traceback, not the local one. + self._showtraceback(etype, value, value.render_traceback()) else: if exception_only: stb = ['An exception has occurred, use %tb to see ' diff --git a/IPython/extensions/parallelmagic.py b/IPython/extensions/parallelmagic.py index 80c30e2..da29a4c 100644 --- a/IPython/extensions/parallelmagic.py +++ b/IPython/extensions/parallelmagic.py @@ -4,23 +4,7 @@ parallelmagic ============= -Magic command interface for interactive parallel work. - -Usage -===== - -``%autopx`` - -{AUTOPX_DOC} - -``%px`` - -{PX_DOC} - -``%result`` - -{RESULT_DOC} - +Deprecated, parallel magics are no longer an extension. """ #----------------------------------------------------------------------------- @@ -30,314 +14,7 @@ Usage # the file COPYING, distributed as part of this software. #----------------------------------------------------------------------------- -#----------------------------------------------------------------------------- -# Imports -#----------------------------------------------------------------------------- - -import ast -import re - -from IPython.core.error import UsageError -from IPython.core.magic import Magics, magics_class, line_magic, cell_magic -from IPython.testing.skipdoctest import skip_doctest - -#----------------------------------------------------------------------------- -# Definitions of magic functions for use with IPython -#----------------------------------------------------------------------------- - - -NO_ACTIVE_VIEW = "Use activate() on a DirectView object to use it with magics." - - -@magics_class -class ParallelMagics(Magics): - """A set of magics useful when controlling a parallel IPython cluster. - """ - - # A flag showing if autopx is activated or not - _autopx = False - # the current view used by the magics: - active_view = None - - @skip_doctest - @line_magic - def result(self, parameter_s=''): - """Print the result of command i on all engines. - - To use this a :class:`DirectView` instance must be created - and then activated by calling its :meth:`activate` method. - - This lets you recall the results of %px computations after - asynchronous submission (view.block=False). - - Then you can do the following:: - - In [23]: %px os.getpid() - Async parallel execution on engine(s): all - - In [24]: %result - [ 8] Out[10]: 60920 - [ 9] Out[10]: 60921 - [10] Out[10]: 60922 - [11] Out[10]: 60923 - """ - - if self.active_view is None: - raise UsageError(NO_ACTIVE_VIEW) - - stride = len(self.active_view) - try: - index = int(parameter_s) - except: - index = -1 - msg_ids = self.active_view.history[stride * index:(stride * (index + 1)) or None] - - result = self.active_view.get_result(msg_ids) - - result.get() - result.display_outputs() - - @skip_doctest - @line_magic - def px(self, parameter_s=''): - """Executes the given python command in parallel. - - To use this a :class:`DirectView` instance must be created - and then activated by calling its :meth:`activate` method. - - Then you can do the following:: - - In [24]: %px a = os.getpid() - Parallel execution on engine(s): all - - In [25]: %px print a - [stdout:0] 1234 - [stdout:1] 1235 - [stdout:2] 1236 - [stdout:3] 1237 - """ - return self.parallel_execute(parameter_s) - - def parallel_execute(self, cell, block=None, groupby='type'): - """implementation used by %px and %%parallel""" - - if self.active_view is None: - raise UsageError(NO_ACTIVE_VIEW) - - # defaults: - block = self.active_view.block if block is None else block - - base = "Parallel" if block else "Async parallel" - print base + " execution on engine(s): %s" % self.active_view.targets - - result = self.active_view.execute(cell, silent=False, block=False) - if block: - result.get() - result.display_outputs(groupby) - else: - # return AsyncResult only on non-blocking submission - return result - - @skip_doctest - @cell_magic('px') - def cell_px(self, line='', cell=None): - """Executes the given python command in parallel. - - Cell magic usage: - - %%px [-o] [-e] [--group-options=type|engine|order] [--[no]block] - - Options (%%px cell magic only): - - -o: collate outputs in oder (same as group-outputs=order) - - -e: group outputs by engine (same as group-outputs=engine) - - --group-outputs=type [default behavior]: - each output type (stdout, stderr, displaypub) for all engines - displayed together. - - --group-outputs=order: - The same as 'type', but individual displaypub outputs (e.g. plots) - will be interleaved, so it will display all of the first plots, - then all of the second plots, etc. - - --group-outputs=engine: - All of an engine's output is displayed before moving on to the next. - - --[no]block: - Whether or not to block for the execution to complete - (and display the results). If unspecified, the active view's - - - To use this a :class:`DirectView` instance must be created - and then activated by calling its :meth:`activate` method. - - Then you can do the following:: - - In [24]: %%parallel --noblock a = os.getpid() - Async parallel execution on engine(s): all - - In [25]: %px print a - [stdout:0] 1234 - [stdout:1] 1235 - [stdout:2] 1236 - [stdout:3] 1237 - """ - - block = None - groupby = 'type' - # as a cell magic, we accept args - opts, _ = self.parse_options(line, 'oe', 'group-outputs=', 'block', 'noblock') - - if 'group-outputs' in opts: - groupby = opts['group-outputs'] - elif 'o' in opts: - groupby = 'order' - elif 'e' in opts: - groupby = 'engine' - - if 'block' in opts: - block = True - elif 'noblock' in opts: - block = False - - return self.parallel_execute(cell, block=block, groupby=groupby) - - @skip_doctest - @line_magic - def autopx(self, parameter_s=''): - """Toggles auto parallel mode. - - To use this a :class:`DirectView` instance must be created - and then activated by calling its :meth:`activate` method. Once this - is called, all commands typed at the command line are send to - the engines to be executed in parallel. To control which engine - are used, set the ``targets`` attributed of the multiengine client - before entering ``%autopx`` mode. - - Then you can do the following:: - - In [25]: %autopx - %autopx to enabled - - In [26]: a = 10 - Parallel execution on engine(s): [0,1,2,3] - In [27]: print a - Parallel execution on engine(s): [0,1,2,3] - [stdout:0] 10 - [stdout:1] 10 - [stdout:2] 10 - [stdout:3] 10 - - - In [27]: %autopx - %autopx disabled - """ - if self._autopx: - self._disable_autopx() - else: - self._enable_autopx() - - def _enable_autopx(self): - """Enable %autopx mode by saving the original run_cell and installing - pxrun_cell. - """ - if self.active_view is None: - raise UsageError(NO_ACTIVE_VIEW) - - # override run_cell - self._original_run_cell = self.shell.run_cell - self.shell.run_cell = self.pxrun_cell - - self._autopx = True - print "%autopx enabled" - - def _disable_autopx(self): - """Disable %autopx by restoring the original InteractiveShell.run_cell. - """ - if self._autopx: - self.shell.run_cell = self._original_run_cell - self._autopx = False - print "%autopx disabled" - - def pxrun_cell(self, raw_cell, store_history=False, silent=False): - """drop-in replacement for InteractiveShell.run_cell. - - This executes code remotely, instead of in the local namespace. - - See InteractiveShell.run_cell for details. - """ - - if (not raw_cell) or raw_cell.isspace(): - return - - ipself = self.shell - - with ipself.builtin_trap: - cell = ipself.prefilter_manager.prefilter_lines(raw_cell) - - # Store raw and processed history - if store_history: - ipself.history_manager.store_inputs(ipself.execution_count, - cell, raw_cell) - - # ipself.logger.log(cell, raw_cell) - - cell_name = ipself.compile.cache(cell, ipself.execution_count) - - try: - ast.parse(cell, filename=cell_name) - except (OverflowError, SyntaxError, ValueError, TypeError, - MemoryError): - # Case 1 - ipself.showsyntaxerror() - ipself.execution_count += 1 - return None - except NameError: - # ignore name errors, because we don't know the remote keys - pass - - if store_history: - # Write output to the database. Does nothing unless - # history output logging is enabled. - ipself.history_manager.store_output(ipself.execution_count) - # Each cell is a *single* input, regardless of how many lines it has - ipself.execution_count += 1 - if re.search(r'get_ipython\(\)\.magic\(u?["\']%?autopx', cell): - self._disable_autopx() - return False - else: - try: - result = self.active_view.execute(cell, silent=False, block=False) - except: - ipself.showtraceback() - return True - else: - if self.active_view.block: - try: - result.get() - except: - self.shell.showtraceback() - return True - else: - with ipself.builtin_trap: - result.display_outputs() - return False - - -__doc__ = __doc__.format( - AUTOPX_DOC = ' '*8 + ParallelMagics.autopx.__doc__, - PX_DOC = ' '*8 + ParallelMagics.px.__doc__, - RESULT_DOC = ' '*8 + ParallelMagics.result.__doc__ -) - -_loaded = False - +from warnings import warn def load_ipython_extension(ip): - """Load the extension in IPython.""" - global _loaded - if not _loaded: - ip.register_magics(ParallelMagics) - _loaded = True + warn("Parallel Magics are no longer defined in an extension", DeprecationWarning) diff --git a/IPython/parallel/__init__.py b/IPython/parallel/__init__.py index 22d3b69..511e10d 100644 --- a/IPython/parallel/__init__.py +++ b/IPython/parallel/__init__.py @@ -36,8 +36,9 @@ from .client.asyncresult import * from .client.client import Client from .client.remotefunction import * from .client.view import * -from .util import interactive from .controller.dependency import * +from .error import * +from .util import interactive #----------------------------------------------------------------------------- # Functions diff --git a/IPython/parallel/client/asyncresult.py b/IPython/parallel/client/asyncresult.py index 0c4937d..7b3e352 100644 --- a/IPython/parallel/client/asyncresult.py +++ b/IPython/parallel/client/asyncresult.py @@ -158,6 +158,8 @@ class AsyncResult(object): self._success = True finally: self._metadata = map(self._client.metadata.get, self.msg_ids) + self._wait_for_outputs(10) + def successful(self): @@ -424,6 +426,19 @@ class AsyncResult(object): if self.pyout is not None: display(self.get()) + def _wait_for_outputs(self, timeout=-1): + """wait for the 'status=idle' message that indicates we have all outputs + """ + if not self._success: + # don't wait on errors + return + tic = time.time() + while not all(md['outputs_ready'] for md in self._metadata): + time.sleep(0.01) + self._client._flush_iopub(self._client._iopub_socket) + if timeout >= 0 and time.time() > tic + timeout: + break + @check_ready def display_outputs(self, groupby="type"): """republish the outputs of the computation @@ -453,8 +468,6 @@ class AsyncResult(object): several plots, and you would like to see all of the first plots together, then all of the second plots, and so on. """ - # flush iopub, just in case - self._client._flush_iopub(self._client._iopub_socket) if self._single_result: self._display_single_result() return @@ -619,7 +632,6 @@ class AsyncMapResult(AsyncResult): yield r - class AsyncHubResult(AsyncResult): """Class to wrap pending results that must be requested from the Hub. @@ -627,6 +639,10 @@ class AsyncHubResult(AsyncResult): so use `AsyncHubResult.wait()` sparingly. """ + def _wait_for_outputs(self, timeout=None): + """no-op, because HubResults are never incomplete""" + return + def wait(self, timeout=-1): """wait for result to complete.""" start = time.time() diff --git a/IPython/parallel/client/client.py b/IPython/parallel/client/client.py index 37f233f..16313f9 100644 --- a/IPython/parallel/client/client.py +++ b/IPython/parallel/client/client.py @@ -32,6 +32,7 @@ import zmq from IPython.config.configurable import MultipleInstanceError from IPython.core.application import BaseIPythonApplication +from IPython.core.profiledir import ProfileDir, ProfileDirError from IPython.utils.coloransi import TermColors from IPython.utils.jsonutil import rekey @@ -50,7 +51,6 @@ from IPython.parallel import util from IPython.zmq.session import Session, Message from .asyncresult import AsyncResult, AsyncHubResult -from IPython.core.profiledir import ProfileDir, ProfileDirError from .view import DirectView, LoadBalancedView if sys.version_info[0] >= 3: @@ -184,6 +184,7 @@ class Metadata(dict): 'stdout' : '', 'stderr' : '', 'outputs' : [], + 'outputs_ready' : False, } self.update(md) self.update(dict(*args, **kwargs)) @@ -480,6 +481,18 @@ class Client(HasTraits): self._queue_handlers = {'execute_reply' : self._handle_execute_reply, 'apply_reply' : self._handle_apply_reply} self._connect(sshserver, ssh_kwargs, timeout) + + # last step: setup magics, if we are in IPython: + + try: + ip = get_ipython() + except NameError: + return + else: + if 'px' not in ip.magics_manager.magics: + # in IPython but we are the first Client. + # activate a default view for parallel magics. + self.activate() def __del__(self): """cleanup sockets, but _not_ context.""" @@ -868,6 +881,10 @@ class Client(HasTraits): md['outputs'].append(content) elif msg_type == 'pyout': md['pyout'] = content + elif msg_type == 'status': + # idle message comes after all outputs + if content['execution_state'] == 'idle': + md['outputs_ready'] = True else: # unhandled msg_type (status, etc.) pass @@ -905,6 +922,29 @@ class Client(HasTraits): # always copy: return list(self._ids) + def activate(self, targets='all', suffix=''): + """Create a DirectView and register it with IPython magics + + Defines the magics `%px, %autopx, %pxresult, %%px` + + Parameters + ---------- + + targets: int, list of ints, or 'all' + The engines on which the view's magics will run + suffix: str [default: ''] + The suffix, if any, for the magics. This allows you to have + multiple views associated with parallel magics at the same time. + + e.g. ``rc.activate(targets=0, suffix='0')`` will give you + the magics ``%px0``, ``%pxresult0``, etc. for running magics just + on engine 0. + """ + view = self.direct_view(targets) + view.block = True + view.activate(suffix) + return view + def close(self): if self._closed: return @@ -1099,8 +1139,26 @@ class Client(HasTraits): raise error @spin_first - def shutdown(self, targets=None, restart=False, hub=False, block=None): - """Terminates one or more engine processes, optionally including the hub.""" + def shutdown(self, targets='all', restart=False, hub=False, block=None): + """Terminates one or more engine processes, optionally including the hub. + + Parameters + ---------- + + targets: list of ints or 'all' [default: all] + Which engines to shutdown. + hub: bool [default: False] + Whether to include the Hub. hub=True implies targets='all'. + block: bool [default: self.block] + Whether to wait for clean shutdown replies or not. + restart: bool [default: False] + NOT IMPLEMENTED + whether to restart engines after shutting them down. + """ + + if restart: + raise NotImplementedError("Engine restart is not yet implemented") + block = self.block if block is None else block if hub: targets = 'all' diff --git a/IPython/parallel/client/magics.py b/IPython/parallel/client/magics.py new file mode 100644 index 0000000..9e7e22c --- /dev/null +++ b/IPython/parallel/client/magics.py @@ -0,0 +1,424 @@ +# encoding: utf-8 +""" +============= +parallelmagic +============= + +Magic command interface for interactive parallel work. + +Usage +===== + +``%autopx`` + +{AUTOPX_DOC} + +``%px`` + +{PX_DOC} + +``%pxresult`` + +{RESULT_DOC} + +``%pxconfig`` + +{CONFIG_DOC} + +""" + +#----------------------------------------------------------------------------- +# Copyright (C) 2008 The IPython Development Team +# +# Distributed under the terms of the BSD License. The full license is in +# the file COPYING, distributed as part of this software. +#----------------------------------------------------------------------------- + +#----------------------------------------------------------------------------- +# Imports +#----------------------------------------------------------------------------- + +import ast +import re + +from IPython.core.error import UsageError +from IPython.core.magic import Magics +from IPython.core import magic_arguments +from IPython.testing.skipdoctest import skip_doctest + +#----------------------------------------------------------------------------- +# Definitions of magic functions for use with IPython +#----------------------------------------------------------------------------- + + +NO_LAST_RESULT = "%pxresult recalls last %px result, which has not yet been used." + +def exec_args(f): + """decorator for adding block/targets args for execution + + applied to %pxconfig and %%px + """ + args = [ + magic_arguments.argument('-b', '--block', action="store_const", + const=True, dest='block', + help="use blocking (sync) execution", + ), + magic_arguments.argument('-a', '--noblock', action="store_const", + const=False, dest='block', + help="use non-blocking (async) execution", + ), + magic_arguments.argument('-t', '--targets', type=str, + help="specify the targets on which to execute", + ), + magic_arguments.argument('--verbose', action="store_const", + const=True, dest="set_verbose", + help="print a message at each execution", + ), + magic_arguments.argument('--no-verbose', action="store_const", + const=False, dest="set_verbose", + help="don't print any messages", + ), + ] + for a in args: + f = a(f) + return f + +def output_args(f): + """decorator for output-formatting args + + applied to %pxresult and %%px + """ + args = [ + magic_arguments.argument('-r', action="store_const", dest='groupby', + const='order', + help="collate outputs in order (same as group-outputs=order)" + ), + magic_arguments.argument('-e', action="store_const", dest='groupby', + const='engine', + help="group outputs by engine (same as group-outputs=engine)" + ), + magic_arguments.argument('--group-outputs', dest='groupby', type=str, + choices=['engine', 'order', 'type'], default='type', + help="""Group the outputs in a particular way. + + Choices are: + + type: group outputs of all engines by type (stdout, stderr, displaypub, etc.). + + engine: display all output for each engine together. + + order: like type, but individual displaypub output from each engine is collated. + For example, if multiple plots are generated by each engine, the first + figure of each engine will be displayed, then the second of each, etc. + """ + ), + magic_arguments.argument('-o', '--out', dest='save_name', type=str, + help="""store the AsyncResult object for this computation + in the global namespace under this name. + """ + ), + ] + for a in args: + f = a(f) + return f + +class ParallelMagics(Magics): + """A set of magics useful when controlling a parallel IPython cluster. + """ + + # magic-related + magics = None + registered = True + + # suffix for magics + suffix = '' + # A flag showing if autopx is activated or not + _autopx = False + # the current view used by the magics: + view = None + # last result cache for %pxresult + last_result = None + # verbose flag + verbose = False + + def __init__(self, shell, view, suffix=''): + self.view = view + self.suffix = suffix + + # register magics + self.magics = dict(cell={},line={}) + line_magics = self.magics['line'] + + px = 'px' + suffix + if not suffix: + # keep %result for legacy compatibility + line_magics['result'] = self.result + + line_magics['pxresult' + suffix] = self.result + line_magics[px] = self.px + line_magics['pxconfig' + suffix] = self.pxconfig + line_magics['auto' + px] = self.autopx + + self.magics['cell'][px] = self.cell_px + + super(ParallelMagics, self).__init__(shell=shell) + + def _eval_target_str(self, ts): + if ':' in ts: + targets = eval("self.view.client.ids[%s]" % ts) + elif 'all' in ts: + targets = 'all' + else: + targets = eval(ts) + return targets + + @magic_arguments.magic_arguments() + @exec_args + def pxconfig(self, line): + """configure default targets/blocking for %px magics""" + args = magic_arguments.parse_argstring(self.pxconfig, line) + if args.targets: + self.view.targets = self._eval_target_str(args.targets) + if args.block is not None: + self.view.block = args.block + if args.set_verbose is not None: + self.verbose = args.set_verbose + + @magic_arguments.magic_arguments() + @output_args + @skip_doctest + def result(self, line=''): + """Print the result of the last asynchronous %px command. + + This lets you recall the results of %px computations after + asynchronous submission (block=False). + + Examples + -------- + :: + + In [23]: %px os.getpid() + Async parallel execution on engine(s): all + + In [24]: %pxresult + Out[8:10]: 60920 + Out[9:10]: 60921 + Out[10:10]: 60922 + Out[11:10]: 60923 + """ + args = magic_arguments.parse_argstring(self.result, line) + + if self.last_result is None: + raise UsageError(NO_LAST_RESULT) + + self.last_result.get() + self.last_result.display_outputs(groupby=args.groupby) + + @skip_doctest + def px(self, line=''): + """Executes the given python command in parallel. + + Examples + -------- + :: + + In [24]: %px a = os.getpid() + Parallel execution on engine(s): all + + In [25]: %px print a + [stdout:0] 1234 + [stdout:1] 1235 + [stdout:2] 1236 + [stdout:3] 1237 + """ + return self.parallel_execute(line) + + def parallel_execute(self, cell, block=None, groupby='type', save_name=None): + """implementation used by %px and %%parallel""" + + # defaults: + block = self.view.block if block is None else block + + base = "Parallel" if block else "Async parallel" + + targets = self.view.targets + if isinstance(targets, list) and len(targets) > 10: + str_targets = str(targets[:4])[:-1] + ', ..., ' + str(targets[-4:])[1:] + else: + str_targets = str(targets) + if self.verbose: + print base + " execution on engine(s): %s" % str_targets + + result = self.view.execute(cell, silent=False, block=False) + self.last_result = result + + if save_name: + self.shell.user_ns[save_name] = result + + if block: + result.get() + result.display_outputs(groupby) + else: + # return AsyncResult only on non-blocking submission + return result + + @magic_arguments.magic_arguments() + @exec_args + @output_args + @skip_doctest + def cell_px(self, line='', cell=None): + """Executes the cell in parallel. + + Examples + -------- + :: + + In [24]: %%px --noblock + ....: a = os.getpid() + Async parallel execution on engine(s): all + + In [25]: %%px + ....: print a + [stdout:0] 1234 + [stdout:1] 1235 + [stdout:2] 1236 + [stdout:3] 1237 + """ + + args = magic_arguments.parse_argstring(self.cell_px, line) + + if args.targets: + save_targets = self.view.targets + self.view.targets = self._eval_target_str(args.targets) + try: + return self.parallel_execute(cell, block=args.block, + groupby=args.groupby, + save_name=args.save_name, + ) + finally: + if args.targets: + self.view.targets = save_targets + + @skip_doctest + def autopx(self, line=''): + """Toggles auto parallel mode. + + Once this is called, all commands typed at the command line are send to + the engines to be executed in parallel. To control which engine are + used, the ``targets`` attribute of the view before + entering ``%autopx`` mode. + + + Then you can do the following:: + + In [25]: %autopx + %autopx to enabled + + In [26]: a = 10 + Parallel execution on engine(s): [0,1,2,3] + In [27]: print a + Parallel execution on engine(s): [0,1,2,3] + [stdout:0] 10 + [stdout:1] 10 + [stdout:2] 10 + [stdout:3] 10 + + + In [27]: %autopx + %autopx disabled + """ + if self._autopx: + self._disable_autopx() + else: + self._enable_autopx() + + def _enable_autopx(self): + """Enable %autopx mode by saving the original run_cell and installing + pxrun_cell. + """ + # override run_cell + self._original_run_cell = self.shell.run_cell + self.shell.run_cell = self.pxrun_cell + + self._autopx = True + print "%autopx enabled" + + def _disable_autopx(self): + """Disable %autopx by restoring the original InteractiveShell.run_cell. + """ + if self._autopx: + self.shell.run_cell = self._original_run_cell + self._autopx = False + print "%autopx disabled" + + def pxrun_cell(self, raw_cell, store_history=False, silent=False): + """drop-in replacement for InteractiveShell.run_cell. + + This executes code remotely, instead of in the local namespace. + + See InteractiveShell.run_cell for details. + """ + + if (not raw_cell) or raw_cell.isspace(): + return + + ipself = self.shell + + with ipself.builtin_trap: + cell = ipself.prefilter_manager.prefilter_lines(raw_cell) + + # Store raw and processed history + if store_history: + ipself.history_manager.store_inputs(ipself.execution_count, + cell, raw_cell) + + # ipself.logger.log(cell, raw_cell) + + cell_name = ipself.compile.cache(cell, ipself.execution_count) + + try: + ast.parse(cell, filename=cell_name) + except (OverflowError, SyntaxError, ValueError, TypeError, + MemoryError): + # Case 1 + ipself.showsyntaxerror() + ipself.execution_count += 1 + return None + except NameError: + # ignore name errors, because we don't know the remote keys + pass + + if store_history: + # Write output to the database. Does nothing unless + # history output logging is enabled. + ipself.history_manager.store_output(ipself.execution_count) + # Each cell is a *single* input, regardless of how many lines it has + ipself.execution_count += 1 + if re.search(r'get_ipython\(\)\.magic\(u?["\']%?autopx', cell): + self._disable_autopx() + return False + else: + try: + result = self.view.execute(cell, silent=False, block=False) + except: + ipself.showtraceback() + return True + else: + if self.view.block: + try: + result.get() + except: + self.shell.showtraceback() + return True + else: + with ipself.builtin_trap: + result.display_outputs() + return False + + +__doc__ = __doc__.format( + AUTOPX_DOC = ' '*8 + ParallelMagics.autopx.__doc__, + PX_DOC = ' '*8 + ParallelMagics.px.__doc__, + RESULT_DOC = ' '*8 + ParallelMagics.result.__doc__, + CONFIG_DOC = ' '*8 + ParallelMagics.pxconfig.__doc__, +) diff --git a/IPython/parallel/client/view.py b/IPython/parallel/client/view.py index 7707516..5d047da 100644 --- a/IPython/parallel/client/view.py +++ b/IPython/parallel/client/view.py @@ -60,8 +60,6 @@ def sync_results(f, self, *args, **kwargs): delta = self.outstanding.difference(self.client.outstanding) completed = self.outstanding.intersection(delta) self.outstanding = self.outstanding.difference(completed) - for msg_id in completed: - self.results[msg_id] = self.client.results[msg_id] return ret @decorator @@ -122,6 +120,7 @@ class View(HasTraits): def __init__(self, client=None, socket=None, **flags): super(View, self).__init__(client=client, _socket=socket) + self.results = client.results self.block = client.block self.set_flags(**flags) @@ -792,33 +791,37 @@ class DirectView(View): return self.client.kill(targets=targets, block=block) #---------------------------------------- - # activate for %px,%autopx magics + # activate for %px, %autopx, etc. magics #---------------------------------------- - def activate(self): - """Make this `View` active for parallel magic commands. - IPython has a magic command syntax to work with `MultiEngineClient` objects. - In a given IPython session there is a single active one. While - there can be many `Views` created and used by the user, - there is only one active one. The active `View` is used whenever - the magic commands %px and %autopx are used. - - The activate() method is called on a given `View` to make it - active. Once this has been done, the magic commands can be used. + def activate(self, suffix=''): + """Activate IPython magics associated with this View + + Defines the magics `%px, %autopx, %pxresult, %%px, %pxconfig` + + Parameters + ---------- + + suffix: str [default: ''] + The suffix, if any, for the magics. This allows you to have + multiple views associated with parallel magics at the same time. + + e.g. ``rc[::2].activate(suffix='_even')`` will give you + the magics ``%px_even``, ``%pxresult_even``, etc. for running magics + on the even engines. """ - + + from IPython.parallel.client.magics import ParallelMagics + try: # This is injected into __builtins__. ip = get_ipython() except NameError: - print "The IPython parallel magics (%result, %px, %autopx) only work within IPython." - else: - pmagic = ip.magics_manager.registry.get('ParallelMagics') - if pmagic is None: - ip.magic('load_ext parallelmagic') - pmagic = ip.magics_manager.registry.get('ParallelMagics') - - pmagic.active_view = self + print "The IPython parallel magics (%px, etc.) only work within IPython." + return + + M = ParallelMagics(ip, self, suffix) + ip.magics_manager.register(M) @skip_doctest diff --git a/IPython/parallel/error.py b/IPython/parallel/error.py index abfb578..910c422 100644 --- a/IPython/parallel/error.py +++ b/IPython/parallel/error.py @@ -188,11 +188,17 @@ class RemoteError(KernelError): return ""%(engineid, self.ename, self.evalue) def __str__(self): - sig = "%s(%s)"%(self.ename, self.evalue) - if self.traceback: - return sig + '\n' + self.traceback - else: - return sig + return "%s(%s)" % (self.ename, self.evalue) + + def render_traceback(self): + """render traceback to a list of lines""" + return (self.traceback or "No traceback available").splitlines() + + def print_traceback(self, excid=None): + """print my traceback""" + print('\n'.join(self.render_traceback())) + + class TaskRejectError(KernelError): @@ -243,21 +249,28 @@ class CompositeError(RemoteError): def __repr__(self): return "CompositeError(%i)"%len(self.elist) - - def print_tracebacks(self, excid=None): + + def render_traceback(self, excid=None): + """render one or all of my tracebacks to a list of lines""" + lines = [] if excid is None: for (en,ev,etb,ei) in self.elist: - print (self._get_engine_str(ei)) - print (etb or 'No traceback available') - print () + lines.append(self._get_engine_str(ei)) + lines.extend((etb or 'No traceback available').splitlines()) + lines.append('') else: try: en,ev,etb,ei = self.elist[excid] except: raise IndexError("an exception with index %i does not exist"%excid) else: - print (self._get_engine_str(ei)) - print (etb or 'No traceback available') + lines.append(self._get_engine_str(ei)) + lines.extend((etb or 'No traceback available').splitlines()) + + return lines + + def print_traceback(self, excid=None): + print('\n'.join(self.render_traceback(excid))) def raise_exception(self, excid=0): try: diff --git a/IPython/parallel/tests/test_client.py b/IPython/parallel/tests/test_client.py index 263c0a4..6f108e5 100644 --- a/IPython/parallel/tests/test_client.py +++ b/IPython/parallel/tests/test_client.py @@ -24,6 +24,7 @@ from tempfile import mktemp import zmq +from IPython import parallel from IPython.parallel.client import client as clientmod from IPython.parallel import error from IPython.parallel import AsyncResult, AsyncHubResult @@ -284,16 +285,16 @@ class TestClient(ClusterTestCase): """wait for an engine to become idle, according to the Hub""" rc = self.client - # timeout 2s, polling every 100ms - for i in range(20): - qs = rc.queue_status() + # timeout 5s, polling every 100ms + qs = rc.queue_status() + for i in range(50): if qs['unassigned'] or any(qs[eid]['tasks'] for eid in rc.ids): time.sleep(0.1) + qs = rc.queue_status() else: break # ensure Hub up to date: - qs = rc.queue_status() self.assertEquals(qs['unassigned'], 0) for eid in rc.ids: self.assertEquals(qs[eid]['tasks'], 0) @@ -420,3 +421,16 @@ class TestClient(ClusterTestCase): "Shouldn't be spinning, but got wall_time=%f" % ar.wall_time ) + def test_activate(self): + ip = get_ipython() + magics = ip.magics_manager.magics + self.assertTrue('px' in magics['line']) + self.assertTrue('px' in magics['cell']) + v0 = self.client.activate(-1, '0') + self.assertTrue('px0' in magics['line']) + self.assertTrue('px0' in magics['cell']) + self.assertEquals(v0.targets, self.client.ids[-1]) + v0 = self.client.activate('all', 'all') + self.assertTrue('pxall' in magics['line']) + self.assertTrue('pxall' in magics['cell']) + self.assertEquals(v0.targets, 'all') diff --git a/IPython/parallel/tests/test_magics.py b/IPython/parallel/tests/test_magics.py index eeda8bd..f488db1 100644 --- a/IPython/parallel/tests/test_magics.py +++ b/IPython/parallel/tests/test_magics.py @@ -88,7 +88,7 @@ class TestParallelMagics(ClusterTestCase, ParametricTestCase): for block in (True, False): v.block = block - + ip.magic("pxconfig --verbose") with capture_output() as io: ip.run_cell_magic("px", "", "1") if block: @@ -117,7 +117,7 @@ class TestParallelMagics(ClusterTestCase, ParametricTestCase): ip.run_cell_magic('px', '--group-outputs=engine', 'generate_output()') self.assertFalse('\n\n' in io.stdout) - lines = io.stdout.splitlines()[1:] + lines = io.stdout.splitlines() expected = [ r'\[stdout:\d+\]', 'stdout', @@ -151,7 +151,7 @@ class TestParallelMagics(ClusterTestCase, ParametricTestCase): ip.run_cell_magic('px', '--group-outputs=order', 'generate_output()') self.assertFalse('\n\n' in io.stdout) - lines = io.stdout.splitlines()[1:] + lines = io.stdout.splitlines() expected = [] expected.extend([ r'\[stdout:\d+\]', @@ -192,7 +192,7 @@ class TestParallelMagics(ClusterTestCase, ParametricTestCase): ip.run_cell_magic('px', '--group-outputs=type', 'generate_output()') self.assertFalse('\n\n' in io.stdout) - lines = io.stdout.splitlines()[1:] + lines = io.stdout.splitlines() expected = [] expected.extend([ @@ -229,6 +229,7 @@ class TestParallelMagics(ClusterTestCase, ParametricTestCase): self.assertEquals(v['a'], [5]) ip.magic('px a=10') self.assertEquals(v['a'], [10]) + ip.magic('pxconfig --verbose') with capture_output() as io: ar = ip.magic('px print (a)') self.assertTrue(isinstance(ar, AsyncResult)) @@ -258,7 +259,7 @@ class TestParallelMagics(ClusterTestCase, ParametricTestCase): self.assertTrue(output.startswith('%autopx enabled'), output) self.assertTrue(output.rstrip().endswith('%autopx disabled'), output) - self.assertTrue('RemoteError: ZeroDivisionError' in output, output) + self.assertTrue('ZeroDivisionError' in output, output) self.assertTrue('\nOut[' in output, output) self.assertTrue(': 24690' in output, output) ar = v.get_result(-1) @@ -301,20 +302,13 @@ class TestParallelMagics(ClusterTestCase, ParametricTestCase): data = dict(a=111,b=222) v.push(data, block=True) - ip.magic('px a') - ip.magic('px b') - for idx, name in [ - ('', 'b'), - ('-1', 'b'), - ('2', 'b'), - ('1', 'a'), - ('-2', 'a'), - ]: + for name in ('a', 'b'): + ip.magic('px ' + name) with capture_output() as io: - ip.magic('result ' + idx) + ip.magic('pxresult') output = io.stdout msg = "expected %s output to include %s, but got: %s" % \ - ('%result '+idx, str(data[name]), output) + ('%pxresult', str(data[name]), output) self.assertTrue(str(data[name]) in output, msg) @dec.skipif_not_matplotlib @@ -336,5 +330,57 @@ class TestParallelMagics(ClusterTestCase, ParametricTestCase): self.assertTrue('Out[' in io.stdout, io.stdout) self.assertTrue('matplotlib.lines' in io.stdout, io.stdout) - + + def test_pxconfig(self): + ip = get_ipython() + rc = self.client + v = rc.activate(-1, '_tst') + self.assertEquals(v.targets, rc.ids[-1]) + ip.magic("%pxconfig_tst -t :") + self.assertEquals(v.targets, rc.ids) + ip.magic("%pxconfig_tst -t ::2") + self.assertEquals(v.targets, rc.ids[::2]) + ip.magic("%pxconfig_tst -t 1::2") + self.assertEquals(v.targets, rc.ids[1::2]) + ip.magic("%pxconfig_tst -t 1") + self.assertEquals(v.targets, 1) + ip.magic("%pxconfig_tst --block") + self.assertEquals(v.block, True) + ip.magic("%pxconfig_tst --noblock") + self.assertEquals(v.block, False) + + def test_cellpx_targets(self): + """%%px --targets doesn't change defaults""" + ip = get_ipython() + rc = self.client + view = rc.activate(rc.ids) + self.assertEquals(view.targets, rc.ids) + ip.magic('pxconfig --verbose') + for cell in ("pass", "1/0"): + with capture_output() as io: + try: + ip.run_cell_magic("px", "--targets all", cell) + except pmod.RemoteError: + pass + self.assertTrue('engine(s): all' in io.stdout) + self.assertEquals(view.targets, rc.ids) + + + def test_cellpx_block(self): + """%%px --block doesn't change default""" + ip = get_ipython() + rc = self.client + view = rc.activate(rc.ids) + view.block = False + self.assertEquals(view.targets, rc.ids) + ip.magic('pxconfig --verbose') + for cell in ("pass", "1/0"): + with capture_output() as io: + try: + ip.run_cell_magic("px", "--block", cell) + except pmod.RemoteError: + pass + self.assertFalse('Async' in io.stdout) + self.assertFalse(view.block) + diff --git a/IPython/parallel/tests/test_view.py b/IPython/parallel/tests/test_view.py index e39b494..4f146a8 100644 --- a/IPython/parallel/tests/test_view.py +++ b/IPython/parallel/tests/test_view.py @@ -469,7 +469,6 @@ class TestView(ClusterTestCase, ParametricTestCase): e0.block = True ar = e0.execute("5", silent=False) er = ar.get() - self._wait_for(lambda : bool(er.pyout)) self.assertEquals(str(er), "" % er.execution_count) self.assertEquals(er.pyout['data']['text/plain'], '5') @@ -478,14 +477,12 @@ class TestView(ClusterTestCase, ParametricTestCase): e0.block = True ar = e0.execute("print (5)", silent=False) er = ar.get() - self._wait_for(lambda : bool(er.stdout)) self.assertEquals(er.stdout.strip(), '5') def test_execute_pyout(self): """execute triggers pyout with silent=False""" view = self.client[:] ar = view.execute("5", silent=False, block=True) - self._wait_for(lambda : all(ar.pyout)) expected = [{'text/plain' : '5'}] * len(view) mimes = [ out['data'] for out in ar.pyout ] @@ -505,7 +502,6 @@ class TestView(ClusterTestCase, ParametricTestCase): ar = view.execute("%whos", block=True) # this will raise, if that failed ar.get(5) - self._wait_for(lambda : all(ar.stdout)) for stdout in ar.stdout: lines = stdout.splitlines() self.assertEquals(lines[0].split(), ['Variable', 'Type', 'Data/Info']) @@ -523,7 +519,6 @@ class TestView(ClusterTestCase, ParametricTestCase): view.execute("from IPython.core.display import *") ar = view.execute("[ display(i) for i in range(5) ]", block=True) - self._wait_for(lambda : all(len(er.outputs) >= 5 for er in ar)) expected = [ {u'text/plain' : unicode(j)} for j in range(5) ] for outputs in ar.outputs: mimes = [ out['data'] for out in outputs ] @@ -540,7 +535,6 @@ class TestView(ClusterTestCase, ParametricTestCase): ar = view.apply_async(publish) ar.get(5) - self._wait_for(lambda : all(len(out) >= 5 for out in ar.outputs)) expected = [ {u'text/plain' : unicode(j)} for j in range(5) ] for outputs in ar.outputs: mimes = [ out['data'] for out in outputs ] @@ -562,7 +556,6 @@ class TestView(ClusterTestCase, ParametricTestCase): # include imports, in case user config ar = view.execute("plot(rand(100))", silent=False) reply = ar.get(5) - self._wait_for(lambda : all(ar.outputs)) self.assertEquals(len(reply.outputs), 1) output = reply.outputs[0] self.assertTrue("data" in output) diff --git a/IPython/zmq/ipkernel.py b/IPython/zmq/ipkernel.py index a2f8c91..9722ba4 100755 --- a/IPython/zmq/ipkernel.py +++ b/IPython/zmq/ipkernel.py @@ -308,16 +308,22 @@ class Kernel(Configurable): {u'code':code, u'execution_count': execution_count}, parent=parent, ident=self._topic('pyin') ) - - def execute_request(self, stream, ident, parent): - + + def _publish_status(self, status, parent=None): + """send status (busy/idle) on IOPub""" self.session.send(self.iopub_socket, u'status', - {u'execution_state':u'busy'}, + {u'execution_state': status}, parent=parent, ident=self._topic('status'), ) + + def execute_request(self, stream, ident, parent): + """handle an execute_request""" + + self._publish_status(u'busy', parent) + try: content = parent[u'content'] code = content[u'code'] @@ -433,11 +439,7 @@ class Kernel(Configurable): if not silent and reply_msg['content']['status'] == u'error': self._abort_queues() - self.session.send(self.iopub_socket, - u'status', - {u'execution_state':u'idle'}, - parent=parent, - ident=self._topic('status')) + self._publish_status(u'idle', parent) def complete_request(self, stream, ident, parent): txt, matches = self._complete(parent) @@ -529,9 +531,12 @@ class Kernel(Configurable): self.log.error("Got bad msg: %s", parent, exc_info=True) return + self._publish_status(u'busy', parent) + # Set the parent message of the display hook and out streams. - self.shell.displayhook.set_parent(parent) - self.shell.display_pub.set_parent(parent) + shell = self.shell + shell.displayhook.set_parent(parent) + shell.display_pub.set_parent(parent) sys.stdout.set_parent(parent) sys.stderr.set_parent(parent) @@ -540,7 +545,7 @@ class Kernel(Configurable): # self.session.send(self.iopub_socket, u'pyin', {u'code':code},parent=parent) sub = self._make_subheader() try: - working = self.shell.user_ns + working = shell.user_ns prefix = "_"+str(msg_id).replace("-","")+"_" @@ -558,7 +563,7 @@ class Kernel(Configurable): working.update(ns) code = "%s = %s(*%s,**%s)" % (resultname, fname, argname, kwargname) try: - exec code in self.shell.user_global_ns, self.shell.user_ns + exec code in shell.user_global_ns, shell.user_ns result = working.get(resultname) finally: for key in ns.iterkeys(): @@ -567,14 +572,23 @@ class Kernel(Configurable): packed_result,buf = serialize_object(result) result_buf = [packed_result]+buf except: - exc_content = self._wrap_exception('apply') - # exc_msg = self.session.msg(u'pyerr', exc_content, parent) - self.session.send(self.iopub_socket, u'pyerr', exc_content, parent=parent, + # invoke IPython traceback formatting + shell.showtraceback() + # FIXME - fish exception info out of shell, possibly left there by + # run_code. We'll need to clean up this logic later. + reply_content = {} + if shell._reply_content is not None: + reply_content.update(shell._reply_content) + e_info = dict(engine_uuid=self.ident, engine_id=self.int_id, method='apply') + reply_content['engine_info'] = e_info + # reset after use + shell._reply_content = None + + self.session.send(self.iopub_socket, u'pyerr', reply_content, parent=parent, ident=self._topic('pyerr')) - reply_content = exc_content result_buf = [] - if exc_content['ename'] == 'UnmetDependency': + if reply_content['ename'] == 'UnmetDependency': sub['dependencies_met'] = False else: reply_content = {'status' : 'ok'} @@ -589,6 +603,8 @@ class Kernel(Configurable): reply_msg = self.session.send(stream, u'apply_reply', reply_content, parent=parent, ident=ident,buffers=result_buf, subheader=sub) + self._publish_status(u'idle', parent) + #--------------------------------------------------------------------------- # Control messages #--------------------------------------------------------------------------- diff --git a/docs/examples/parallel/Parallel Magics.ipynb b/docs/examples/parallel/Parallel Magics.ipynb index b0a02eb..1768b1a 100644 --- a/docs/examples/parallel/Parallel Magics.ipynb +++ b/docs/examples/parallel/Parallel Magics.ipynb @@ -16,21 +16,19 @@ { "cell_type": "markdown", "source": [ - "IPython has a few magics for working with your engines.", - "", - "This assumes you have started an IPython cluster, either with the notebook interface,", + "IPython has a few magics for working with your engines.\n", + "\n", + "This assumes you have started an IPython cluster, either with the notebook interface,\n", "or the `ipcluster/controller/engine` commands." ] }, { "cell_type": "code", - "collapsed": false, "input": [ - "from IPython import parallel", - "rc = parallel.Client()", - "dv = rc[:]", - "dv.block = True", - "dv" + "from IPython import parallel\n", + "rc = parallel.Client()\n", + "dv = rc[:]\n", + "rc.ids" ], "language": "python", "outputs": [] @@ -38,21 +36,11 @@ { "cell_type": "markdown", "source": [ - "The parallel magics come from the `parallelmagics` IPython extension.", - "The magics are set to work with a particular View object,", - "so to activate them, you call the `activate()` method on a particular view:" + "Creating a Client registers the parallel magics `%px`, `%%px`, `%pxresult`, `pxconfig`, and `%autopx`. \n", + "These magics are initially associated with a DirectView always associated with all currently registered engines." ] }, { - "cell_type": "code", - "collapsed": true, - "input": [ - "dv.activate()" - ], - "language": "python", - "outputs": [] - }, - { "cell_type": "markdown", "source": [ "Now we can execute code remotely with `%px`:" @@ -60,7 +48,6 @@ }, { "cell_type": "code", - "collapsed": false, "input": [ "%px a=5" ], @@ -69,7 +56,6 @@ }, { "cell_type": "code", - "collapsed": false, "input": [ "%px print a" ], @@ -78,7 +64,6 @@ }, { "cell_type": "code", - "collapsed": false, "input": [ "%px a" ], @@ -87,9 +72,8 @@ }, { "cell_type": "code", - "collapsed": false, "input": [ - "with dv.sync_imports():", + "with dv.sync_imports():\n", " import sys" ], "language": "python", @@ -97,7 +81,6 @@ }, { "cell_type": "code", - "collapsed": false, "input": [ "%px print >> sys.stderr, \"ERROR\"" ], @@ -107,24 +90,22 @@ { "cell_type": "markdown", "source": [ - "You don't have to wait for results:" + "You don't have to wait for results. The `%pxconfig` magic lets you change the default blocking/targets for the `%px` magics:" ] }, { "cell_type": "code", - "collapsed": true, "input": [ - "dv.block = False" + "%pxconfig --noblock" ], "language": "python", "outputs": [] }, { "cell_type": "code", - "collapsed": false, "input": [ - "%px import time", - "%px time.sleep(5)", + "%px import time\n", + "%px time.sleep(5)\n", "%px time.time()" ], "language": "python", @@ -133,15 +114,14 @@ { "cell_type": "markdown", "source": [ - "But you will notice that this didn't output the result of the last command.", - "For this, we have `%result`, which displays the output of the latest request:" + "But you will notice that this didn't output the result of the last command.\n", + "For this, we have `%pxresult`, which displays the output of the latest request:" ] }, { "cell_type": "code", - "collapsed": false, "input": [ - "%result" + "%pxresult" ], "language": "python", "outputs": [] @@ -154,9 +134,8 @@ }, { "cell_type": "code", - "collapsed": false, "input": [ - "dv.block = True", + "%pxconfig --block\n", "%px %pylab inline" ], "language": "python", @@ -165,17 +144,15 @@ { "cell_type": "markdown", "source": [ - "`%%px` can also be used as a cell magic, for submitting whole blocks.", - "This one acceps `--block` and `--noblock` flags to specify", - "the blocking behavior, though the default is unchanged.", - "" + "`%%px` can also be used as a cell magic, for submitting whole blocks.\n", + "This one acceps `--block` and `--noblock` flags to specify\n", + "the blocking behavior, though the default is unchanged.\n" ] }, { "cell_type": "code", - "collapsed": true, "input": [ - "dv.scatter('id', dv.targets, flatten=True)", + "dv.scatter('id', dv.targets, flatten=True)\n", "dv['stride'] = len(dv)" ], "language": "python", @@ -183,13 +160,12 @@ }, { "cell_type": "code", - "collapsed": false, "input": [ - "%%px --noblock", - "x = linspace(0,pi,1000)", - "for n in range(id,12, stride):", - " print n", - " plt.plot(x,sin(n*x))", + "%%px --noblock\n", + "x = linspace(0,pi,1000)\n", + "for n in range(id,12, stride):\n", + " print n\n", + " plt.plot(x,sin(n*x))\n", "plt.title(\"Plot %i\" % id)" ], "language": "python", @@ -197,9 +173,8 @@ }, { "cell_type": "code", - "collapsed": false, "input": [ - "%result" + "%pxresult" ], "language": "python", "outputs": [] @@ -207,28 +182,27 @@ { "cell_type": "markdown", "source": [ - "It also lets you choose some amount of the grouping of the outputs with `--group-outputs`:", - "", - "The choices are:", - "", - "* `engine` - all of an engine's output is collected together", - "* `type` - where stdout of each engine is grouped, etc. (the default)", - "* `order` - same as `type`, but individual displaypub outputs are interleaved.", - " That is, it will output the first plot from each engine, then the second from each,", + "It also lets you choose some amount of the grouping of the outputs with `--group-outputs`:\n", + "\n", + "The choices are:\n", + "\n", + "* `engine` - all of an engine's output is collected together\n", + "* `type` - where stdout of each engine is grouped, etc. (the default)\n", + "* `order` - same as `type`, but individual displaypub outputs are interleaved.\n", + " That is, it will output the first plot from each engine, then the second from each,\n", " etc." ] }, { "cell_type": "code", - "collapsed": false, "input": [ - "%%px --group-outputs=engine", - "x = linspace(0,pi,1000)", - "for n in range(id,12, stride):", - " print n", - " plt.figure()", - " plt.plot(x,sin(n*x))", - "plt.title(\"Plot %i\" % id)" + "%%px --group-outputs=engine\n", + "x = linspace(0,pi,1000)\n", + "for n in range(id+1,12, stride):\n", + " print n\n", + " plt.figure()\n", + " plt.plot(x,sin(n*x))\n", + " plt.title(\"Plot %i\" % n)" ], "language": "python", "outputs": [] @@ -236,20 +210,16 @@ { "cell_type": "markdown", "source": [ - "When you specify 'order', then individual display outputs (e.g. plots) will be interleaved:" + "When you specify 'order', then individual display outputs (e.g. plots) will be interleaved.\n", + "\n", + "`%pxresult` takes the same output-ordering arguments as `%%px`, \n", + "so you can view the previous result in a variety of different ways with a few sequential calls to `%pxresult`:" ] }, { "cell_type": "code", - "collapsed": false, "input": [ - "%%px --group-outputs=order", - "x = linspace(0,pi,1000)", - "for n in range(id,12, stride):", - " print n", - " plt.figure()", - " plt.plot(x,sin(n*x))", - "plt.title(\"Plot %i\" % id)" + "%pxresult --group-outputs=order" ], "language": "python", "outputs": [] @@ -269,53 +239,144 @@ }, { "cell_type": "code", - "collapsed": true, "input": [ - "def generate_output():", - " \"\"\"function for testing output", - " ", - " publishes two outputs of each type, and returns something", - " \"\"\"", - " ", - " import sys,os", - " from IPython.core.display import display, HTML, Math", - " ", - " print \"stdout\"", - " print >> sys.stderr, \"stderr\"", - " ", - " display(HTML(\"HTML\"))", - " ", - " print \"stdout2\"", - " print >> sys.stderr, \"stderr2\"", - " ", - " display(Math(r\"\\alpha=\\beta\"))", - " ", - " return os.getpid()", - "", + "def generate_output():\n", + " \"\"\"function for testing output\n", + " \n", + " publishes two outputs of each type, and returns something\n", + " \"\"\"\n", + " \n", + " import sys,os\n", + " from IPython.core.display import display, HTML, Math\n", + " \n", + " print \"stdout\"\n", + " print >> sys.stderr, \"stderr\"\n", + " \n", + " display(HTML(\"HTML\"))\n", + " \n", + " print \"stdout2\"\n", + " print >> sys.stderr, \"stderr2\"\n", + " \n", + " display(Math(r\"\\alpha=\\beta\"))\n", + " \n", + " return os.getpid()\n", + "\n", "dv['generate_output'] = generate_output" ], "language": "python", "outputs": [] }, { + "cell_type": "markdown", + "source": [ + "You can also have more than one set of parallel magics registered at a time.\n", + "\n", + "The `View.activate()` method takes a suffix argument, which is added to `'px'`." + ] + }, + { + "cell_type": "code", + "input": [ + "e0 = rc[-1]\n", + "e0.block = True\n", + "e0.activate('0')" + ], + "language": "python", + "outputs": [] + }, + { "cell_type": "code", - "collapsed": true, "input": [ - "e0 = rc[-1]", - "e0.block = True", - "e0.activate()" + "%px0 generate_output()" ], "language": "python", "outputs": [] }, { "cell_type": "code", - "collapsed": false, "input": [ "%px generate_output()" ], "language": "python", "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "As mentioned above, we can redisplay those same results with various grouping:" + ] + }, + { + "cell_type": "code", + "input": [ + "%pxresult --group-outputs order" + ], + "language": "python", + "outputs": [] + }, + { + "cell_type": "code", + "input": [ + "%pxresult --group-outputs engine" + ], + "language": "python", + "outputs": [] + }, + { + "cell_type": "heading", + "level": 2, + "source": [ + "Parallel Exceptions" + ] + }, + { + "cell_type": "markdown", + "source": [ + "When you raise exceptions with the parallel exception,\n", + "the CompositeError raised locally will display your remote traceback." + ] + }, + { + "cell_type": "code", + "input": [ + "%%px\n", + "from numpy.random import random\n", + "A = random((100,100,'invalid shape'))" + ], + "language": "python", + "outputs": [] + }, + { + "cell_type": "heading", + "level": 2, + "source": [ + "Remote Cell Magics" + ] + }, + { + "cell_type": "markdown", + "source": [ + "Remember, Engines are IPython too, so the cell that is run remotely by %%px can in turn use a cell magic." + ] + }, + { + "cell_type": "code", + "input": [ + "%%px\n", + "%%timeit\n", + "from numpy.random import random\n", + "from numpy.linalg import norm\n", + "A = random((100,100))\n", + "norm(A, 2) " + ], + "language": "python", + "outputs": [] + }, + { + "cell_type": "code", + "input": [], + "language": "python", + "outputs": [] } ] } diff --git a/docs/source/config/extensions/index.txt b/docs/source/config/extensions/index.txt index b986f76..f722a1e 100644 --- a/docs/source/config/extensions/index.txt +++ b/docs/source/config/extensions/index.txt @@ -72,7 +72,6 @@ Extensions bundled with IPython autoreload cythonmagic - parallelmagic rmagic storemagic sympyprinting diff --git a/docs/source/config/extensions/parallelmagic.txt b/docs/source/config/extensions/parallelmagic.txt deleted file mode 100644 index 10716a9..0000000 --- a/docs/source/config/extensions/parallelmagic.txt +++ /dev/null @@ -1,7 +0,0 @@ -.. _extensions_parallelmagic: - -============= -parallelmagic -============= - -.. automodule:: IPython.extensions.parallelmagic diff --git a/docs/source/parallel/asyncresult.txt b/docs/source/parallel/asyncresult.txt index a479081..ffe3593 100644 --- a/docs/source/parallel/asyncresult.txt +++ b/docs/source/parallel/asyncresult.txt @@ -101,11 +101,45 @@ Map results are iterable! ========================= When an AsyncResult object has multiple results (e.g. the :class:`~AsyncMapResult` -object), you can actually iterate through them, and act on the results as they arrive: +object), you can actually iterate through results themselves, and act on them as they arrive: .. literalinclude:: ../../examples/parallel/itermapresult.py :language: python :lines: 20-67 + +That is to say, if you treat an AsyncMapResult as if it were a list of your actual +results, it should behave as you would expect, with the only difference being +that you can start iterating through the results before they have even been computed. + +This lets you do a dumb version of map/reduce with the builtin Python functions, +and the only difference between doing this locally and doing it remotely in parallel +is using the asynchronous view.map instead of the builtin map. + + +Here is a simple one-line RMS (root-mean-square) implemented with Python's builtin map/reduce. + +.. sourcecode:: ipython + + In [38]: X = np.linspace(0,100) + + In [39]: from math import sqrt + + In [40]: add = lambda a,b: a+b + + In [41]: sq = lambda x: x*x + + In [42]: sqrt(reduce(add, map(sq, X)) / len(X)) + Out[42]: 58.028845747399714 + + In [43]: sqrt(reduce(add, view.map(sq, X)) / len(X)) + Out[43]: 58.028845747399714 + +To break that down: + +1. ``map(sq, X)`` Compute the square of each element in the list (locally, or in parallel) +2. ``reduce(add, sqX) / len(X)`` compute the mean by summing over the list (or AsyncMapResult) + and dividing by the size +3. take the square root of the resulting number .. seealso:: diff --git a/docs/source/parallel/index.txt b/docs/source/parallel/index.txt index 9bfbc76..be8b957 100644 --- a/docs/source/parallel/index.txt +++ b/docs/source/parallel/index.txt @@ -10,6 +10,7 @@ Using IPython for parallel computing parallel_intro.txt parallel_process.txt parallel_multiengine.txt + magics.txt parallel_task.txt asyncresult.txt parallel_mpi.txt diff --git a/docs/source/parallel/magics.txt b/docs/source/parallel/magics.txt new file mode 100644 index 0000000..d9db175 --- /dev/null +++ b/docs/source/parallel/magics.txt @@ -0,0 +1,389 @@ +.. _parallel_magics: + +======================= +Parallel Magic Commands +======================= + +We provide a few IPython magic commands +that make it a bit more pleasant to execute Python commands on the engines interactively. +These are mainly shortcuts to :meth:`.DirectView.execute` +and :meth:`.AsyncResult.display_outputs` methods repsectively. + +These magics will automatically become available when you create a Client: + +.. sourcecode:: ipython + + In [2]: rc = parallel.Client() + +The initially active View will have attributes ``targets='all', block=True``, +which is a blocking view of all engines, evaluated at request time +(adding/removing engines will change where this view's tasks will run). + +The Magics +========== + +%px +--- + +The %px magic executes a single Python command on the engines +specified by the :attr:`targets` attribute of the :class:`DirectView` instance: + +.. sourcecode:: ipython + + # import numpy here and everywhere + In [25]: with rc[:].sync_imports(): + ....: import numpy + importing numpy on engine(s) + + In [27]: %px a = numpy.random.rand(2,2) + Parallel execution on engines: [0, 1, 2, 3] + + In [28]: %px numpy.linalg.eigvals(a) + Parallel execution on engines: [0, 1, 2, 3] + Out [0:68]: array([ 0.77120707, -0.19448286]) + Out [1:68]: array([ 1.10815921, 0.05110369]) + Out [2:68]: array([ 0.74625527, -0.37475081]) + Out [3:68]: array([ 0.72931905, 0.07159743]) + + In [29]: %px print 'hi' + Parallel execution on engine(s): all + [stdout:0] hi + [stdout:1] hi + [stdout:2] hi + [stdout:3] hi + + +Since engines are IPython as well, you can even run magics remotely: + +.. sourcecode:: ipython + + In [28]: %px %pylab inline + Parallel execution on engine(s): all + [stdout:0] + Welcome to pylab, a matplotlib-based Python environment... + For more information, type 'help(pylab)'. + [stdout:1] + Welcome to pylab, a matplotlib-based Python environment... + For more information, type 'help(pylab)'. + [stdout:2] + Welcome to pylab, a matplotlib-based Python environment... + For more information, type 'help(pylab)'. + [stdout:3] + Welcome to pylab, a matplotlib-based Python environment... + For more information, type 'help(pylab)'. + +And once in pylab mode with the inline backend, +you can make plots and they will be displayed in your frontend +if it suports the inline figures (e.g. notebook or qtconsole): + +.. sourcecode:: ipython + + In [40]: %px plot(rand(100)) + Parallel execution on engine(s): all + + + + + Out[0:79]: [] + Out[1:79]: [] + Out[2:79]: [] + Out[3:79]: [] + + +%%px Cell Magic +--------------- + +%%px can be used as a Cell Magic, which accepts some arguments for controlling +the execution. + + +Targets and Blocking +******************** + +%%px accepts ``--targets`` for controlling which engines on which to run, +and ``--[no]block`` for specifying the blocking behavior of this cell, +independent of the defaults for the View. + +.. sourcecode:: ipython + + In [6]: %%px --targets ::2 + ...: print "I am even" + ...: + Parallel execution on engine(s): [0, 2] + [stdout:0] I am even + [stdout:2] I am even + + In [7]: %%px --targets 1 + ...: print "I am number 1" + ...: + Parallel execution on engine(s): 1 + I am number 1 + + In [8]: %%px + ...: print "still 'all' by default" + ...: + Parallel execution on engine(s): all + [stdout:0] still 'all' by default + [stdout:1] still 'all' by default + [stdout:2] still 'all' by default + [stdout:3] still 'all' by default + + In [9]: %%px --noblock + ...: import time + ...: time.sleep(1) + ...: time.time() + ...: + Async parallel execution on engine(s): all + Out[9]: + + In [10]: %pxresult + Out[0:12]: 1339454561.069116 + Out[1:10]: 1339454561.076752 + Out[2:12]: 1339454561.072837 + Out[3:10]: 1339454561.066665 + + +.. seealso:: + + :ref:`%pxconfig` accepts these same arguments for changing the *default* + values of targets/blocking for the active View. + + +Output Display +************** + + +%%px also accepts a ``--group-outputs`` argument, +which adjusts how the outputs of multiple engines are presented. + +.. seealso:: + + :meth:`.AsyncResult.display_outputs` for the grouping options. + +.. sourcecode:: ipython + + In [50]: %%px --block --group-outputs=engine + ....: import numpy as np + ....: A = np.random.random((2,2)) + ....: ev = numpy.linalg.eigvals(A) + ....: print ev + ....: ev.max() + ....: + Parallel execution on engine(s): all + [stdout:0] [ 0.60640442 0.95919621] + Out [0:73]: 0.9591962130899806 + [stdout:1] [ 0.38501813 1.29430871] + Out [1:73]: 1.2943087091452372 + [stdout:2] [-0.85925141 0.9387692 ] + Out [2:73]: 0.93876920456230284 + [stdout:3] [ 0.37998269 1.24218246] + Out [3:73]: 1.2421824618493817 + + +%pxresult +--------- + +If you are using %px in non-blocking mode, you won't get output. +You can use %pxresult to display the outputs of the latest command, +just as is done when %px is blocking: + +.. sourcecode:: ipython + + In [39]: dv.block = False + + In [40]: %px print 'hi' + Async parallel execution on engine(s): all + + In [41]: %pxresult + [stdout:0] hi + [stdout:1] hi + [stdout:2] hi + [stdout:3] hi + +%pxresult simply calls :meth:`.AsyncResult.display_outputs` on the most recent request. +It accepts the same output-grouping arguments as %%px, so you can use it to view +a result in different ways. + + +%autopx +------- + +The %autopx magic switches to a mode where everything you type is executed +on the engines until you do %autopx again. + +.. sourcecode:: ipython + + In [30]: dv.block=True + + In [31]: %autopx + %autopx enabled + + In [32]: max_evals = [] + + In [33]: for i in range(100): + ....: a = numpy.random.rand(10,10) + ....: a = a+a.transpose() + ....: evals = numpy.linalg.eigvals(a) + ....: max_evals.append(evals[0].real) + ....: + + In [34]: print "Average max eigenvalue is: %f" % (sum(max_evals)/len(max_evals)) + [stdout:0] Average max eigenvalue is: 10.193101 + [stdout:1] Average max eigenvalue is: 10.064508 + [stdout:2] Average max eigenvalue is: 10.055724 + [stdout:3] Average max eigenvalue is: 10.086876 + + In [35]: %autopx + Auto Parallel Disabled + +%pxconfig +--------- + +The default targets and blocking behavior for the magics are governed by the :attr:`block` +and :attr:`targets` attribute of the active View. If you have a handle for the view, +you can set these attributes directly, but if you don't, you can change them with +the %pxconfig magic: + +.. sourcecode:: ipython + + In [3]: %pxconfig --block + + In [5]: %px print 'hi' + Parallel execution on engine(s): all + [stdout:0] hi + [stdout:1] hi + [stdout:2] hi + [stdout:3] hi + + In [6]: %pxconfig --targets ::2 + + In [7]: %px print 'hi' + Parallel execution on engine(s): [0, 2] + [stdout:0] hi + [stdout:2] hi + + In [8]: %pxconfig --noblock + + In [9]: %px print 'are you there?' + Async parallel execution on engine(s): [0, 2] + Out[9]: + + In [10]: %pxresult + [stdout:0] are you there? + [stdout:2] are you there? + + +Multiple Active Views +===================== + +The parallel magics are associated with a particular :class:`~.DirectView` object. +You can change the active view by calling the :meth:`~.DirectView.activate` method +on any view. + +.. sourcecode:: ipython + + In [11]: even = rc[::2] + + In [12]: even.activate() + + In [13]: %px print 'hi' + Async parallel execution on engine(s): [0, 2] + Out[13]: + + In [14]: even.block = True + + In [15]: %px print 'hi' + Parallel execution on engine(s): [0, 2] + [stdout:0] hi + [stdout:2] hi + +When activating a View, you can also specify a *suffix*, so that a whole different +set of magics are associated with that view, without replacing the existing ones. + +.. sourcecode:: ipython + + # restore the original DirecView to the base %px magics + In [16]: rc.activate() + Out[16]: + + In [17]: even.activate('_even') + + In [18]: %px print 'hi all' + Parallel execution on engine(s): all + [stdout:0] hi all + [stdout:1] hi all + [stdout:2] hi all + [stdout:3] hi all + + In [19]: %px_even print "We aren't odd!" + Parallel execution on engine(s): [0, 2] + [stdout:0] We aren't odd! + [stdout:2] We aren't odd! + +This suffix is applied to the end of all magics, e.g. %autopx_even, %pxresult_even, etc. + +For convenience, the :class:`~.Client` has a :meth:`~.Client.activate` method as well, +which creates a DirectView with block=True, activates it, and returns the new View. + +The initial magics registered when you create a client are the result of a call to +:meth:`rc.activate` with default args. + + +Engines as Kernels +================== + +Engines are really the same object as the Kernels used elsewhere in IPython, +with the minor exception that engines connect to a controller, while regular kernels +bind their sockets, listening for connections from a QtConsole or other frontends. + +Sometimes for debugging or inspection purposes, you would like a QtConsole connected +to an engine for more direct interaction. You can do this by first instructing +the Engine to *also* bind its kernel, to listen for connections: + +.. sourcecode:: ipython + + In [50]: %px from IPython.parallel import bind_kernel; bind_kernel() + +Then, if your engines are local, you can start a qtconsole right on the engine(s): + +.. sourcecode:: ipython + + In [51]: %px %qtconsole + +Careful with this one, because if your view is of 16 engines it will start 16 QtConsoles! + +Or you can view just the connection info, and work out the right way to connect to the engines, +depending on where they live and where you are: + +.. sourcecode:: ipython + + In [51]: %px %connect_info + Parallel execution on engine(s): all + [stdout:0] + { + "stdin_port": 60387, + "ip": "127.0.0.1", + "hb_port": 50835, + "key": "eee2dd69-7dd3-4340-bf3e-7e2e22a62542", + "shell_port": 55328, + "iopub_port": 58264 + } + + Paste the above JSON into a file, and connect with: + $> ipython --existing + or, if you are local, you can connect with just: + $> ipython --existing kernel-60125.json + or even just: + $> ipython --existing + if this is the most recent IPython session you have started. + [stdout:1] + { + "stdin_port": 61869, + ... + +.. note:: + + ``%qtconsole`` will call :func:`bind_kernel` on an engine if it hasn't been done already, + so you can often skip that first step. + + diff --git a/docs/source/parallel/parallel_details.txt b/docs/source/parallel/parallel_details.txt index cf31219..a7b2553 100644 --- a/docs/source/parallel/parallel_details.txt +++ b/docs/source/parallel/parallel_details.txt @@ -34,19 +34,13 @@ The following will fail: In [3]: A = numpy.zeros(2) In [4]: def setter(a): - ...: a[0]=1 - ...: return a + ...: a[0]=1 + ...: return a In [5]: rc[0].apply_sync(setter, A) --------------------------------------------------------------------------- - RemoteError Traceback (most recent call last) - ... - RemoteError: RuntimeError(array is not writeable) - Traceback (most recent call last): - File "/path/to/site-packages/IPython/parallel/streamkernel.py", line 329, in apply_request - exec code in working, working - File "", line 1, in - File "", line 2, in setter + RuntimeError Traceback (most recent call last) in () + in setter(a) RuntimeError: array is not writeable If you do need to edit the array in-place, just remember to copy the array if it's read-only. diff --git a/docs/source/parallel/parallel_multiengine.txt b/docs/source/parallel/parallel_multiengine.txt index f8082a7..ef25fe1 100644 --- a/docs/source/parallel/parallel_multiengine.txt +++ b/docs/source/parallel/parallel_multiengine.txt @@ -387,229 +387,9 @@ The following examples demonstrate how to use the instance attributes: The :attr:`block` and :attr:`targets` instance attributes of the :class:`.DirectView` also determine the behavior of the parallel magic commands. -Parallel magic commands ------------------------ - -We provide a few IPython magic commands (``%px``, ``%autopx`` and ``%result``) -that make it a bit more pleasant to execute Python commands on the engines interactively. -These are simply shortcuts to :meth:`.DirectView.execute` -and :meth:`.AsyncResult.display_outputs` methods repsectively. -The ``%px`` magic executes a single Python command on the engines -specified by the :attr:`targets` attribute of the :class:`DirectView` instance: - -.. sourcecode:: ipython - - # Create a DirectView for all targets - In [22]: dv = rc[:] - - # Make this DirectView active for parallel magic commands - In [23]: dv.activate() - - In [24]: dv.block=True - - # import numpy here and everywhere - In [25]: with dv.sync_imports(): - ....: import numpy - importing numpy on engine(s) - - In [27]: %px a = numpy.random.rand(2,2) - Parallel execution on engines: [0, 1, 2, 3] - - In [28]: %px numpy.linalg.eigvals(a) - Parallel execution on engines: [0, 1, 2, 3] - [0] Out[68]: array([ 0.77120707, -0.19448286]) - [1] Out[68]: array([ 1.10815921, 0.05110369]) - [2] Out[68]: array([ 0.74625527, -0.37475081]) - [3] Out[68]: array([ 0.72931905, 0.07159743]) - - In [29]: %px print 'hi' - Parallel execution on engine(s): [0, 1, 2, 3] - [stdout:0] hi - [stdout:1] hi - [stdout:2] hi - [stdout:3] hi - - -Since engines are IPython as well, you can even run magics remotely: - -.. sourcecode:: ipython - - In [28]: %px %pylab inline - Parallel execution on engine(s): [0, 1, 2, 3] - [stdout:0] - Welcome to pylab, a matplotlib-based Python environment... - For more information, type 'help(pylab)'. - [stdout:1] - Welcome to pylab, a matplotlib-based Python environment... - For more information, type 'help(pylab)'. - [stdout:2] - Welcome to pylab, a matplotlib-based Python environment... - For more information, type 'help(pylab)'. - [stdout:3] - Welcome to pylab, a matplotlib-based Python environment... - For more information, type 'help(pylab)'. - -And once in pylab mode with the inline backend, -you can make plots and they will be displayed in your frontend -if it suports the inline figures (e.g. notebook or qtconsole): - -.. sourcecode:: ipython - - In [40]: %px plot(rand(100)) - Parallel execution on engine(s): [0, 1, 2, 3] - - - - - [0] Out[79]: [] - [1] Out[79]: [] - [2] Out[79]: [] - [3] Out[79]: [] - - -``%%px`` Cell Magic -******************* - -`%%px` can also be used as a Cell Magic, which accepts ``--[no]block`` flags, -and a ``--group-outputs`` argument, which adjust how the outputs of multiple -engines are presented. - .. seealso:: - :meth:`.AsyncResult.display_outputs` for the grouping options. - -.. sourcecode:: ipython - - In [50]: %%px --block --group-outputs=engine - ....: import numpy as np - ....: A = np.random.random((2,2)) - ....: ev = numpy.linalg.eigvals(A) - ....: print ev - ....: ev.max() - ....: - Parallel execution on engine(s): [0, 1, 2, 3] - [stdout:0] [ 0.60640442 0.95919621] - [0] Out[73]: 0.9591962130899806 - [stdout:1] [ 0.38501813 1.29430871] - [1] Out[73]: 1.2943087091452372 - [stdout:2] [-0.85925141 0.9387692 ] - [2] Out[73]: 0.93876920456230284 - [stdout:3] [ 0.37998269 1.24218246] - [3] Out[73]: 1.2421824618493817 - -``%result`` Magic -***************** - -If you are using ``%px`` in non-blocking mode, you won't get output. -You can use ``%result`` to display the outputs of the latest command, -just as is done when ``%px`` is blocking: - -.. sourcecode:: ipython - - In [39]: dv.block = False - - In [40]: %px print 'hi' - Async parallel execution on engine(s): [0, 1, 2, 3] - - In [41]: %result - [stdout:0] hi - [stdout:1] hi - [stdout:2] hi - [stdout:3] hi - -``%result`` simply calls :meth:`.AsyncResult.display_outputs` on the most recent request. -You can pass integers as indices if you want a result other than the latest, -e.g. ``%result -2``, or ``%result 0`` for the first. - - -``%autopx`` -*********** - -The ``%autopx`` magic switches to a mode where everything you type is executed -on the engines until you do ``%autopx`` again. - -.. sourcecode:: ipython - - In [30]: dv.block=True - - In [31]: %autopx - %autopx enabled - - In [32]: max_evals = [] - - In [33]: for i in range(100): - ....: a = numpy.random.rand(10,10) - ....: a = a+a.transpose() - ....: evals = numpy.linalg.eigvals(a) - ....: max_evals.append(evals[0].real) - ....: - - In [34]: print "Average max eigenvalue is: %f" % (sum(max_evals)/len(max_evals)) - [stdout:0] Average max eigenvalue is: 10.193101 - [stdout:1] Average max eigenvalue is: 10.064508 - [stdout:2] Average max eigenvalue is: 10.055724 - [stdout:3] Average max eigenvalue is: 10.086876 - - In [35]: %autopx - Auto Parallel Disabled - - -Engines as Kernels -****************** - -Engines are really the same object as the Kernels used elsewhere in IPython, -with the minor exception that engines connect to a controller, while regular kernels -bind their sockets, listening for connections from a QtConsole or other frontends. - -Sometimes for debugging or inspection purposes, you would like a QtConsole connected -to an engine for more direct interaction. You can do this by first instructing -the Engine to *also* bind its kernel, to listen for connections: - -.. sourcecode:: ipython - - In [50]: %px from IPython.parallel import bind_kernel; bind_kernel() - -Then, if your engines are local, you can start a qtconsole right on the engine(s): - -.. sourcecode:: ipython - - In [51]: %px %qtconsole - -Careful with this one, because if your view is of 16 engines it will start 16 QtConsoles! - -Or you can view just the connection info, and work out the right way to connect to the engines, -depending on where they live and where you are: - -.. sourcecode:: ipython - - In [51]: %px %connect_info - Parallel execution on engine(s): [0, 1, 2, 3] - [stdout:0] - { - "stdin_port": 60387, - "ip": "127.0.0.1", - "hb_port": 50835, - "key": "eee2dd69-7dd3-4340-bf3e-7e2e22a62542", - "shell_port": 55328, - "iopub_port": 58264 - } - - Paste the above JSON into a file, and connect with: - $> ipython --existing - or, if you are local, you can connect with just: - $> ipython --existing kernel-60125.json - or even just: - $> ipython --existing - if this is the most recent IPython session you have started. - [stdout:1] - { - "stdin_port": 61869, - ... - -.. note:: - - ``%qtconsole`` will call :func:`bind_kernel` on an engine if it hasn't been done already, - so you can often skip that first step. + See the documentation of the :ref:`Parallel Magics `. Moving Python objects around @@ -753,13 +533,20 @@ be collected and raise a CompositeError, as demonstrated in the next section. ....: time.sleep(t) ....: return t +.. note:: + + :func:`sync_imports` does not allow ``import foo as bar`` syntax, + because the assignment represented by the ``as bar`` part is not + available to the import hook. + + .. _parallel_exceptions: Parallel exceptions ------------------- In the multiengine interface, parallel commands can raise Python exceptions, -just like serial commands. But, it is a little subtle, because a single +just like serial commands. But it is a little subtle, because a single parallel command can actually raise multiple exceptions (one for each engine the command was run on). To express this idea, we have a :exc:`CompositeError` exception class that will be raised in most cases. The @@ -768,58 +555,32 @@ more other types of exceptions. Here is how it works: .. sourcecode:: ipython - In [76]: dview.block=True + In [78]: dview.block = True + + In [79]: dview.execute("1/0") + [0:execute]: + --------------------------------------------------------------------------- + ZeroDivisionError Traceback (most recent call last) in () + ----> 1 1/0 + ZeroDivisionError: integer division or modulo by zero - In [77]: dview.execute('1/0') + [1:execute]: --------------------------------------------------------------------------- - CompositeError Traceback (most recent call last) - /home/user/ in () - ----> 1 dview.execute('1/0') - - /path/to/site-packages/IPython/parallel/client/view.pyc in execute(self, code, targets, block) - 591 default: self.block - 592 """ - --> 593 return self._really_apply(util._execute, args=(code,), block=block, targets=targets) - 594 - 595 def run(self, filename, targets=None, block=None): - - /home/user/ in _really_apply(self, f, args, kwargs, targets, block, track) - - /path/to/site-packages/IPython/parallel/client/view.pyc in sync_results(f, self, *args, **kwargs) - 55 def sync_results(f, self, *args, **kwargs): - 56 """sync relevant results from self.client to our results attribute.""" - ---> 57 ret = f(self, *args, **kwargs) - 58 delta = self.outstanding.difference(self.client.outstanding) - 59 completed = self.outstanding.intersection(delta) - - /home/user/ in _really_apply(self, f, args, kwargs, targets, block, track) - - /path/to/site-packages/IPython/parallel/client/view.pyc in save_ids(f, self, *args, **kwargs) - 44 n_previous = len(self.client.history) - 45 try: - ---> 46 ret = f(self, *args, **kwargs) - 47 finally: - 48 nmsgs = len(self.client.history) - n_previous - - /path/to/site-packages/IPython/parallel/client/view.pyc in _really_apply(self, f, args, kwargs, targets, block, track) - 529 if block: - 530 try: - --> 531 return ar.get() - 532 except KeyboardInterrupt: - 533 pass - - /path/to/site-packages/IPython/parallel/client/asyncresult.pyc in get(self, timeout) - 101 return self._result - 102 else: - --> 103 raise self._exception - 104 else: - 105 raise error.TimeoutError("Result not ready.") - - CompositeError: one or more exceptions from call to method: _execute - [0:apply]: ZeroDivisionError: integer division or modulo by zero - [1:apply]: ZeroDivisionError: integer division or modulo by zero - [2:apply]: ZeroDivisionError: integer division or modulo by zero - [3:apply]: ZeroDivisionError: integer division or modulo by zero + ZeroDivisionError Traceback (most recent call last) in () + ----> 1 1/0 + ZeroDivisionError: integer division or modulo by zero + + [2:execute]: + --------------------------------------------------------------------------- + ZeroDivisionError Traceback (most recent call last) in () + ----> 1 1/0 + ZeroDivisionError: integer division or modulo by zero + + [3:execute]: + --------------------------------------------------------------------------- + ZeroDivisionError Traceback (most recent call last) in () + ----> 1 1/0 + ZeroDivisionError: integer division or modulo by zero Notice how the error message printed when :exc:`CompositeError` is raised has information about the individual exceptions that were raised on each engine. @@ -828,33 +589,14 @@ If you want, you can even raise one of these original exceptions: .. sourcecode:: ipython In [80]: try: - ....: dview.execute('1/0') + ....: dview.execute('1/0', block=True) ....: except parallel.error.CompositeError, e: ....: e.raise_exception() - ....: - ....: + ....: + ....: --------------------------------------------------------------------------- - RemoteError Traceback (most recent call last) - /home/user/ in () - 2 dview.execute('1/0') - 3 except CompositeError as e: - ----> 4 e.raise_exception() - - /path/to/site-packages/IPython/parallel/error.pyc in raise_exception(self, excid) - 266 raise IndexError("an exception with index %i does not exist"%excid) - 267 else: - --> 268 raise RemoteError(en, ev, etb, ei) - 269 - 270 - - RemoteError: ZeroDivisionError(integer division or modulo by zero) - Traceback (most recent call last): - File "/path/to/site-packages/IPython/parallel/engine/streamkernel.py", line 330, in apply_request - exec code in working,working - File "", line 1, in - File "/path/to/site-packages/IPython/parallel/util.py", line 354, in _execute - exec code in globals() - File "", line 1, in + ZeroDivisionError Traceback (most recent call last) in () + ----> 1 1/0 ZeroDivisionError: integer division or modulo by zero If you are working in IPython, you can simple type ``%debug`` after one of @@ -864,112 +606,56 @@ instance: .. sourcecode:: ipython In [81]: dview.execute('1/0') + [0:execute]: --------------------------------------------------------------------------- - CompositeError Traceback (most recent call last) - /home/user/ in () - ----> 1 dview.execute('1/0') - - /path/to/site-packages/IPython/parallel/client/view.pyc in execute(self, code, targets, block) - 591 default: self.block - 592 """ - --> 593 return self._really_apply(util._execute, args=(code,), block=block, targets=targets) - 594 - 595 def run(self, filename, targets=None, block=None): - - /home/user/ in _really_apply(self, f, args, kwargs, targets, block, track) - - /path/to/site-packages/IPython/parallel/client/view.pyc in sync_results(f, self, *args, **kwargs) - 55 def sync_results(f, self, *args, **kwargs): - 56 """sync relevant results from self.client to our results attribute.""" - ---> 57 ret = f(self, *args, **kwargs) - 58 delta = self.outstanding.difference(self.client.outstanding) - 59 completed = self.outstanding.intersection(delta) - - /home/user/ in _really_apply(self, f, args, kwargs, targets, block, track) - - /path/to/site-packages/IPython/parallel/client/view.pyc in save_ids(f, self, *args, **kwargs) - 44 n_previous = len(self.client.history) - 45 try: - ---> 46 ret = f(self, *args, **kwargs) - 47 finally: - 48 nmsgs = len(self.client.history) - n_previous - - /path/to/site-packages/IPython/parallel/client/view.pyc in _really_apply(self, f, args, kwargs, targets, block, track) - 529 if block: - 530 try: - --> 531 return ar.get() - 532 except KeyboardInterrupt: - 533 pass - - /path/to/site-packages/IPython/parallel/client/asyncresult.pyc in get(self, timeout) - 101 return self._result - 102 else: - --> 103 raise self._exception - 104 else: - 105 raise error.TimeoutError("Result not ready.") - - CompositeError: one or more exceptions from call to method: _execute - [0:apply]: ZeroDivisionError: integer division or modulo by zero - [1:apply]: ZeroDivisionError: integer division or modulo by zero - [2:apply]: ZeroDivisionError: integer division or modulo by zero - [3:apply]: ZeroDivisionError: integer division or modulo by zero - - In [82]: %debug - > /path/to/site-packages/IPython/parallel/client/asyncresult.py(103)get() - 102 else: - --> 103 raise self._exception - 104 else: - - # With the debugger running, self._exception is the exceptions instance. We can tab complete - # on it and see the extra methods that are available. - ipdb> self._exception. - e.__class__ e.__getitem__ e.__new__ e.__setstate__ e.args - e.__delattr__ e.__getslice__ e.__reduce__ e.__str__ e.elist - e.__dict__ e.__hash__ e.__reduce_ex__ e.__weakref__ e.message - e.__doc__ e.__init__ e.__repr__ e._get_engine_str e.print_tracebacks - e.__getattribute__ e.__module__ e.__setattr__ e._get_traceback e.raise_exception - ipdb> self._exception.print_tracebacks() - [0:apply]: - Traceback (most recent call last): - File "/path/to/site-packages/IPython/parallel/engine/streamkernel.py", line 330, in apply_request - exec code in working,working - File "", line 1, in - File "/path/to/site-packages/IPython/parallel/util.py", line 354, in _execute - exec code in globals() - File "", line 1, in + ZeroDivisionError Traceback (most recent call last) in () + ----> 1 1/0 ZeroDivisionError: integer division or modulo by zero - - [1:apply]: - Traceback (most recent call last): - File "/path/to/site-packages/IPython/parallel/engine/streamkernel.py", line 330, in apply_request - exec code in working,working - File "", line 1, in - File "/path/to/site-packages/IPython/parallel/util.py", line 354, in _execute - exec code in globals() - File "", line 1, in + [1:execute]: + --------------------------------------------------------------------------- + ZeroDivisionError Traceback (most recent call last) in () + ----> 1 1/0 ZeroDivisionError: integer division or modulo by zero - - [2:apply]: - Traceback (most recent call last): - File "/path/to/site-packages/IPython/parallel/engine/streamkernel.py", line 330, in apply_request - exec code in working,working - File "", line 1, in - File "/path/to/site-packages/IPython/parallel/util.py", line 354, in _execute - exec code in globals() - File "", line 1, in + [2:execute]: + --------------------------------------------------------------------------- + ZeroDivisionError Traceback (most recent call last) in () + ----> 1 1/0 ZeroDivisionError: integer division or modulo by zero - - [3:apply]: - Traceback (most recent call last): - File "/path/to/site-packages/IPython/parallel/engine/streamkernel.py", line 330, in apply_request - exec code in working,working - File "", line 1, in - File "/path/to/site-packages/IPython/parallel/util.py", line 354, in _execute - exec code in globals() - File "", line 1, in + [3:execute]: + --------------------------------------------------------------------------- + ZeroDivisionError Traceback (most recent call last) in () + ----> 1 1/0 + ZeroDivisionError: integer division or modulo by zero + + In [82]: %debug + > /.../site-packages/IPython/parallel/client/asyncresult.py(125)get() + 124 else: + --> 125 raise self._exception + 126 else: + + # Here, self._exception is the CompositeError instance: + + ipdb> e = self._exception + ipdb> e + CompositeError(4) + + # we can tab-complete on e to see available methods: + ipdb> e. + e.args e.message e.traceback + e.elist e.msg + e.ename e.print_traceback + e.engine_info e.raise_exception + e.evalue e.render_traceback + + # We can then display the individual tracebacks, if we want: + ipdb> e.print_traceback(1) + [1:execute]: + --------------------------------------------------------------------------- + ZeroDivisionError Traceback (most recent call last) in () + ----> 1 1/0 ZeroDivisionError: integer division or modulo by zero @@ -982,21 +668,27 @@ All of this same error handling magic even works in non-blocking mode: In [84]: ar = dview.execute('1/0') In [85]: ar.get() + [0:execute]: --------------------------------------------------------------------------- - CompositeError Traceback (most recent call last) - /home/user/ in () - ----> 1 ar.get() - - /path/to/site-packages/IPython/parallel/client/asyncresult.pyc in get(self, timeout) - 101 return self._result - 102 else: - --> 103 raise self._exception - 104 else: - 105 raise error.TimeoutError("Result not ready.") - - CompositeError: one or more exceptions from call to method: _execute - [0:apply]: ZeroDivisionError: integer division or modulo by zero - [1:apply]: ZeroDivisionError: integer division or modulo by zero - [2:apply]: ZeroDivisionError: integer division or modulo by zero - [3:apply]: ZeroDivisionError: integer division or modulo by zero + ZeroDivisionError Traceback (most recent call last) in () + ----> 1 1/0 + ZeroDivisionError: integer division or modulo by zero + + [1:execute]: + --------------------------------------------------------------------------- + ZeroDivisionError Traceback (most recent call last) in () + ----> 1 1/0 + ZeroDivisionError: integer division or modulo by zero + + [2:execute]: + --------------------------------------------------------------------------- + ZeroDivisionError Traceback (most recent call last) in () + ----> 1 1/0 + ZeroDivisionError: integer division or modulo by zero + + [3:execute]: + --------------------------------------------------------------------------- + ZeroDivisionError Traceback (most recent call last) in () + ----> 1 1/0 + ZeroDivisionError: integer division or modulo by zero