Merge pull request #7005 from minrk/payloads-3.0

update payloads for msg spec 5

IPython/core/payloadpage.py

@@ -1,49 +1,48 b''
 # encoding: utf-8
 """A payload based version of page."""
 
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
 from IPython.core.getipython import get_ipython
 
 #-----------------------------------------------------------------------------
 # Classes and functions
 #-----------------------------------------------------------------------------
 
 def page(strng, start=0, screen_lines=0, pager_cmd=None):
     """Print a string, piping through a pager.
 
     This version ignores the screen_lines and pager_cmd arguments and uses
     IPython's payload system instead.
 
     Parameters
     ----------
     strng : str or mime-dict
       Text to page, or a mime-type keyed dict of already formatted data.
 
     start : int
       Starting line at which to place the display.
     """
 
     # Some routines may auto-compute start offsets incorrectly and pass a
     # negative value.  Offset to 0 for robustness.
     start = max(0, start)
     shell = get_ipython()
     
     if isinstance(strng, dict):
         data = strng
     else:
         data = {'text/plain' : strng}
     payload = dict(
         source='page',
         data=data,
         start=start,
-        screen_lines=screen_lines,
         )
     shell.payload_manager.write_payload(payload)
 
 
 def install_payload_page():
     """Install this version of page as IPython.core.page.page."""
     from IPython.core import page as corepage
     corepage.page = page

IPython/kernel/zmq/zmqshell.py

@@ -1,567 +1,487 b''
 """A ZMQ-based subclass of InteractiveShell.
 
 This code is meant to ease the refactoring of the base InteractiveShell into
 something with a cleaner architecture for 2-process use, without actually
 breaking InteractiveShell itself.  So we're doing something a bit ugly, where
 we subclass and override what we want to fix.  Once this is working well, we
 can go back to the base class and refactor the code for a cleaner inheritance
 implementation that doesn't rely on so much monkeypatching.
 
 But this lets us maintain a fully working IPython as we develop the new
 machinery.  This should thus be thought of as scaffolding.
 """
 
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
 from __future__ import print_function
 
 import os
 import sys
 import time
 
 from zmq.eventloop import ioloop
 
 from IPython.core.interactiveshell import (
     InteractiveShell, InteractiveShellABC
 )
 from IPython.core import page
 from IPython.core.autocall import ZMQExitAutocall
 from IPython.core.displaypub import DisplayPublisher
 from IPython.core.error import UsageError
 from IPython.core.magics import MacroToEdit, CodeMagics
 from IPython.core.magic import magics_class, line_magic, Magics
 from IPython.core.payloadpage import install_payload_page
 from IPython.core.usage import default_gui_banner
 from IPython.display import display, Javascript
 from IPython.kernel.inprocess.socket import SocketABC
 from IPython.kernel import (
     get_connection_file, get_connection_info, connect_qtconsole
 )
 from IPython.testing.skipdoctest import skip_doctest
 from IPython.utils import openpy
 from IPython.utils.jsonutil import json_clean, encode_images
 from IPython.utils.process import arg_split
 from IPython.utils import py3compat
 from IPython.utils.py3compat import unicode_type
 from IPython.utils.traitlets import Instance, Type, Dict, CBool, CBytes, Any
 from IPython.utils.warn import error
 from IPython.kernel.zmq.displayhook import ZMQShellDisplayHook
 from IPython.kernel.zmq.datapub import ZMQDataPublisher
 from IPython.kernel.zmq.session import extract_header
 from .session import Session
 
 #-----------------------------------------------------------------------------
 # Functions and classes
 #-----------------------------------------------------------------------------
 
 class ZMQDisplayPublisher(DisplayPublisher):
     """A display publisher that publishes data using a ZeroMQ PUB socket."""
 
     session = Instance(Session)
     pub_socket = Instance(SocketABC)
     parent_header = Dict({})
     topic = CBytes(b'display_data')
 
     def set_parent(self, parent):
         """Set the parent for outbound messages."""
         self.parent_header = extract_header(parent)
     
     def _flush_streams(self):
         """flush IO Streams prior to display"""
         sys.stdout.flush()
         sys.stderr.flush()
 
     def publish(self, data, metadata=None, source=None):
         self._flush_streams()
         if metadata is None:
             metadata = {}
         self._validate_data(data, metadata)
         content = {}
         content['data'] = encode_images(data)
         content['metadata'] = metadata
         self.session.send(
             self.pub_socket, u'display_data', json_clean(content),
             parent=self.parent_header, ident=self.topic,
         )
 
     def clear_output(self, wait=False):
         content = dict(wait=wait)
         self._flush_streams()
         self.session.send(
             self.pub_socket, u'clear_output', content,
             parent=self.parent_header, ident=self.topic,
         )
 
 @magics_class
 class KernelMagics(Magics):
     #------------------------------------------------------------------------
     # Magic overrides
     #------------------------------------------------------------------------
     # Once the base class stops inheriting from magic, this code needs to be
     # moved into a separate machinery as well.  For now, at least isolate here
     # the magics which this class needs to implement differently from the base
     # class, or that are unique to it.
-
-    @line_magic
-    def doctest_mode(self, parameter_s=''):
-        """Toggle doctest mode on and off.
-
-        This mode is intended to make IPython behave as much as possible like a
-        plain Python shell, from the perspective of how its prompts, exceptions
-        and output look.  This makes it easy to copy and paste parts of a
-        session into doctests.  It does so by:
-
-        - Changing the prompts to the classic ``>>>`` ones.
-        - Changing the exception reporting mode to 'Plain'.
-        - Disabling pretty-printing of output.
-
-        Note that IPython also supports the pasting of code snippets that have
-        leading '>>>' and '...' prompts in them.  This means that you can paste
-        doctests from files or docstrings (even if they have leading
-        whitespace), and the code will execute correctly.  You can then use
-        '%history -t' to see the translated history; this will give you the
-        input after removal of all the leading prompts and whitespace, which
-        can be pasted back into an editor.
-
-        With these features, you can switch into this mode easily whenever you
-        need to do testing and changes to doctests, without having to leave
-        your existing IPython session.
-        """
-
-        from IPython.utils.ipstruct import Struct
-
-        # Shorthands
-        shell = self.shell
-        disp_formatter = self.shell.display_formatter
-        ptformatter = disp_formatter.formatters['text/plain']
-        # dstore is a data store kept in the instance metadata bag to track any
-        # changes we make, so we can undo them later.
-        dstore = shell.meta.setdefault('doctest_mode', Struct())
-        save_dstore = dstore.setdefault
-
-        # save a few values we'll need to recover later
-        mode = save_dstore('mode', False)
-        save_dstore('rc_pprint', ptformatter.pprint)
-        save_dstore('rc_active_types',disp_formatter.active_types)
-        save_dstore('xmode', shell.InteractiveTB.mode)
-
-        if mode == False:
-            # turn on
-            ptformatter.pprint = False
-            disp_formatter.active_types = ['text/plain']
-            shell.magic('xmode Plain')
-        else:
-            # turn off
-            ptformatter.pprint = dstore.rc_pprint
-            disp_formatter.active_types = dstore.rc_active_types
-            shell.magic("xmode " + dstore.xmode)
-
-        # Store new mode and inform on console
-        dstore.mode = bool(1-int(mode))
-        mode_label = ['OFF','ON'][dstore.mode]
-        print('Doctest mode is:', mode_label)
-
-        # Send the payload back so that clients can modify their prompt display
-        payload = dict(
-            source='doctest_mode',
-            mode=dstore.mode)
-        shell.payload_manager.write_payload(payload)
-        
     
     _find_edit_target = CodeMagics._find_edit_target
 
     @skip_doctest
     @line_magic
     def edit(self, parameter_s='', last_call=['','']):
         """Bring up an editor and execute the resulting code.
 
         Usage:
           %edit [options] [args]
 
         %edit runs an external text editor. You will need to set the command for
         this editor via the ``TerminalInteractiveShell.editor`` option in your
         configuration file before it will work.
 
         This command allows you to conveniently edit multi-line code right in
         your IPython session.
 
         If called without arguments, %edit opens up an empty editor with a
         temporary file and will execute the contents of this file when you
         close it (don't forget to save it!).
 
         Options:
 
         -n <number>
           Open the editor at a specified line number. By default, the IPython
           editor hook uses the unix syntax 'editor +N filename', but you can
           configure this by providing your own modified hook if your favorite
           editor supports line-number specifications with a different syntax.
 
         -p
           Call the editor with the same data as the previous time it was used,
           regardless of how long ago (in your current session) it was.
 
         -r
           Use 'raw' input. This option only applies to input taken from the
           user's history.  By default, the 'processed' history is used, so that
           magics are loaded in their transformed version to valid Python.  If
           this option is given, the raw input as typed as the command line is
           used instead.  When you exit the editor, it will be executed by
           IPython's own processor.
 
         Arguments:
 
         If arguments are given, the following possibilites exist:
 
         - The arguments are numbers or pairs of colon-separated numbers (like
           1 4:8 9). These are interpreted as lines of previous input to be
           loaded into the editor. The syntax is the same of the %macro command.
 
         - If the argument doesn't start with a number, it is evaluated as a
           variable and its contents loaded into the editor. You can thus edit
           any string which contains python code (including the result of
           previous edits).
 
         - If the argument is the name of an object (other than a string),
           IPython will try to locate the file where it was defined and open the
           editor at the point where it is defined. You can use ``%edit function``
           to load an editor exactly at the point where 'function' is defined,
           edit it and have the file be executed automatically.
 
           If the object is a macro (see %macro for details), this opens up your
           specified editor with a temporary file containing the macro's data.
           Upon exit, the macro is reloaded with the contents of the file.
 
           Note: opening at an exact line is only supported under Unix, and some
           editors (like kedit and gedit up to Gnome 2.8) do not understand the
           '+NUMBER' parameter necessary for this feature. Good editors like
           (X)Emacs, vi, jed, pico and joe all do.
 
         - If the argument is not found as a variable, IPython will look for a
           file with that name (adding .py if necessary) and load it into the
           editor. It will execute its contents with execfile() when you exit,
           loading any code in the file into your interactive namespace.
 
         Unlike in the terminal, this is designed to use a GUI editor, and we do
         not know when it has closed. So the file you edit will not be
         automatically executed or printed.
 
         Note that %edit is also available through the alias %ed.
         """
 
         opts,args = self.parse_options(parameter_s,'prn:')
 
         try:
             filename, lineno, _ = CodeMagics._find_edit_target(self.shell, args, opts, last_call)
         except MacroToEdit as e:
             # TODO: Implement macro editing over 2 processes.
             print("Macro editing not yet implemented in 2-process model.")
             return
 
         # Make sure we send to the client an absolute path, in case the working
         # directory of client and kernel don't match
         filename = os.path.abspath(filename)
 
         payload = {
             'source' : 'edit_magic',
             'filename' : filename,
             'line_number' : lineno
         }
         self.shell.payload_manager.write_payload(payload)
 
     # A few magics that are adapted to the specifics of using pexpect and a
     # remote terminal
 
     @line_magic
     def clear(self, arg_s):
         """Clear the terminal."""
         if os.name == 'posix':
             self.shell.system("clear")
         else:
             self.shell.system("cls")
 
     if os.name == 'nt':
         # This is the usual name in windows
         cls = line_magic('cls')(clear)
 
     # Terminal pagers won't work over pexpect, but we do have our own pager
 
     @line_magic
     def less(self, arg_s):
         """Show a file through the pager.
 
         Files ending in .py are syntax-highlighted."""
         if not arg_s:
             raise UsageError('Missing filename.')
 
         cont = open(arg_s).read()
         if arg_s.endswith('.py'):
             cont = self.shell.pycolorize(openpy.read_py_file(arg_s, skip_encoding_cookie=False))
         else:
             cont = open(arg_s).read()
         page.page(cont)
 
     more = line_magic('more')(less)
 
     # Man calls a pager, so we also need to redefine it
     if os.name == 'posix':
         @line_magic
         def man(self, arg_s):
             """Find the man page for the given command and display in pager."""
             page.page(self.shell.getoutput('man %s | col -b' % arg_s,
                                            split=False))
 
     @line_magic
     def connect_info(self, arg_s):
         """Print information for connecting other clients to this kernel
         
         It will print the contents of this session's connection file, as well as
         shortcuts for local clients.
         
         In the simplest case, when called from the most recently launched kernel,
         secondary clients can be connected, simply with:
         
         $> ipython <app> --existing
         
         """
         
         from IPython.core.application import BaseIPythonApplication as BaseIPApp
         
         if BaseIPApp.initialized():
             app = BaseIPApp.instance()
             security_dir = app.profile_dir.security_dir
             profile = app.profile
         else:
             profile = 'default'
             security_dir = ''
         
         try:
             connection_file = get_connection_file()
             info = get_connection_info(unpack=False)
         except Exception as e:
             error("Could not get connection info: %r" % e)
             return
         
         # add profile flag for non-default profile
         profile_flag = "--profile %s" % profile if profile != 'default' else ""
         
         # if it's in the security dir, truncate to basename
         if security_dir == os.path.dirname(connection_file):
             connection_file = os.path.basename(connection_file)
         
         
         print (info + '\n')
         print ("Paste the above JSON into a file, and connect with:\n"
             "    $> ipython <app> --existing <file>\n"
             "or, if you are local, you can connect with just:\n"
             "    $> ipython <app> --existing {0} {1}\n"
             "or even just:\n"
             "    $> ipython <app> --existing {1}\n"
             "if this is the most recent IPython session you have started.".format(
             connection_file, profile_flag
             )
         )
 
     @line_magic
     def qtconsole(self, arg_s):
         """Open a qtconsole connected to this kernel.
         
         Useful for connecting a qtconsole to running notebooks, for better
         debugging.
         """
         
         # %qtconsole should imply bind_kernel for engines:
         try:
             from IPython.parallel import bind_kernel
         except ImportError:
             # technically possible, because parallel has higher pyzmq min-version
             pass
         else:
             bind_kernel()
         
         try:
             p = connect_qtconsole(argv=arg_split(arg_s, os.name=='posix'))
         except Exception as e:
             error("Could not start qtconsole: %r" % e)
             return
     
     @line_magic
     def autosave(self, arg_s):
         """Set the autosave interval in the notebook (in seconds).
         
         The default value is 120, or two minutes.
         ``%autosave 0`` will disable autosave.
         
         This magic only has an effect when called from the notebook interface.
         It has no effect when called in a startup file.
         """
         
         try:
             interval = int(arg_s)
         except ValueError:
             raise UsageError("%%autosave requires an integer, got %r" % arg_s)
         
         # javascript wants milliseconds
         milliseconds = 1000 * interval
         display(Javascript("IPython.notebook.set_autosave_interval(%i)" % milliseconds),
             include=['application/javascript']
         )
         if interval:
             print("Autosaving every %i seconds" % interval)
         else:
             print("Autosave disabled")
 
 
 class ZMQInteractiveShell(InteractiveShell):
     """A subclass of InteractiveShell for ZMQ."""
 
     displayhook_class = Type(ZMQShellDisplayHook)
     display_pub_class = Type(ZMQDisplayPublisher)
     data_pub_class = Type(ZMQDataPublisher)
     kernel = Any()
     parent_header = Any()
     
     def _banner1_default(self):
         return default_gui_banner
 
     # Override the traitlet in the parent class, because there's no point using
     # readline for the kernel. Can be removed when the readline code is moved
     # to the terminal frontend.
     colors_force = CBool(True)
     readline_use = CBool(False)
     # autoindent has no meaning in a zmqshell, and attempting to enable it
     # will print a warning in the absence of readline.
     autoindent = CBool(False)
 
     exiter = Instance(ZMQExitAutocall)
     def _exiter_default(self):
         return ZMQExitAutocall(self)
     
     def _exit_now_changed(self, name, old, new):
         """stop eventloop when exit_now fires"""
         if new:
             loop = ioloop.IOLoop.instance()
             loop.add_timeout(time.time()+0.1, loop.stop)
 
     keepkernel_on_exit = None
 
     # Over ZeroMQ, GUI control isn't done with PyOS_InputHook as there is no
     # interactive input being read; we provide event loop support in ipkernel
     @staticmethod
     def enable_gui(gui):
         from .eventloops import enable_gui as real_enable_gui
         try:
             real_enable_gui(gui)
         except ValueError as e:
             raise UsageError("%s" % e)
 
     def init_environment(self):
         """Configure the user's environment.
 
         """
         env = os.environ
         # These two ensure 'ls' produces nice coloring on BSD-derived systems
         env['TERM'] = 'xterm-color'
         env['CLICOLOR'] = '1'
         # Since normal pagers don't work at all (over pexpect we don't have
         # single-key control of the subprocess), try to disable paging in
         # subprocesses as much as possible.
         env['PAGER'] = 'cat'
         env['GIT_PAGER'] = 'cat'
         
         # And install the payload version of page.
         install_payload_page()
 
-    def auto_rewrite_input(self, cmd):
-        """Called to show the auto-rewritten input for autocall and friends.
-
-        FIXME: this payload is currently not correctly processed by the
-        frontend.
-        """
-        new = self.prompt_manager.render('rewrite') + cmd
-        payload = dict(
-            source='auto_rewrite_input',
-            transformed_input=new,
-            )
-        self.payload_manager.write_payload(payload)
-
     def ask_exit(self):
         """Engage the exit actions."""
         self.exit_now = (not self.keepkernel_on_exit)
         payload = dict(
             source='ask_exit',
-            exit=True,
             keepkernel=self.keepkernel_on_exit,
             )
         self.payload_manager.write_payload(payload)
 
     def _showtraceback(self, etype, evalue, stb):
         # try to preserve ordering of tracebacks and print statements
         sys.stdout.flush()
         sys.stderr.flush()
 
         exc_content = {
             u'traceback' : stb,
             u'ename' : unicode_type(etype.__name__),
             u'evalue' : py3compat.safe_unicode(evalue),
         }
 
         dh = self.displayhook
         # Send exception info over pub socket for other clients than the caller
         # to pick up
         topic = None
         if dh.topic:
             topic = dh.topic.replace(b'execute_result', b'error')
         
         exc_msg = dh.session.send(dh.pub_socket, u'error', json_clean(exc_content), dh.parent_header, ident=topic)
 
         # FIXME - Hack: store exception info in shell object.  Right now, the
         # caller is reading this info after the fact, we need to fix this logic
         # to remove this hack.  Even uglier, we need to store the error status
         # here, because in the main loop, the logic that sets it is being
         # skipped because runlines swallows the exceptions.
         exc_content[u'status'] = u'error'
         self._reply_content = exc_content
         # /FIXME
 
         return exc_content
 
     def set_next_input(self, text):
         """Send the specified text to the frontend to be presented at the next
         input cell."""
         payload = dict(
             source='set_next_input',
             text=text
         )
         self.payload_manager.write_payload(payload)
     
     def set_parent(self, parent):
         """Set the parent header for associating output with its triggering input"""
         self.parent_header = parent
         self.displayhook.set_parent(parent)
         self.display_pub.set_parent(parent)
         self.data_pub.set_parent(parent)
         try:
             sys.stdout.set_parent(parent)
         except AttributeError:
             pass
         try:
             sys.stderr.set_parent(parent)
         except AttributeError:
             pass
     
     def get_parent(self):
         return self.parent_header
     
     #-------------------------------------------------------------------------
     # Things related to magics
     #-------------------------------------------------------------------------
 
     def init_magics(self):
         super(ZMQInteractiveShell, self).init_magics()
         self.register_magics(KernelMagics)
         self.magics_manager.register_alias('ed', 'edit')
 
 
 InteractiveShellABC.register(ZMQInteractiveShell)

IPython/qt/console/ipython_widget.py

@@ -1,599 +1,599 b''
 """A FrontendWidget that emulates the interface of the console IPython.
 
 This supports the additional functionality provided by the IPython kernel.
 """
 
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
 from collections import namedtuple
 import os.path
 import re
 from subprocess import Popen
 import sys
 import time
 from textwrap import dedent
 
 from IPython.external.qt import QtCore, QtGui
 
 from IPython.core.inputsplitter import IPythonInputSplitter
 from IPython.core.release import version
 from IPython.core.inputtransformer import ipy_prompt
 from IPython.utils.traitlets import Bool, Unicode
 from .frontend_widget import FrontendWidget
 from . import styles
 
 #-----------------------------------------------------------------------------
 # Constants
 #-----------------------------------------------------------------------------
 
 # Default strings to build and display input and output prompts (and separators
 # in between)
 default_in_prompt = 'In [<span class="in-prompt-number">%i</span>]: '
 default_out_prompt = 'Out[<span class="out-prompt-number">%i</span>]: '
 default_input_sep = '\n'
 default_output_sep = ''
 default_output_sep2 = ''
 
 # Base path for most payload sources.
 zmq_shell_source = 'IPython.kernel.zmq.zmqshell.ZMQInteractiveShell'
 
 if sys.platform.startswith('win'):
     default_editor = 'notepad'
 else:
     default_editor = ''
 
 #-----------------------------------------------------------------------------
 # IPythonWidget class
 #-----------------------------------------------------------------------------
 
 class IPythonWidget(FrontendWidget):
     """ A FrontendWidget for an IPython kernel.
     """
 
     # If set, the 'custom_edit_requested(str, int)' signal will be emitted when
     # an editor is needed for a file. This overrides 'editor' and 'editor_line'
     # settings.
     custom_edit = Bool(False)
     custom_edit_requested = QtCore.Signal(object, object)
 
     editor = Unicode(default_editor, config=True,
         help="""
         A command for invoking a system text editor. If the string contains a
         {filename} format specifier, it will be used. Otherwise, the filename
         will be appended to the end the command.
         """)
 
     editor_line = Unicode(config=True,
         help="""
         The editor command to use when a specific line number is requested. The
         string should contain two format specifiers: {line} and {filename}. If
         this parameter is not specified, the line number option to the %edit
         magic will be ignored.
         """)
 
     style_sheet = Unicode(config=True,
         help="""
         A CSS stylesheet. The stylesheet can contain classes for:
             1. Qt: QPlainTextEdit, QFrame, QWidget, etc
             2. Pygments: .c, .k, .o, etc. (see PygmentsHighlighter)
             3. IPython: .error, .in-prompt, .out-prompt, etc
         """)
 
     syntax_style = Unicode(config=True,
         help="""
         If not empty, use this Pygments style for syntax highlighting.
         Otherwise, the style sheet is queried for Pygments style
         information.
         """)
 
     # Prompts.
     in_prompt = Unicode(default_in_prompt, config=True)
     out_prompt = Unicode(default_out_prompt, config=True)
     input_sep = Unicode(default_input_sep, config=True)
     output_sep = Unicode(default_output_sep, config=True)
     output_sep2 = Unicode(default_output_sep2, config=True)
 
     # FrontendWidget protected class variables.
     _input_splitter_class = IPythonInputSplitter
     _prompt_transformer = IPythonInputSplitter(physical_line_transforms=[ipy_prompt()],
                                                logical_line_transforms=[],
                                                python_line_transforms=[],
                                               )
 
     # IPythonWidget protected class variables.
     _PromptBlock = namedtuple('_PromptBlock', ['block', 'length', 'number'])
-    _payload_source_edit = 'edit_magic'
+    _payload_source_edit = 'edit'
     _payload_source_exit = 'ask_exit'
     _payload_source_next_input = 'set_next_input'
     _payload_source_page = 'page'
     _retrying_history_request = False
     _starting = False
 
     #---------------------------------------------------------------------------
     # 'object' interface
     #---------------------------------------------------------------------------
 
     def __init__(self, *args, **kw):
         super(IPythonWidget, self).__init__(*args, **kw)
 
         # IPythonWidget protected variables.
         self._payload_handlers = {
             self._payload_source_edit : self._handle_payload_edit,
             self._payload_source_exit : self._handle_payload_exit,
             self._payload_source_page : self._handle_payload_page,
             self._payload_source_next_input : self._handle_payload_next_input }
         self._previous_prompt_obj = None
         self._keep_kernel_on_exit = None
 
         # Initialize widget styling.
         if self.style_sheet:
             self._style_sheet_changed()
             self._syntax_style_changed()
         else:
             self.set_default_style()
 
         self._guiref_loaded = False
 
     #---------------------------------------------------------------------------
     # 'BaseFrontendMixin' abstract interface
     #---------------------------------------------------------------------------
     def _handle_complete_reply(self, rep):
         """ Reimplemented to support IPython's improved completion machinery.
         """
         self.log.debug("complete: %s", rep.get('content', ''))
         cursor = self._get_cursor()
         info = self._request_info.get('complete')
         if info and info.id == rep['parent_header']['msg_id'] and \
                 info.pos == cursor.position():
             content = rep['content']
             matches = content['matches']
             start = content['cursor_start']
             end = content['cursor_end']
             
             start = max(start, 0)
             end = max(end, start)
 
             # Move the control's cursor to the desired end point
             cursor_pos = self._get_input_buffer_cursor_pos()
             if end < cursor_pos:
                 cursor.movePosition(QtGui.QTextCursor.Left,
                                     n=(cursor_pos - end))
             elif end > cursor_pos:
                 cursor.movePosition(QtGui.QTextCursor.Right,
                                     n=(end - cursor_pos))
             # This line actually applies the move to control's cursor
             self._control.setTextCursor(cursor)
 
             offset = end - start
             # Move the local cursor object to the start of the match and
             # complete.
             cursor.movePosition(QtGui.QTextCursor.Left, n=offset)
             self._complete_with_items(cursor, matches)
 
     def _handle_execute_reply(self, msg):
         """ Reimplemented to support prompt requests.
         """
         msg_id = msg['parent_header'].get('msg_id')
         info = self._request_info['execute'].get(msg_id)
         if info and info.kind == 'prompt':
             content = msg['content']
             if content['status'] == 'aborted':
                 self._show_interpreter_prompt()
             else:
                 number = content['execution_count'] + 1
                 self._show_interpreter_prompt(number)
             self._request_info['execute'].pop(msg_id)
         else:
             super(IPythonWidget, self)._handle_execute_reply(msg)
 
     def _handle_history_reply(self, msg):
         """ Implemented to handle history tail replies, which are only supported
             by the IPython kernel.
         """
         content = msg['content']
         if 'history' not in content:
             self.log.error("History request failed: %r"%content)
             if content.get('status', '') == 'aborted' and \
                                             not self._retrying_history_request:
                 # a *different* action caused this request to be aborted, so
                 # we should try again.
                 self.log.error("Retrying aborted history request")
                 # prevent multiple retries of aborted requests:
                 self._retrying_history_request = True
                 # wait out the kernel's queue flush, which is currently timed at 0.1s
                 time.sleep(0.25)
                 self.kernel_client.shell_channel.history(hist_access_type='tail',n=1000)
             else:
                 self._retrying_history_request = False
             return
         # reset retry flag
         self._retrying_history_request = False
         history_items = content['history']
         self.log.debug("Received history reply with %i entries", len(history_items))
         items = []
         last_cell = u""
         for _, _, cell in history_items:
             cell = cell.rstrip()
             if cell != last_cell:
                 items.append(cell)
                 last_cell = cell
         self._set_history(items)
     
     def _insert_other_input(self, cursor, content):
         """Insert function for input from other frontends"""
         cursor.beginEditBlock()
         start = cursor.position()
         n = content.get('execution_count', 0)
         cursor.insertText('\n')
         self._insert_html(cursor, self._make_in_prompt(n))
         cursor.insertText(content['code'])
         self._highlighter.rehighlightBlock(cursor.block())
         cursor.endEditBlock()
         
     def _handle_execute_input(self, msg):
         """Handle an execute_input message"""
         self.log.debug("execute_input: %s", msg.get('content', ''))
         if self.include_output(msg):
             self._append_custom(self._insert_other_input, msg['content'], before_prompt=True)
 
     
     def _handle_execute_result(self, msg):
         """ Reimplemented for IPython-style "display hook".
         """
         self.log.debug("execute_result: %s", msg.get('content', ''))
         if self.include_output(msg):
             self.flush_clearoutput()
             content = msg['content']
             prompt_number = content.get('execution_count', 0)
             data = content['data']
             if 'text/plain' in data:
                 self._append_plain_text(self.output_sep, True)
                 self._append_html(self._make_out_prompt(prompt_number), True)
                 text = data['text/plain']
                 # If the repr is multiline, make sure we start on a new line,
                 # so that its lines are aligned.
                 if "\n" in text and not self.output_sep.endswith("\n"):
                     self._append_plain_text('\n', True)
                 self._append_plain_text(text + self.output_sep2, True)
 
     def _handle_display_data(self, msg):
         """ The base handler for the ``display_data`` message.
         """
         self.log.debug("display: %s", msg.get('content', ''))
         # For now, we don't display data from other frontends, but we
         # eventually will as this allows all frontends to monitor the display
         # data. But we need to figure out how to handle this in the GUI.
         if self.include_output(msg):
             self.flush_clearoutput()
             data = msg['content']['data']
             metadata = msg['content']['metadata']
             # In the regular IPythonWidget, we simply print the plain text
             # representation.
             if 'text/plain' in data:
                 text = data['text/plain']
                 self._append_plain_text(text, True)
             # This newline seems to be needed for text and html output.
             self._append_plain_text(u'\n', True)
 
     def _handle_kernel_info_reply(self, rep):
         """Handle kernel info replies."""
         content = rep['content']
         if not self._guiref_loaded:
             if content.get('language') == 'python':
                 self._load_guiref_magic()
             self._guiref_loaded = True
         
         self.kernel_banner = content.get('banner', '')
         if self._starting:
             # finish handling started channels
             self._starting = False
             super(IPythonWidget, self)._started_channels()
 
     def _started_channels(self):
         """Reimplemented to make a history request and load %guiref."""
         self._starting = True
         # The reply will trigger %guiref load provided language=='python'
         self.kernel_client.kernel_info()
 
         self.kernel_client.shell_channel.history(hist_access_type='tail',
                                                   n=1000)
     
     def _load_guiref_magic(self):
         """Load %guiref magic."""
         self.kernel_client.shell_channel.execute('\n'.join([
             "try:",
             "    _usage",
             "except:",
             "    from IPython.core import usage as _usage",
             "    get_ipython().register_magic_function(_usage.page_guiref, 'line', 'guiref')",
             "    del _usage",
         ]), silent=True)
         
     #---------------------------------------------------------------------------
     # 'ConsoleWidget' public interface
     #---------------------------------------------------------------------------
 
     #---------------------------------------------------------------------------
     # 'FrontendWidget' public interface
     #---------------------------------------------------------------------------
 
     def execute_file(self, path, hidden=False):
         """ Reimplemented to use the 'run' magic.
         """
         # Use forward slashes on Windows to avoid escaping each separator.
         if sys.platform == 'win32':
             path = os.path.normpath(path).replace('\\', '/')
 
         # Perhaps we should not be using %run directly, but while we
         # are, it is necessary to quote or escape filenames containing spaces 
         # or quotes. 
         
         # In earlier code here, to minimize escaping, we sometimes quoted the 
         # filename with single quotes. But to do this, this code must be
         # platform-aware, because run uses shlex rather than python string
         # parsing, so that:
         # * In Win: single quotes can be used in the filename without quoting,
         #   and we cannot use single quotes to quote the filename.
         # * In *nix: we can escape double quotes in a double quoted filename,
         #   but can't escape single quotes in a single quoted filename.
         
         # So to keep this code non-platform-specific and simple, we now only
         # use double quotes to quote filenames, and escape when needed:
         if ' ' in path or "'" in path or '"' in path:
             path = '"%s"' % path.replace('"', '\\"')
         self.execute('%%run %s' % path, hidden=hidden)
 
     #---------------------------------------------------------------------------
     # 'FrontendWidget' protected interface
     #---------------------------------------------------------------------------
 
     def _process_execute_error(self, msg):
         """ Reimplemented for IPython-style traceback formatting.
         """
         content = msg['content']
         traceback = '\n'.join(content['traceback']) + '\n'
         if False:
             # FIXME: For now, tracebacks come as plain text, so we can't use
             # the html renderer yet.  Once we refactor ultratb to produce
             # properly styled tracebacks, this branch should be the default
             traceback = traceback.replace(' ', '&nbsp;')
             traceback = traceback.replace('\n', '<br/>')
 
             ename = content['ename']
             ename_styled = '<span class="error">%s</span>' % ename
             traceback = traceback.replace(ename, ename_styled)
 
             self._append_html(traceback)
         else:
             # This is the fallback for now, using plain text with ansi escapes
             self._append_plain_text(traceback)
 
     def _process_execute_payload(self, item):
         """ Reimplemented to dispatch payloads to handler methods.
         """
         handler = self._payload_handlers.get(item['source'])
         if handler is None:
             # We have no handler for this type of payload, simply ignore it
             return False
         else:
             handler(item)
             return True
 
     def _show_interpreter_prompt(self, number=None):
         """ Reimplemented for IPython-style prompts.
         """
         # If a number was not specified, make a prompt number request.
         if number is None:
             msg_id = self.kernel_client.shell_channel.execute('', silent=True)
             info = self._ExecutionRequest(msg_id, 'prompt')
             self._request_info['execute'][msg_id] = info
             return
 
         # Show a new prompt and save information about it so that it can be
         # updated later if the prompt number turns out to be wrong.
         self._prompt_sep = self.input_sep
         self._show_prompt(self._make_in_prompt(number), html=True)
         block = self._control.document().lastBlock()
         length = len(self._prompt)
         self._previous_prompt_obj = self._PromptBlock(block, length, number)
 
         # Update continuation prompt to reflect (possibly) new prompt length.
         self._set_continuation_prompt(
             self._make_continuation_prompt(self._prompt), html=True)
 
     def _show_interpreter_prompt_for_reply(self, msg):
         """ Reimplemented for IPython-style prompts.
         """
         # Update the old prompt number if necessary.
         content = msg['content']
         # abort replies do not have any keys:
         if content['status'] == 'aborted':
             if self._previous_prompt_obj:
                 previous_prompt_number = self._previous_prompt_obj.number
             else:
                 previous_prompt_number = 0
         else:
             previous_prompt_number = content['execution_count']
         if self._previous_prompt_obj and \
                 self._previous_prompt_obj.number != previous_prompt_number:
             block = self._previous_prompt_obj.block
 
             # Make sure the prompt block has not been erased.
             if block.isValid() and block.text():
 
                 # Remove the old prompt and insert a new prompt.
                 cursor = QtGui.QTextCursor(block)
                 cursor.movePosition(QtGui.QTextCursor.Right,
                                     QtGui.QTextCursor.KeepAnchor,
                                     self._previous_prompt_obj.length)
                 prompt = self._make_in_prompt(previous_prompt_number)
                 self._prompt = self._insert_html_fetching_plain_text(
                     cursor, prompt)
 
                 # When the HTML is inserted, Qt blows away the syntax
                 # highlighting for the line, so we need to rehighlight it.
                 self._highlighter.rehighlightBlock(cursor.block())
 
             self._previous_prompt_obj = None
 
         # Show a new prompt with the kernel's estimated prompt number.
         self._show_interpreter_prompt(previous_prompt_number + 1)
 
     #---------------------------------------------------------------------------
     # 'IPythonWidget' interface
     #---------------------------------------------------------------------------
 
     def set_default_style(self, colors='lightbg'):
         """ Sets the widget style to the class defaults.
 
         Parameters
         ----------
         colors : str, optional (default lightbg)
             Whether to use the default IPython light background or dark
             background or B&W style.
         """
         colors = colors.lower()
         if colors=='lightbg':
             self.style_sheet = styles.default_light_style_sheet
             self.syntax_style = styles.default_light_syntax_style
         elif colors=='linux':
             self.style_sheet = styles.default_dark_style_sheet
             self.syntax_style = styles.default_dark_syntax_style
         elif colors=='nocolor':
             self.style_sheet = styles.default_bw_style_sheet
             self.syntax_style = styles.default_bw_syntax_style
         else:
             raise KeyError("No such color scheme: %s"%colors)
 
     #---------------------------------------------------------------------------
     # 'IPythonWidget' protected interface
     #---------------------------------------------------------------------------
 
     def _edit(self, filename, line=None):
         """ Opens a Python script for editing.
 
         Parameters
         ----------
         filename : str
             A path to a local system file.
 
         line : int, optional
             A line of interest in the file.
         """
         if self.custom_edit:
             self.custom_edit_requested.emit(filename, line)
         elif not self.editor:
             self._append_plain_text('No default editor available.\n'
             'Specify a GUI text editor in the `IPythonWidget.editor` '
             'configurable to enable the %edit magic')
         else:
             try:
                 filename = '"%s"' % filename
                 if line and self.editor_line:
                     command = self.editor_line.format(filename=filename,
                                                       line=line)
                 else:
                     try:
                         command = self.editor.format()
                     except KeyError:
                         command = self.editor.format(filename=filename)
                     else:
                         command += ' ' + filename
             except KeyError:
                 self._append_plain_text('Invalid editor command.\n')
             else:
                 try:
                     Popen(command, shell=True)
                 except OSError:
                     msg = 'Opening editor with command "%s" failed.\n'
                     self._append_plain_text(msg % command)
 
     def _make_in_prompt(self, number):
         """ Given a prompt number, returns an HTML In prompt.
         """
         try:
             body = self.in_prompt % number
         except TypeError:
             # allow in_prompt to leave out number, e.g. '>>> '
             from xml.sax.saxutils import escape
             body = escape(self.in_prompt)
         return '<span class="in-prompt">%s</span>' % body
 
     def _make_continuation_prompt(self, prompt):
         """ Given a plain text version of an In prompt, returns an HTML
             continuation prompt.
         """
         end_chars = '...: '
         space_count = len(prompt.lstrip('\n')) - len(end_chars)
         body = '&nbsp;' * space_count + end_chars
         return '<span class="in-prompt">%s</span>' % body
 
     def _make_out_prompt(self, number):
         """ Given a prompt number, returns an HTML Out prompt.
         """
         try:
             body = self.out_prompt % number
         except TypeError:
             # allow out_prompt to leave out number, e.g. '<<< '
             from xml.sax.saxutils import escape
             body = escape(self.out_prompt)
         return '<span class="out-prompt">%s</span>' % body
 
     #------ Payload handlers --------------------------------------------------
 
     # Payload handlers with a generic interface: each takes the opaque payload
     # dict, unpacks it and calls the underlying functions with the necessary
     # arguments.
 
     def _handle_payload_edit(self, item):
         self._edit(item['filename'], item['line_number'])
 
     def _handle_payload_exit(self, item):
         self._keep_kernel_on_exit = item['keepkernel']
         self.exit_requested.emit(self)
 
     def _handle_payload_next_input(self, item):
         self.input_buffer = item['text']
 
     def _handle_payload_page(self, item):
         # Since the plain text widget supports only a very small subset of HTML
         # and we have no control over the HTML source, we only page HTML
         # payloads in the rich text widget.
         data = item['data']
         if 'text/html' in data and self.kind == 'rich':
             self._page(data['text/html'], html=True)
         else:
             self._page(data['text/plain'], html=False)
 
     #------ Trait change handlers --------------------------------------------
 
     def _style_sheet_changed(self):
         """ Set the style sheets of the underlying widgets.
         """
         self.setStyleSheet(self.style_sheet)
         if self._control is not None:
             self._control.document().setDefaultStyleSheet(self.style_sheet)
             bg_color = self._control.palette().window().color()
             self._ansi_processor.set_background_color(bg_color)
         
         if self._page_control is not None:
             self._page_control.document().setDefaultStyleSheet(self.style_sheet)
 
 
 
     def _syntax_style_changed(self):
         """ Set the style for the syntax highlighter.
         """
         if self._highlighter is None:
             # ignore premature calls
             return
         if self.syntax_style:
             self._highlighter.set_style(self.syntax_style)
         else:
             self._highlighter.set_style_sheet(self.style_sheet)
 
     #------ Trait default initializers -----------------------------------------
 
     def _banner_default(self):
         return "IPython QtConsole {version}\n".format(version=version)

docs/source/development/messaging.rst

@@ -1,1138 +1,1184 b''
 .. _messaging:
 
 ======================
  Messaging in IPython
 ======================
 
 
 Versioning
 ==========
 
 The IPython message specification is versioned independently of IPython.
 The current version of the specification is 5.0.
 
 
 Introduction
 ============
 
 This document explains the basic communications design and messaging
 specification for how the various IPython objects interact over a network
 transport.  The current implementation uses the ZeroMQ_ library for messaging
 within and between hosts.
 
 .. Note::
 
    This document should be considered the authoritative description of the
    IPython messaging protocol, and all developers are strongly encouraged to
    keep it updated as the implementation evolves, so that we have a single
    common reference for all protocol details.
 
 The basic design is explained in the following diagram:
 
 .. image:: figs/frontend-kernel.png
    :width: 450px
    :alt: IPython kernel/frontend messaging architecture.
    :align: center
    :target: ../_images/frontend-kernel.png
 
 A single kernel can be simultaneously connected to one or more frontends.  The
 kernel has three sockets that serve the following functions:
 
 1. Shell: this single ROUTER socket allows multiple incoming connections from
    frontends, and this is the socket where requests for code execution, object
    information, prompts, etc. are made to the kernel by any frontend.  The
    communication on this socket is a sequence of request/reply actions from
    each frontend and the kernel.
 
 2. IOPub: this socket is the 'broadcast channel' where the kernel publishes all
    side effects (stdout, stderr, etc.) as well as the requests coming from any
    client over the shell socket and its own requests on the stdin socket.  There
    are a number of actions in Python which generate side effects: :func:`print`
    writes to ``sys.stdout``, errors generate tracebacks, etc.  Additionally, in
    a multi-client scenario, we want all frontends to be able to know what each
    other has sent to the kernel (this can be useful in collaborative scenarios,
    for example).  This socket allows both side effects and the information
    about communications taking place with one client over the shell channel
    to be made available to all clients in a uniform manner.
 
 3. stdin: this ROUTER socket is connected to all frontends, and it allows
    the kernel to request input from the active frontend when :func:`raw_input` is called.
    The frontend that executed the code has a DEALER socket that acts as a 'virtual keyboard'
    for the kernel while this communication is happening (illustrated in the
    figure by the black outline around the central keyboard).  In practice,
    frontends may display such kernel requests using a special input widget or
    otherwise indicating that the user is to type input for the kernel instead
    of normal commands in the frontend.
 
    All messages are tagged with enough information (details below) for clients
    to know which messages come from their own interaction with the kernel and
    which ones are from other clients, so they can display each type
    appropriately.
 
 4. Control: This channel is identical to Shell, but operates on a separate socket,
    to allow important messages to avoid queueing behind execution requests (e.g. shutdown or abort).
 
 The actual format of the messages allowed on each of these channels is
 specified below.  Messages are dicts of dicts with string keys and values that
 are reasonably representable in JSON.  Our current implementation uses JSON
 explicitly as its message format, but this shouldn't be considered a permanent
 feature.  As we've discovered that JSON has non-trivial performance issues due
 to excessive copying, we may in the future move to a pure pickle-based raw
 message format.  However, it should be possible to easily convert from the raw
 objects to JSON, since we may have non-python clients (e.g. a web frontend).
 As long as it's easy to make a JSON version of the objects that is a faithful
 representation of all the data, we can communicate with such clients.
 
 .. Note::
 
    Not all of these have yet been fully fleshed out, but the key ones are, see
    kernel and frontend files for actual implementation details.
 
 General Message Format
 ======================
 
 A message is defined by the following four-dictionary structure::
 
     {
       # The message header contains a pair of unique identifiers for the
       # originating session and the actual message id, in addition to the
       # username for the process that generated the message.  This is useful in
       # collaborative settings where multiple users may be interacting with the
       # same kernel simultaneously, so that frontends can label the various
       # messages in a meaningful way.
       'header' : {
                     'msg_id' : uuid,
                     'username' : str,
                     'session' : uuid,
                     # All recognized message type strings are listed below.
                     'msg_type' : str,
                     # the message protocol version
                     'version' : '5.0',
          },
 
       # In a chain of messages, the header from the parent is copied so that
       # clients can track where messages come from.
       'parent_header' : dict,
       
       # Any metadata associated with the message.
       'metadata' : dict,
       
       # The actual content of the message must be a dict, whose structure
       # depends on the message type.
       'content' : dict,
     }
 
 .. versionchanged:: 5.0
 
     ``version`` key added to the header.
 
 .. _wire_protocol:
 
 The Wire Protocol
 =================
 
 
 This message format exists at a high level,
 but does not describe the actual *implementation* at the wire level in zeromq.
 The canonical implementation of the message spec is our :class:`~IPython.kernel.zmq.session.Session` class.
 
 .. note::
 
     This section should only be relevant to non-Python consumers of the protocol.
     Python consumers should simply import and use IPython's own implementation of the wire protocol
     in the :class:`IPython.kernel.zmq.session.Session` object.
 
 Every message is serialized to a sequence of at least six blobs of bytes:
 
 .. sourcecode:: python
 
     [
       b'u-u-i-d',         # zmq identity(ies)
       b'<IDS|MSG>',       # delimiter
       b'baddad42',        # HMAC signature
       b'{header}',        # serialized header dict
       b'{parent_header}', # serialized parent header dict
       b'{metadata}',      # serialized metadata dict
       b'{content},        # serialized content dict
       b'blob',            # extra raw data buffer(s)
       ...
     ]
 
 The front of the message is the ZeroMQ routing prefix,
 which can be zero or more socket identities.
 This is every piece of the message prior to the delimiter key ``<IDS|MSG>``.
 In the case of IOPub, there should be just one prefix component,
 which is the topic for IOPub subscribers, e.g. ``execute_result``, ``display_data``.
 
 .. note::
 
     In most cases, the IOPub topics are irrelevant and completely ignored,
     because frontends just subscribe to all topics.
     The convention used in the IPython kernel is to use the msg_type as the topic,
     and possibly extra information about the message, e.g. ``execute_result`` or ``stream.stdout``
 
 After the delimiter is the `HMAC`_ signature of the message, used for authentication.
 If authentication is disabled, this should be an empty string.
 By default, the hashing function used for computing these signatures is sha256.
 
 .. _HMAC: http://en.wikipedia.org/wiki/HMAC
 
 .. note::
 
     To disable authentication and signature checking,
     set the `key` field of a connection file to an empty string.
 
 The signature is the HMAC hex digest of the concatenation of:
 
 - A shared key (typically the ``key`` field of a connection file)
 - The serialized header dict
 - The serialized parent header dict
 - The serialized metadata dict
 - The serialized content dict
 
 In Python, this is implemented via:
 
 .. sourcecode:: python
 
     # once:
     digester = HMAC(key, digestmod=hashlib.sha256)
     
     # for each message
     d = digester.copy()
     for serialized_dict in (header, parent, metadata, content):
         d.update(serialized_dict)
     signature = d.hexdigest()
 
 After the signature is the actual message, always in four frames of bytes.
 The four dictionaries that compose a message are serialized separately,
 in the order of header, parent header, metadata, and content.
 These can be serialized by any function that turns a dict into bytes.
 The default and most common serialization is JSON, but msgpack and pickle
 are common alternatives.
 
 After the serialized dicts are zero to many raw data buffers,
 which can be used by message types that support binary data (mainly apply and data_pub).
 
 
 Python functional API
 =====================
 
 As messages are dicts, they map naturally to a ``func(**kw)`` call form.  We
 should develop, at a few key points, functional forms of all the requests that
 take arguments in this manner and automatically construct the necessary dict
 for sending.
 
 In addition, the Python implementation of the message specification extends
 messages upon deserialization to the following form for convenience::
 
     {
       'header' : dict,
       # The msg's unique identifier and type are always stored in the header,
       # but the Python implementation copies them to the top level.
       'msg_id' : uuid,
       'msg_type' : str,
       'parent_header' : dict,
       'content' : dict,
       'metadata' : dict,
     }
 
 All messages sent to or received by any IPython process should have this
 extended structure.
 
 
 Messages on the shell ROUTER/DEALER sockets
 ===========================================
 
 .. _execute:
 
 Execute
 -------
 
 This message type is used by frontends to ask the kernel to execute code on
 behalf of the user, in a namespace reserved to the user's variables (and thus
 separate from the kernel's own internal code and variables).
 
 Message type: ``execute_request``::
 
     content = {
         # Source code to be executed by the kernel, one or more lines.
     'code' : str,
 
     # A boolean flag which, if True, signals the kernel to execute
     # this code as quietly as possible.
     # silent=True forces store_history to be False,
     # and will *not*:
     #   - broadcast output on the IOPUB channel
     #   - have an execute_result
     # The default is False.
     'silent' : bool,
 
     # A boolean flag which, if True, signals the kernel to populate history
     # The default is True if silent is False.  If silent is True, store_history
     # is forced to be False.
     'store_history' : bool,
 
     # A dict mapping names to expressions to be evaluated in the
     # user's dict. The rich display-data representation of each will be evaluated after execution.
     # See the display_data content for the structure of the representation data.
     'user_expressions' : dict,
 
     # Some frontends do not support stdin requests.
     # If raw_input is called from code executed from such a frontend,
     # a StdinNotImplementedError will be raised.
     'allow_stdin' : True,
     }
 
 .. versionchanged:: 5.0
 
     ``user_variables`` removed, because it is redundant with user_expressions.
 
 The ``code`` field contains a single string (possibly multiline) to be executed.
 
 The ``user_expressions`` field deserves a detailed explanation.  In the past, IPython had
 the notion of a prompt string that allowed arbitrary code to be evaluated, and
 this was put to good use by many in creating prompts that displayed system
 status, path information, and even more esoteric uses like remote instrument
 status acquired over the network.  But now that IPython has a clean separation
 between the kernel and the clients, the kernel has no prompt knowledge; prompts
 are a frontend feature, and it should be even possible for different
 frontends to display different prompts while interacting with the same kernel.
 ``user_expressions`` can be used to retrieve this information.
 
 Any error in evaluating any expression in ``user_expressions`` will result in
 only that key containing a standard error message, of the form::
 
     {
         'status' : 'error',
         'ename' : 'NameError',
         'evalue' : 'foo',
         'traceback' : ...
     }
 
 .. Note::
 
    In order to obtain the current execution counter for the purposes of
    displaying input prompts, frontends may make an execution request with an
    empty code string and ``silent=True``.
 
 Upon completion of the execution request, the kernel *always* sends a reply,
 with a status code indicating what happened and additional data depending on
 the outcome.  See :ref:`below <execution_results>` for the possible return
 codes and associated data.
 
 .. seealso::
 
     :ref:`execution_semantics`
 
 .. _execution_counter:
 
 Execution counter (prompt number)
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 The kernel should have a single, monotonically increasing counter of all execution
 requests that are made with ``store_history=True``. This counter is used to populate
 the ``In[n]`` and ``Out[n]`` prompts.  The value of this counter will be returned as the
 ``execution_count`` field of all ``execute_reply`` and ``execute_input`` messages.
 
 .. _execution_results:
 
 Execution results
 ~~~~~~~~~~~~~~~~~
     
 Message type: ``execute_reply``::
 
     content = {
       # One of: 'ok' OR 'error' OR 'abort'
       'status' : str,
 
       # The global kernel counter that increases by one with each request that 
       # stores history.  This will typically be used by clients to display
       # prompt numbers to the user.  If the request did not store history, this will
       # be the current value of the counter in the kernel.
       'execution_count' : int,
     }
 
 When status is 'ok', the following extra fields are present::
 
     {
-      # 'payload' will be a list of payload dicts.
-      # Each execution payload is a dict with string keys that may have been
-      # produced by the code being executed.  It is retrieved by the kernel at
-      # the end of the execution and sent back to the front end, which can take
-      # action on it as needed.
+      # 'payload' will be a list of payload dicts, and is optional.
+      # payloads are considered deprecated.
       # The only requirement of each payload dict is that it have a 'source' key,
-      # which is a string classifying the payload (e.g. 'pager').
+      # which is a string classifying the payload (e.g. 'page').
+      
       'payload' : list(dict),
 
       # Results for the user_expressions.
       'user_expressions' : dict,
     }
 
 .. versionchanged:: 5.0
 
     ``user_variables`` is removed, use user_expressions instead.
 
-.. admonition:: Execution payloads
-    
-   The notion of an 'execution payload' is different from a return value of a
-   given set of code, which normally is just displayed on the execute_result stream
-   through the PUB socket.  The idea of a payload is to allow special types of
-   code, typically magics, to populate a data container in the IPython kernel
-   that will be shipped back to the caller via this channel.  The kernel
-   has an API for this in the PayloadManager::
-
-       ip.payload_manager.write_payload(payload_dict)
-
-   which appends a dictionary to the list of payloads.
-   
-   The payload API is not yet stabilized,
-   and should probably not be supported by non-Python kernels at this time.
-   In such cases, the payload list should always be empty.
-
-   
 When status is 'error', the following extra fields are present::
 
     {
       'ename' : str,   # Exception name, as a string
       'evalue' : str,  # Exception value, as a string
 
       # The traceback will contain a list of frames, represented each as a
       # string.  For now we'll stick to the existing design of ultraTB, which
       # controls exception level of detail statefully.  But eventually we'll
       # want to grow into a model where more information is collected and
       # packed into the traceback object, with clients deciding how little or
       # how much of it to unpack.  But for now, let's start with a simple list
       # of strings, since that requires only minimal changes to ultratb as
       # written.
       'traceback' : list,
     }
 
 
 When status is 'abort', there are for now no additional data fields.  This
 happens when the kernel was interrupted by a signal.
 
+Payloads
+********
+
+.. admonition:: Execution payloads
+
+    Payloads are considered deprecated, though their replacement is not yet implemented.
+
+Payloads are a way to trigger frontend actions from the kernel. Current payloads:
+
+**page**: display data in a pager.
+
+Pager output is used for introspection, or other displayed information that's not considered output.
+Pager payloads are generally displayed in a separate pane, that can be viewed alongside code,
+and are not included in notebook documents.
+
+.. sourcecode:: python
+
+    {
+      "source": "page",
+      # mime-bundle of data to display in the pager.
+      # Must include text/plain.
+      "data": mimebundle,
+      # line offset to start from
+      "start": int,
+    }
+
+**set_next_input**: create a new output
+
+used to create new cells in the notebook,
+or set the next input in a console interface.
+The main example being ``%load``.
+
+.. sourcecode:: python
+
+    {
+      "source": "set_next_input",
+      # the text contents of the cell to create
+      "text": "some cell content",
+    }
+
+**edit**: open a file for editing.
+
+Triggered by `%edit`. Only the QtConsole currently supports edit payloads.
+
+.. sourcecode:: python
+
+    {
+      "source": "edit",
+      "filename": "/path/to/file.py", # the file to edit
+      "line_number": int, # the line number to start with
+    }
+
+**ask_exit**: instruct the frontend to prompt the user for exit
+
+Allows the kernel to request exit, e.g. via ``%exit`` in IPython.
+Only for console frontends.
+
+.. sourcecode:: python
+
+    {
+      "source": "ask_exit",
+      # whether the kernel should be left running, only closing the client
+      "keepkernel": bool,
+    }
+
+
 .. _msging_inspection:
 
 Introspection
 -------------
 
 Code can be inspected to show useful information to the user.
 It is up to the Kernel to decide what information should be displayed, and its formatting.
 
 Message type: ``inspect_request``::
 
     content = {
         # The code context in which introspection is requested
         # this may be up to an entire multiline cell.
         'code' : str,
         
         # The cursor position within 'code' (in unicode characters) where inspection is requested
         'cursor_pos' : int,
 
         # The level of detail desired.  In IPython, the default (0) is equivalent to typing
         # 'x?' at the prompt, 1 is equivalent to 'x??'.
         # The difference is up to kernels, but in IPython level 1 includes the source code
         # if available.
         'detail_level' : 0 or 1,
     }
 
 .. versionchanged:: 5.0
 
     ``object_info_request`` renamed to ``inspect_request``.
 
 .. versionchanged:: 5.0
 
     ``name`` key replaced with ``code`` and ``cursor_pos``,
     moving the lexing responsibility to the kernel.
 
 The reply is a mime-bundle, like a `display_data`_ message,
 which should be a formatted representation of information about the context.
 In the notebook, this is used to show tooltips over function calls, etc.
 
 Message type: ``inspect_reply``::
 
     content = {
         # 'ok' if the request succeeded or 'error', with error information as in all other replies.
         'status' : 'ok',
         
         # data can be empty if nothing is found
         'data' : dict,
         'metadata' : dict,
     }
 
 .. versionchanged:: 5.0
 
     ``object_info_reply`` renamed to ``inspect_reply``.
 
 .. versionchanged:: 5.0
 
     Reply is changed from structured data to a mime bundle,  allowing formatting decisions to be made by the kernel.
 
 .. _msging_completion:
 
 Completion
 ----------
 
 Message type: ``complete_request``::
 
     content = {
         # The code context in which completion is requested
         # this may be up to an entire multiline cell, such as
         # 'foo = a.isal'
         'code' : str,
         
         # The cursor position within 'code' (in unicode characters) where completion is requested
         'cursor_pos' : int,
     }
 
 .. versionchanged:: 5.0
 
     ``line``, ``block``, and ``text`` keys are removed in favor of a single ``code`` for context.
     Lexing is up to the kernel.
 
 
 Message type: ``complete_reply``::
 
     content = {
     # The list of all matches to the completion request, such as
     # ['a.isalnum', 'a.isalpha'] for the above example.
     'matches' : list,
     
     # The range of text that should be replaced by the above matches when a completion is accepted.
     # typically cursor_end is the same as cursor_pos in the request.
     'cursor_start' : int,
     'cursor_end' : int,
     
     # Information that frontend plugins might use for extra display information about completions.
     'metadata' : dict,
     
     # status should be 'ok' unless an exception was raised during the request,
     # in which case it should be 'error', along with the usual error message content
     # in other messages.
     'status' : 'ok'
     }
 
 .. versionchanged:: 5.0
 
     - ``matched_text`` is removed in favor of ``cursor_start`` and ``cursor_end``.
     - ``metadata`` is added for extended information.
 
 .. _msging_history:
 
 History
 -------
 
 For clients to explicitly request history from a kernel.  The kernel has all
 the actual execution history stored in a single location, so clients can
 request it from the kernel when needed.
 
 Message type: ``history_request``::
 
     content = {
     
       # If True, also return output history in the resulting dict.
       'output' : bool,
 
       # If True, return the raw input history, else the transformed input.
       'raw' : bool,
 
       # So far, this can be 'range', 'tail' or 'search'.
       'hist_access_type' : str,
       
       # If hist_access_type is 'range', get a range of input cells. session can
       # be a positive session number, or a negative number to count back from
       # the current session.
       'session' : int,
       # start and stop are line numbers within that session.
       'start' : int,
       'stop' : int,
       
       # If hist_access_type is 'tail' or 'search', get the last n cells.
       'n' : int,
       
       # If hist_access_type is 'search', get cells matching the specified glob
       # pattern (with * and ? as wildcards).
       'pattern' : str,
 
       # If hist_access_type is 'search' and unique is true, do not
       # include duplicated history.  Default is false.
       'unique' : bool,
       
     }
 
 .. versionadded:: 4.0
    The key ``unique`` for ``history_request``.
 
 Message type: ``history_reply``::
 
     content = {
       # A list of 3 tuples, either:
       # (session, line_number, input) or
       # (session, line_number, (input, output)),
       # depending on whether output was False or True, respectively.
       'history' : list,
     }
 
 .. _msging_is_complete:
 
 Code completeness
 -----------------
 
 .. versionadded:: 5.0
 
 When the user enters a line in a console style interface, the console must
 decide whether to immediately execute the current code, or whether to show a
 continuation prompt for further input. For instance, in Python ``a = 5`` would
 be executed immediately, while ``for i in range(5):`` would expect further input.
 
 There are four possible replies:
 
 - *complete* code is ready to be executed
 - *incomplete* code should prompt for another line
 - *invalid* code will typically be sent for execution, so that the user sees the
   error soonest.
 - *unknown* - if the kernel is not able to determine this. The frontend should
   also handle the kernel not replying promptly. It may default to sending the
   code for execution, or it may implement simple fallback heuristics for whether
   to execute the code (e.g. execute after a blank line).
 
 Frontends may have ways to override this, forcing the code to be sent for
 execution or forcing a continuation prompt.
 
 Message type: ``is_complete_request``::
 
     content = {
         # The code entered so far as a multiline string
         'code' : str,
     }
 
 Message type: ``is_complete_reply``::
 
     content = {
         # One of 'complete', 'incomplete', 'invalid', 'unknown'
         'status' : str,
         
         # If status is 'incomplete', indent should contain the characters to use
         # to indent the next line. This is only a hint: frontends may ignore it
         # and use their own autoindentation rules. For other statuses, this
         # field does not exist.
         'indent': str,
     }
 
 Connect
 -------
 
 When a client connects to the request/reply socket of the kernel, it can issue
 a connect request to get basic information about the kernel, such as the ports
 the other ZeroMQ sockets are listening on. This allows clients to only have
 to know about a single port (the shell channel) to connect to a kernel.
 
 Message type: ``connect_request``::
 
     content = {
     }
 
 Message type: ``connect_reply``::
 
     content = {
         'shell_port' : int,   # The port the shell ROUTER socket is listening on.
         'iopub_port' : int,   # The port the PUB socket is listening on.
         'stdin_port' : int,   # The port the stdin ROUTER socket is listening on.
         'hb_port' : int,      # The port the heartbeat socket is listening on.
     }
 
 .. _msging_kernel_info:
 
 Kernel info
 -----------
 
 If a client needs to know information about the kernel, it can
 make a request of the kernel's information.
 This message can be used to fetch core information of the
 kernel, including language (e.g., Python), language version number and
 IPython version number, and the IPython message spec version number.
 
 Message type: ``kernel_info_request``::
 
     content = {
     }
 
 Message type: ``kernel_info_reply``::
 
     content = {
         # Version of messaging protocol.
         # The first integer indicates major version.  It is incremented when
         # there is any backward incompatible change.
         # The second integer indicates minor version.  It is incremented when
         # there is any backward compatible change.
         'protocol_version': 'X.Y.Z',
 
         # The kernel implementation name
         # (e.g. 'ipython' for the IPython kernel)
         'implementation': str,
 
         # Implementation version number.
         # The version number of the kernel's implementation
         # (e.g. IPython.__version__ for the IPython kernel)
         'implementation_version': 'X.Y.Z', 
 
         # Programming language in which kernel is implemented.
         # Kernel included in IPython returns 'python'.
         'language': str,
         
         # Language version number.
         # It is Python version number (e.g., '2.7.3') for the kernel
         # included in IPython.
         'language_version': 'X.Y.Z',
 
         # Information about the language of code for the kernel
         'language_info': {
             'mimetype': str,
 
             # Pygments lexer, for highlighting
             # Only needed if it differs from the top level 'language' field.
             'pygments_lexer': str,
 
             # Codemirror mode, for for highlighting in the notebook.
             # Only needed if it differs from the top level 'language' field.
             'codemirror_mode': str or dict,
         },
 
         # A banner of information about the kernel,
         # which may be desplayed in console environments.
         'banner' : str,
 
         # Optional: A list of dictionaries, each with keys 'text' and 'url'.
         # These will be displayed in the help menu in the notebook UI.
         'help_links': [
             {'text': str, 'url': str}
         ],
     }
 
 Refer to the lists of available `Pygments lexers <http://pygments.org/docs/lexers/>`_
 and `codemirror modes <http://codemirror.net/mode/index.html>`_ for those fields.
 
 .. versionchanged:: 5.0
 
     Versions changed from lists of integers to strings.
     
 .. versionchanged:: 5.0
 
     ``ipython_version`` is removed.
 
 .. versionchanged:: 5.0
 
     ``language_info``, ``implementation``, ``implementation_version``, ``banner``
     and ``help_links`` keys are added.
 
 .. _msging_shutdown:
 
 Kernel shutdown
 ---------------
 
 The clients can request the kernel to shut itself down; this is used in
 multiple cases:
 
 - when the user chooses to close the client application via a menu or window
   control.
 - when the user types 'exit' or 'quit' (or their uppercase magic equivalents).
 - when the user chooses a GUI method (like the 'Ctrl-C' shortcut in the
   IPythonQt client) to force a kernel restart to get a clean kernel without
   losing client-side state like history or inlined figures.
 
 The client sends a shutdown request to the kernel, and once it receives the
 reply message (which is otherwise empty), it can assume that the kernel has
 completed shutdown safely.
 
 Upon their own shutdown, client applications will typically execute a last
 minute sanity check and forcefully terminate any kernel that is still alive, to
 avoid leaving stray processes in the user's machine.
 
 Message type: ``shutdown_request``::
 
     content = {
         'restart' : bool # whether the shutdown is final, or precedes a restart
     }
 
 Message type: ``shutdown_reply``::
 
     content = {
         'restart' : bool # whether the shutdown is final, or precedes a restart
     }
 
 .. Note::
 
    When the clients detect a dead kernel thanks to inactivity on the heartbeat
    socket, they simply send a forceful process termination signal, since a dead
    process is unlikely to respond in any useful way to messages.
     
 
 Messages on the PUB/SUB socket
 ==============================
 
 Streams (stdout,  stderr, etc)
 ------------------------------
 
 Message type: ``stream``::
 
     content = {
         # The name of the stream is one of 'stdout', 'stderr'
         'name' : str,
     
         # The text is an arbitrary string to be written to that stream
         'text' : str,
     }
 
 .. versionchanged:: 5.0
 
     'data' key renamed to 'text' for conistency with the notebook format.
 
 Display Data
 ------------
 
 This type of message is used to bring back data that should be displayed (text,
 html, svg, etc.) in the frontends. This data is published to all frontends.
 Each message can have multiple representations of the data; it is up to the
 frontend to decide which to use and how. A single message should contain all
 possible representations of the same information. Each representation should
 be a JSON'able data structure, and should be a valid MIME type.
 
 Some questions remain about this design:
 
 * Do we use this message type for execute_result/displayhook? Probably not, because
   the displayhook also has to handle the Out prompt display. On the other hand
   we could put that information into the metadata section.
 
 .. _display_data:
 
 Message type: ``display_data``::
 
     content = {
 
         # Who create the data
         'source' : str,
 
         # The data dict contains key/value pairs, where the keys are MIME
         # types and the values are the raw data of the representation in that
         # format.
         'data' : dict,
 
         # Any metadata that describes the data
         'metadata' : dict
     }
 
 
 The ``metadata`` contains any metadata that describes the output.
 Global keys are assumed to apply to the output as a whole.
 The ``metadata`` dict can also contain mime-type keys, which will be sub-dictionaries,
 which are interpreted as applying only to output of that type.
 Third parties should put any data they write into a single dict
 with a reasonably unique name to avoid conflicts.
 
 The only metadata keys currently defined in IPython are the width and height
 of images::
 
     metadata = {
       'image/png' : {
         'width': 640,
         'height': 480
       }
     }
 
 
 .. versionchanged:: 5.0
 
     `application/json` data should be unpacked JSON data,
     not double-serialized as a JSON string.
 
 
 Raw Data Publication
 --------------------
 
 ``display_data`` lets you publish *representations* of data, such as images and html.
 This ``data_pub`` message lets you publish *actual raw data*, sent via message buffers.
 
 data_pub messages are constructed via the :func:`IPython.lib.datapub.publish_data` function:
 
 .. sourcecode:: python
 
     from IPython.kernel.zmq.datapub import publish_data
     ns = dict(x=my_array)
     publish_data(ns)
 
 
 Message type: ``data_pub``::
 
     content = {
         # the keys of the data dict, after it has been unserialized
         'keys' : ['a', 'b']
     }
     # the namespace dict will be serialized in the message buffers,
     # which will have a length of at least one
     buffers = [b'pdict', ...]
 
 
 The interpretation of a sequence of data_pub messages for a given parent request should be
 to update a single namespace with subsequent results.
 
 .. note::
 
     No frontends directly handle data_pub messages at this time.
     It is currently only used by the client/engines in :mod:`IPython.parallel`,
     where engines may publish *data* to the Client,
     of which the Client can then publish *representations* via ``display_data``
     to various frontends.
 
 Code inputs
 -----------
 
 To let all frontends know what code is being executed at any given time, these
 messages contain a re-broadcast of the ``code`` portion of an
 :ref:`execute_request <execute>`, along with the :ref:`execution_count
 <execution_counter>`.
 
 Message type: ``execute_input``::
 
     content = {
         'code' : str,  # Source code to be executed, one or more lines
 
         # The counter for this execution is also provided so that clients can
         # display it, since IPython automatically creates variables called _iN
         # (for input prompt In[N]).
         'execution_count' : int
     }
 
 .. versionchanged:: 5.0
 
     ``pyin`` is renamed to ``execute_input``.
 
 
 Execution results
 -----------------
 
 Results of an execution are published as an ``execute_result``.
 These are identical to `display_data`_ messages, with the addition of an ``execution_count`` key.
 
 Results can have multiple simultaneous formats depending on its
 configuration. A plain text representation should always be provided
 in the ``text/plain`` mime-type. Frontends are free to display any or all of these
 according to its capabilities.
 Frontends should ignore mime-types they do not understand. The data itself is
 any JSON object and depends on the format. It is often, but not always a string.
 
 Message type: ``execute_result``::
 
     content = {
 
         # The counter for this execution is also provided so that clients can
         # display it, since IPython automatically creates variables called _N
         # (for prompt N).
         'execution_count' : int,
 
         # data and metadata are identical to a display_data message.
         # the object being displayed is that passed to the display hook,
         # i.e. the *result* of the execution.
         'data' : dict,
         'metadata' : dict,
     }
 
 Execution errors
 ----------------
 
 When an error occurs during code execution
 
 Message type: ``error``::
 
     content = {
        # Similar content to the execute_reply messages for the 'error' case,
        # except the 'status' field is omitted.
     }
 
 .. versionchanged:: 5.0
 
     ``pyerr`` renamed to ``error``
 
 Kernel status
 -------------
 
 This message type is used by frontends to monitor the status of the kernel.
 
 Message type: ``status``::
 
     content = {
         # When the kernel starts to handle a message, it will enter the 'busy'
         # state and when it finishes, it will enter the 'idle' state.
         # The kernel will publish state 'starting' exactly once at process startup.
         execution_state : ('busy', 'idle', 'starting')
     }
 
 .. versionchanged:: 5.0
 
     Busy and idle messages should be sent before/after handling every message,
     not just execution.
 
 Clear output
 ------------
 
 This message type is used to clear the output that is visible on the frontend.
 
 Message type: ``clear_output``::
 
     content = {
 
         # Wait to clear the output until new output is available.  Clears the
         # existing output immediately before the new output is displayed.  
         # Useful for creating simple animations with minimal flickering.
         'wait' : bool,
     }
 
 .. versionchanged:: 4.1
 
     ``stdout``, ``stderr``, and ``display`` boolean keys for selective clearing are removed,
     and ``wait`` is added.
     The selective clearing keys are ignored in v4 and the default behavior remains the same,
     so v4 clear_output messages will be safely handled by a v4.1 frontend.
 
 
 Messages on the stdin ROUTER/DEALER sockets
 ===========================================
 
 This is a socket where the request/reply pattern goes in the opposite direction:
 from the kernel to a *single* frontend, and its purpose is to allow
 ``raw_input`` and similar operations that read from ``sys.stdin`` on the kernel
 to be fulfilled by the client. The request should be made to the frontend that
 made the execution request that prompted ``raw_input`` to be called. For now we
 will keep these messages as simple as possible, since they only mean to convey
 the ``raw_input(prompt)`` call.
 
 Message type: ``input_request``::
 
     content = {
         # the text to show at the prompt
         'prompt' : str,
         # Is the request for a password?
         # If so, the frontend shouldn't echo input.
         'password' : bool
     }
 
 Message type: ``input_reply``::
 
     content = { 'value' : str }
 
 
 When ``password`` is True, the frontend should not echo the input as it is entered.
 
 .. versionchanged:: 5.0
 
     ``password`` key added.
 
 .. note::
 
     The stdin socket of the client is required to have the same zmq IDENTITY
     as the client's shell socket.
     Because of this, the ``input_request`` must be sent with the same IDENTITY
     routing prefix as the ``execute_reply`` in order for the frontend to receive
     the message.
 
 .. note::
 
    We do not explicitly try to forward the raw ``sys.stdin`` object, because in
    practice the kernel should behave like an interactive program.  When a
    program is opened on the console, the keyboard effectively takes over the
    ``stdin`` file descriptor, and it can't be used for raw reading anymore.
    Since the IPython kernel effectively behaves like a console program (albeit
    one whose "keyboard" is actually living in a separate process and
    transported over the zmq connection), raw ``stdin`` isn't expected to be
    available.
 
 .. _kernel_heartbeat:
 
 Heartbeat for kernels
 =====================
 
 Clients send ping messages on a REQ socket, which are echoed right back
 from the Kernel's REP socket. These are simple bytestrings, not full JSON messages described above.
 
 
 Custom Messages
 ===============
 
 .. versionadded:: 4.1
 
 IPython 2.0 (msgspec v4.1) adds a messaging system for developers to add their own objects with Frontend
 and Kernel-side components, and allow them to communicate with each other.
 To do this, IPython adds a notion of a ``Comm``, which exists on both sides,
 and can communicate in either direction.
 
 These messages are fully symmetrical - both the Kernel and the Frontend can send each message,
 and no messages expect a reply.
 The Kernel listens for these messages on the Shell channel,
 and the Frontend listens for them on the IOPub channel.
 
 Opening a Comm
 --------------
 
 Opening a Comm produces a ``comm_open`` message, to be sent to the other side::
 
     {
       'comm_id' : 'u-u-i-d',
       'target_name' : 'my_comm',
       'data' : {}
     }
 
 Every Comm has an ID and a target name.
 The code handling the message on the receiving side is responsible for maintaining a mapping
 of target_name keys to constructors.
 After a ``comm_open`` message has been sent,
 there should be a corresponding Comm instance on both sides.
 The ``data`` key is always a dict and can be any extra JSON information used in initialization of the comm.
 
 If the ``target_name`` key is not found on the receiving side,
 then it should immediately reply with a ``comm_close`` message to avoid an inconsistent state.
 
 Comm Messages
 -------------
 
 Comm messages are one-way communications to update comm state,
 used for synchronizing widget state, or simply requesting actions of a comm's counterpart.
 
 Essentially, each comm pair defines their own message specification implemented inside the ``data`` dict.
 
 There are no expected replies (of course, one side can send another ``comm_msg`` in reply).
 
 Message type: ``comm_msg``::
 
     {
       'comm_id' : 'u-u-i-d',
       'data' : {}
     }
 
 Tearing Down Comms
 ------------------
 
 Since comms live on both sides, when a comm is destroyed the other side must be notified.
 This is done with a ``comm_close`` message.
 
 Message type: ``comm_close``::
 
     {
       'comm_id' : 'u-u-i-d',
       'data' : {}
     }
 
 Output Side Effects
 -------------------
 
 Since comm messages can execute arbitrary user code,
 handlers should set the parent header and publish status busy / idle,
 just like an execute request.
 
 
 To Do
 =====
 
 Missing things include:
 
 * Important: finish thinking through the payload concept and API.
 
 .. include:: ../links.txt