Add the new search option `n` to the messaging protocol

IPython/zmq/ipkernel.py

@@ -1,931 +1,932 b''
 #!/usr/bin/env python
 """A simple interactive kernel that talks to a frontend over 0MQ.
 
 Things to do:
 
 * Implement `set_parent` logic. Right before doing exec, the Kernel should
   call set_parent on all the PUB objects with the message about to be executed.
 * Implement random port and security key logic.
 * Implement control messages.
 * Implement event loop and poll version.
 """
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 from __future__ import print_function
 
 # Standard library imports
 import __builtin__
 import atexit
 import sys
 import time
 import traceback
 import logging
 import uuid
 
 from datetime import datetime
 from signal import (
         signal, getsignal, default_int_handler, SIGINT, SIG_IGN
 )
 
 # System library imports
 import zmq
 from zmq.eventloop import ioloop
 from zmq.eventloop.zmqstream import ZMQStream
 
 # Local imports
 from IPython.config.configurable import Configurable
 from IPython.config.application import boolean_flag, catch_config_error
 from IPython.core.application import ProfileDir
 from IPython.core.error import StdinNotImplementedError
 from IPython.core.shellapp import (
     InteractiveShellApp, shell_flags, shell_aliases
 )
 from IPython.utils import io
 from IPython.utils import py3compat
 from IPython.utils.frame import extract_module_locals
 from IPython.utils.jsonutil import json_clean
 from IPython.utils.traitlets import (
     Any, Instance, Float, Dict, CaselessStrEnum, List, Set, Integer, Unicode
 )
 
 from entry_point import base_launch_kernel
 from kernelapp import KernelApp, kernel_flags, kernel_aliases
 from serialize import serialize_object, unpack_apply_message
 from session import Session, Message
 from zmqshell import ZMQInteractiveShell
 
 
 #-----------------------------------------------------------------------------
 # Main kernel class
 #-----------------------------------------------------------------------------
 
 class Kernel(Configurable):
 
     #---------------------------------------------------------------------------
     # Kernel interface
     #---------------------------------------------------------------------------
 
     # attribute to override with a GUI
     eventloop = Any(None)
     def _eventloop_changed(self, name, old, new):
         """schedule call to eventloop from IOLoop"""
         loop = ioloop.IOLoop.instance()
         loop.add_timeout(time.time()+0.1, self.enter_eventloop)
 
     shell = Instance('IPython.core.interactiveshell.InteractiveShellABC')
     session = Instance(Session)
     profile_dir = Instance('IPython.core.profiledir.ProfileDir')
     shell_streams = List()
     control_stream = Instance(ZMQStream)
     iopub_socket = Instance(zmq.Socket)
     stdin_socket = Instance(zmq.Socket)
     log = Instance(logging.Logger)
     
     user_module = Any()
     def _user_module_changed(self, name, old, new):
         if self.shell is not None:
             self.shell.user_module = new
     
     user_ns = Dict(default_value=None)
     def _user_ns_changed(self, name, old, new):
         if self.shell is not None:
             self.shell.user_ns = new
             self.shell.init_user_ns()
 
     # identities:
     int_id = Integer(-1)
     ident = Unicode()
 
     def _ident_default(self):
         return unicode(uuid.uuid4())
 
 
     # Private interface
     
     # Time to sleep after flushing the stdout/err buffers in each execute
     # cycle.  While this introduces a hard limit on the minimal latency of the
     # execute cycle, it helps prevent output synchronization problems for
     # clients.
     # Units are in seconds.  The minimum zmq latency on local host is probably
     # ~150 microseconds, set this to 500us for now.  We may need to increase it
     # a little if it's not enough after more interactive testing.
     _execute_sleep = Float(0.0005, config=True)
 
     # Frequency of the kernel's event loop.
     # Units are in seconds, kernel subclasses for GUI toolkits may need to
     # adapt to milliseconds.
     _poll_interval = Float(0.05, config=True)
 
     # If the shutdown was requested over the network, we leave here the
     # necessary reply message so it can be sent by our registered atexit
     # handler.  This ensures that the reply is only sent to clients truly at
     # the end of our shutdown process (which happens after the underlying
     # IPython shell's own shutdown).
     _shutdown_message = None
 
     # This is a dict of port number that the kernel is listening on. It is set
     # by record_ports and used by connect_request.
     _recorded_ports = Dict()
     
     # set of aborted msg_ids
     aborted = Set()
 
 
     def __init__(self, **kwargs):
         super(Kernel, self).__init__(**kwargs)
 
         # Initialize the InteractiveShell subclass
         self.shell = ZMQInteractiveShell.instance(config=self.config,
             profile_dir = self.profile_dir,
             user_module = self.user_module,
             user_ns     = self.user_ns,
         )
         self.shell.displayhook.session = self.session
         self.shell.displayhook.pub_socket = self.iopub_socket
         self.shell.displayhook.topic = self._topic('pyout')
         self.shell.display_pub.session = self.session
         self.shell.display_pub.pub_socket = self.iopub_socket
         self.shell.data_pub.session = self.session
         self.shell.data_pub.pub_socket = self.iopub_socket
 
         # TMP - hack while developing
         self.shell._reply_content = None
 
         # Build dict of handlers for message types
         msg_types = [ 'execute_request', 'complete_request',
                       'object_info_request', 'history_request',
                       'connect_request', 'shutdown_request',
                       'apply_request',
                     ]
         self.shell_handlers = {}
         for msg_type in msg_types:
             self.shell_handlers[msg_type] = getattr(self, msg_type)
         
         control_msg_types = msg_types + [ 'clear_request', 'abort_request' ]
         self.control_handlers = {}
         for msg_type in control_msg_types:
             self.control_handlers[msg_type] = getattr(self, msg_type)
 
     def dispatch_control(self, msg):
         """dispatch control requests"""
         idents,msg = self.session.feed_identities(msg, copy=False)
         try:
             msg = self.session.unserialize(msg, content=True, copy=False)
         except:
             self.log.error("Invalid Control Message", exc_info=True)
             return
 
         self.log.debug("Control received: %s", msg)
 
         header = msg['header']
         msg_id = header['msg_id']
         msg_type = header['msg_type']
 
         handler = self.control_handlers.get(msg_type, None)
         if handler is None:
             self.log.error("UNKNOWN CONTROL MESSAGE TYPE: %r", msg_type)
         else:
             try:
                 handler(self.control_stream, idents, msg)
             except Exception:
                 self.log.error("Exception in control handler:", exc_info=True)
     
     def dispatch_shell(self, stream, msg):
         """dispatch shell requests"""
         # flush control requests first
         if self.control_stream:
             self.control_stream.flush()
         
         idents,msg = self.session.feed_identities(msg, copy=False)
         try:
             msg = self.session.unserialize(msg, content=True, copy=False)
         except:
             self.log.error("Invalid Message", exc_info=True)
             return
 
         header = msg['header']
         msg_id = header['msg_id']
         msg_type = msg['header']['msg_type']
         
         # Print some info about this message and leave a '--->' marker, so it's
         # easier to trace visually the message chain when debugging.  Each
         # handler prints its message at the end.
         self.log.debug('\n*** MESSAGE TYPE:%s***', msg_type)
         self.log.debug('   Content: %s\n   --->\n   ', msg['content'])
 
         if msg_id in self.aborted:
             self.aborted.remove(msg_id)
             # is it safe to assume a msg_id will not be resubmitted?
             reply_type = msg_type.split('_')[0] + '_reply'
             status = {'status' : 'aborted'}
             md = {'engine' : self.ident}
             md.update(status)
             reply_msg = self.session.send(stream, reply_type, metadata=md,
                         content=status, parent=msg, ident=idents)
             return
         
         handler = self.shell_handlers.get(msg_type, None)
         if handler is None:
             self.log.error("UNKNOWN MESSAGE TYPE: %r", msg_type)
         else:
             # ensure default_int_handler during handler call
             sig = signal(SIGINT, default_int_handler)
             try:
                 handler(stream, idents, msg)
             except Exception:
                 self.log.error("Exception in message handler:", exc_info=True)
             finally:
                 signal(SIGINT, sig)
     
     def enter_eventloop(self):
         """enter eventloop"""
         self.log.info("entering eventloop")
         # restore default_int_handler
         signal(SIGINT, default_int_handler)
         while self.eventloop is not None:
             try:
                 self.eventloop(self)
             except KeyboardInterrupt:
                 # Ctrl-C shouldn't crash the kernel
                 self.log.error("KeyboardInterrupt caught in kernel")
                 continue
             else:
                 # eventloop exited cleanly, this means we should stop (right?)
                 self.eventloop = None
                 break
         self.log.info("exiting eventloop")
 
     def start(self):
         """register dispatchers for streams"""
         self.shell.exit_now = False
         if self.control_stream:
             self.control_stream.on_recv(self.dispatch_control, copy=False)
 
         def make_dispatcher(stream):
             def dispatcher(msg):
                 return self.dispatch_shell(stream, msg)
             return dispatcher
 
         for s in self.shell_streams:
             s.on_recv(make_dispatcher(s), copy=False)
     
     def do_one_iteration(self):
         """step eventloop just once"""
         if self.control_stream:
             self.control_stream.flush()
         for stream in self.shell_streams:
             # handle at most one request per iteration
             stream.flush(zmq.POLLIN, 1)
             stream.flush(zmq.POLLOUT)
 
 
     def record_ports(self, ports):
         """Record the ports that this kernel is using.
 
         The creator of the Kernel instance must call this methods if they
         want the :meth:`connect_request` method to return the port numbers.
         """
         self._recorded_ports = ports
 
     #---------------------------------------------------------------------------
     # Kernel request handlers
     #---------------------------------------------------------------------------
     
     def _make_metadata(self, other=None):
         """init metadata dict, for execute/apply_reply"""
         new_md = {
             'dependencies_met' : True,
             'engine' : self.ident,
             'started': datetime.now(),
         }
         if other:
             new_md.update(other)
         return new_md
     
     def _publish_pyin(self, code, parent, execution_count):
         """Publish the code request on the pyin stream."""
 
         self.session.send(self.iopub_socket, u'pyin',
                             {u'code':code, u'execution_count': execution_count},
                             parent=parent, ident=self._topic('pyin')
         )
     
     def _publish_status(self, status, parent=None):
         """send status (busy/idle) on IOPub"""
         self.session.send(self.iopub_socket,
                           u'status',
                           {u'execution_state': status},
                           parent=parent,
                           ident=self._topic('status'),
                           )
         
 
     def execute_request(self, stream, ident, parent):
         """handle an execute_request"""
         
         self._publish_status(u'busy', parent)
         
         try:
             content = parent[u'content']
             code = content[u'code']
             silent = content[u'silent']
             store_history = content.get(u'store_history', not silent)
         except:
             self.log.error("Got bad msg: ")
             self.log.error("%s", parent)
             return
         
         md = self._make_metadata(parent['metadata'])
 
         shell = self.shell # we'll need this a lot here
 
         # Replace raw_input. Note that is not sufficient to replace
         # raw_input in the user namespace.
         if content.get('allow_stdin', False):
             raw_input = lambda prompt='': self._raw_input(prompt, ident, parent)
         else:
             raw_input = lambda prompt='' : self._no_raw_input()
 
         if py3compat.PY3:
             __builtin__.input = raw_input
         else:
             __builtin__.raw_input = raw_input
 
         # Set the parent message of the display hook and out streams.
         shell.displayhook.set_parent(parent)
         shell.display_pub.set_parent(parent)
         shell.data_pub.set_parent(parent)
         sys.stdout.set_parent(parent)
         sys.stderr.set_parent(parent)
 
         # Re-broadcast our input for the benefit of listening clients, and
         # start computing output
         if not silent:
             self._publish_pyin(code, parent, shell.execution_count)
 
         reply_content = {}
         try:
             # FIXME: the shell calls the exception handler itself.
             shell.run_cell(code, store_history=store_history, silent=silent)
         except:
             status = u'error'
             # FIXME: this code right now isn't being used yet by default,
             # because the run_cell() call above directly fires off exception
             # reporting.  This code, therefore, is only active in the scenario
             # where runlines itself has an unhandled exception.  We need to
             # uniformize this, for all exception construction to come from a
             # single location in the codbase.
             etype, evalue, tb = sys.exc_info()
             tb_list = traceback.format_exception(etype, evalue, tb)
             reply_content.update(shell._showtraceback(etype, evalue, tb_list))
         else:
             status = u'ok'
 
         reply_content[u'status'] = status
 
         # Return the execution counter so clients can display prompts
         reply_content['execution_count'] = shell.execution_count - 1
 
         # FIXME - fish exception info out of shell, possibly left there by
         # runlines.  We'll need to clean up this logic later.
         if shell._reply_content is not None:
             reply_content.update(shell._reply_content)
             e_info = dict(engine_uuid=self.ident, engine_id=self.int_id, method='execute')
             reply_content['engine_info'] = e_info
             # reset after use
             shell._reply_content = None
 
         # At this point, we can tell whether the main code execution succeeded
         # or not.  If it did, we proceed to evaluate user_variables/expressions
         if reply_content['status'] == 'ok':
             reply_content[u'user_variables'] = \
                          shell.user_variables(content.get(u'user_variables', []))
             reply_content[u'user_expressions'] = \
                          shell.user_expressions(content.get(u'user_expressions', {}))
         else:
             # If there was an error, don't even try to compute variables or
             # expressions
             reply_content[u'user_variables'] = {}
             reply_content[u'user_expressions'] = {}
 
         # Payloads should be retrieved regardless of outcome, so we can both
         # recover partial output (that could have been generated early in a
         # block, before an error) and clear the payload system always.
         reply_content[u'payload'] = shell.payload_manager.read_payload()
         # Be agressive about clearing the payload because we don't want
         # it to sit in memory until the next execute_request comes in.
         shell.payload_manager.clear_payload()
 
         # Flush output before sending the reply.
         sys.stdout.flush()
         sys.stderr.flush()
         # FIXME: on rare occasions, the flush doesn't seem to make it to the
         # clients... This seems to mitigate the problem, but we definitely need
         # to better understand what's going on.
         if self._execute_sleep:
             time.sleep(self._execute_sleep)
 
         # Send the reply.
         reply_content = json_clean(reply_content)
         
         md['status'] = reply_content['status']
         if reply_content['status'] == 'error' and \
                         reply_content['ename'] == 'UnmetDependency':
                 md['dependencies_met'] = False
 
         reply_msg = self.session.send(stream, u'execute_reply',
                                       reply_content, parent, metadata=md,
                                       ident=ident)
         
         self.log.debug("%s", reply_msg)
 
         if not silent and reply_msg['content']['status'] == u'error':
             self._abort_queues()
 
         self._publish_status(u'idle', parent)
 
     def complete_request(self, stream, ident, parent):
         txt, matches = self._complete(parent)
         matches = {'matches' : matches,
                    'matched_text' : txt,
                    'status' : 'ok'}
         matches = json_clean(matches)
         completion_msg = self.session.send(stream, 'complete_reply',
                                            matches, parent, ident)
         self.log.debug("%s", completion_msg)
 
     def object_info_request(self, stream, ident, parent):
         content = parent['content']
         object_info = self.shell.object_inspect(content['oname'],
                         detail_level = content.get('detail_level', 0)
         )
         # Before we send this object over, we scrub it for JSON usage
         oinfo = json_clean(object_info)
         msg = self.session.send(stream, 'object_info_reply',
                                 oinfo, parent, ident)
         self.log.debug("%s", msg)
 
     def history_request(self, stream, ident, parent):
         # We need to pull these out, as passing **kwargs doesn't work with
         # unicode keys before Python 2.6.5.
         hist_access_type = parent['content']['hist_access_type']
         raw = parent['content']['raw']
         output = parent['content']['output']
         if hist_access_type == 'tail':
             n = parent['content']['n']
             hist = self.shell.history_manager.get_tail(n, raw=raw, output=output,
                                                             include_latest=True)
 
         elif hist_access_type == 'range':
             session = parent['content']['session']
             start = parent['content']['start']
             stop = parent['content']['stop']
             hist = self.shell.history_manager.get_range(session, start, stop,
                                                         raw=raw, output=output)
 
         elif hist_access_type == 'search':
+            n = parent['content']['n']
             pattern = parent['content']['pattern']
             hist = self.shell.history_manager.search(pattern, raw=raw,
-                                                     output=output) 
+                                                     output=output, n=n)
 
         else:
             hist = []
         hist = list(hist)
         content = {'history' : hist}
         content = json_clean(content)
         msg = self.session.send(stream, 'history_reply',
                                 content, parent, ident)
         self.log.debug("Sending history reply with %i entries", len(hist))
 
     def connect_request(self, stream, ident, parent):
         if self._recorded_ports is not None:
             content = self._recorded_ports.copy()
         else:
             content = {}
         msg = self.session.send(stream, 'connect_reply',
                                 content, parent, ident)
         self.log.debug("%s", msg)
 
     def shutdown_request(self, stream, ident, parent):
         self.shell.exit_now = True
         content = dict(status='ok')
         content.update(parent['content'])
         self.session.send(stream, u'shutdown_reply', content, parent, ident=ident)
         # same content, but different msg_id for broadcasting on IOPub
         self._shutdown_message = self.session.msg(u'shutdown_reply',
                                                   content, parent
         )
 
         self._at_shutdown()
         # call sys.exit after a short delay
         loop = ioloop.IOLoop.instance()
         loop.add_timeout(time.time()+0.1, loop.stop)
 
     #---------------------------------------------------------------------------
     # Engine methods
     #---------------------------------------------------------------------------
 
     def apply_request(self, stream, ident, parent):
         try:
             content = parent[u'content']
             bufs = parent[u'buffers']
             msg_id = parent['header']['msg_id']
         except:
             self.log.error("Got bad msg: %s", parent, exc_info=True)
             return
 
         self._publish_status(u'busy', parent)
         
         # Set the parent message of the display hook and out streams.
         shell = self.shell
         shell.displayhook.set_parent(parent)
         shell.display_pub.set_parent(parent)
         shell.data_pub.set_parent(parent)
         sys.stdout.set_parent(parent)
         sys.stderr.set_parent(parent)
 
         # pyin_msg = self.session.msg(u'pyin',{u'code':code}, parent=parent)
         # self.iopub_socket.send(pyin_msg)
         # self.session.send(self.iopub_socket, u'pyin', {u'code':code},parent=parent)
         md = self._make_metadata(parent['metadata'])
         try:
             working = shell.user_ns
 
             prefix = "_"+str(msg_id).replace("-","")+"_"
 
             f,args,kwargs = unpack_apply_message(bufs, working, copy=False)
 
             fname = getattr(f, '__name__', 'f')
 
             fname = prefix+"f"
             argname = prefix+"args"
             kwargname = prefix+"kwargs"
             resultname = prefix+"result"
 
             ns = { fname : f, argname : args, kwargname : kwargs , resultname : None }
             # print ns
             working.update(ns)
             code = "%s = %s(*%s,**%s)" % (resultname, fname, argname, kwargname)
             try:
                 exec code in shell.user_global_ns, shell.user_ns
                 result = working.get(resultname)
             finally:
                 for key in ns.iterkeys():
                     working.pop(key)
 
             result_buf = serialize_object(result,
                 buffer_threshold=self.session.buffer_threshold,
                 item_threshold=self.session.item_threshold,
             )
         
         except:
             # invoke IPython traceback formatting
             shell.showtraceback()
             # FIXME - fish exception info out of shell, possibly left there by
             # run_code.  We'll need to clean up this logic later.
             reply_content = {}
             if shell._reply_content is not None:
                 reply_content.update(shell._reply_content)
                 e_info = dict(engine_uuid=self.ident, engine_id=self.int_id, method='apply')
                 reply_content['engine_info'] = e_info
                 # reset after use
                 shell._reply_content = None
             
             self.session.send(self.iopub_socket, u'pyerr', reply_content, parent=parent,
                                 ident=self._topic('pyerr'))
             result_buf = []
 
             if reply_content['ename'] == 'UnmetDependency':
                 md['dependencies_met'] = False
         else:
             reply_content = {'status' : 'ok'}
 
         # put 'ok'/'error' status in header, for scheduler introspection:
         md['status'] = reply_content['status']
 
         # flush i/o
         sys.stdout.flush()
         sys.stderr.flush()
         
         reply_msg = self.session.send(stream, u'apply_reply', reply_content,
                     parent=parent, ident=ident,buffers=result_buf, metadata=md)
 
         self._publish_status(u'idle', parent)
 
     #---------------------------------------------------------------------------
     # Control messages
     #---------------------------------------------------------------------------
 
     def abort_request(self, stream, ident, parent):
         """abort a specifig msg by id"""
         msg_ids = parent['content'].get('msg_ids', None)
         if isinstance(msg_ids, basestring):
             msg_ids = [msg_ids]
         if not msg_ids:
             self.abort_queues()
         for mid in msg_ids:
             self.aborted.add(str(mid))
 
         content = dict(status='ok')
         reply_msg = self.session.send(stream, 'abort_reply', content=content,
                 parent=parent, ident=ident)
         self.log.debug("%s", reply_msg)
 
     def clear_request(self, stream, idents, parent):
         """Clear our namespace."""
         self.shell.reset(False)
         msg = self.session.send(stream, 'clear_reply', ident=idents, parent=parent,
                 content = dict(status='ok'))
 
 
     #---------------------------------------------------------------------------
     # Protected interface
     #---------------------------------------------------------------------------
 
 
     def _wrap_exception(self, method=None):
         # import here, because _wrap_exception is only used in parallel,
         # and parallel has higher min pyzmq version
         from IPython.parallel.error import wrap_exception
         e_info = dict(engine_uuid=self.ident, engine_id=self.int_id, method=method)
         content = wrap_exception(e_info)
         return content
 
     def _topic(self, topic):
         """prefixed topic for IOPub messages"""
         if self.int_id >= 0:
             base = "engine.%i" % self.int_id
         else:
             base = "kernel.%s" % self.ident
         
         return py3compat.cast_bytes("%s.%s" % (base, topic))
     
     def _abort_queues(self):
         for stream in self.shell_streams:
             if stream:
                 self._abort_queue(stream)
 
     def _abort_queue(self, stream):
         poller = zmq.Poller()
         poller.register(stream.socket, zmq.POLLIN)
         while True:
             idents,msg = self.session.recv(stream, zmq.NOBLOCK, content=True)
             if msg is None:
                 return
 
             self.log.info("Aborting:")
             self.log.info("%s", msg)
             msg_type = msg['header']['msg_type']
             reply_type = msg_type.split('_')[0] + '_reply'
 
             status = {'status' : 'aborted'}
             md = {'engine' : self.ident}
             md.update(status)
             reply_msg = self.session.send(stream, reply_type, metadata=md,
                         content=status, parent=msg, ident=idents)
             self.log.debug("%s", reply_msg)
             # We need to wait a bit for requests to come in. This can probably
             # be set shorter for true asynchronous clients.
             poller.poll(50)
 
 
     def _no_raw_input(self):
         """Raise StdinNotImplentedError if active frontend doesn't support
         stdin."""
         raise StdinNotImplementedError("raw_input was called, but this "
                                        "frontend does not support stdin.") 
         
     def _raw_input(self, prompt, ident, parent):
         # Flush output before making the request.
         sys.stderr.flush()
         sys.stdout.flush()
 
         # Send the input request.
         content = json_clean(dict(prompt=prompt))
         self.session.send(self.stdin_socket, u'input_request', content, parent,
                           ident=ident)
 
         # Await a response.
         while True:
             try:
                 ident, reply = self.session.recv(self.stdin_socket, 0)
             except Exception:
                 self.log.warn("Invalid Message:", exc_info=True)
             else:
                 break
         try:
             value = reply['content']['value']
         except:
             self.log.error("Got bad raw_input reply: ")
             self.log.error("%s", parent)
             value = ''
         if value == '\x04':
             # EOF
             raise EOFError
         return value
 
     def _complete(self, msg):
         c = msg['content']
         try:
             cpos = int(c['cursor_pos'])
         except:
             # If we don't get something that we can convert to an integer, at
             # least attempt the completion guessing the cursor is at the end of
             # the text, if there's any, and otherwise of the line
             cpos = len(c['text'])
             if cpos==0:
                 cpos = len(c['line'])
         return self.shell.complete(c['text'], c['line'], cpos)
 
     def _object_info(self, context):
         symbol, leftover = self._symbol_from_context(context)
         if symbol is not None and not leftover:
             doc = getattr(symbol, '__doc__', '')
         else:
             doc = ''
         object_info = dict(docstring = doc)
         return object_info
 
     def _symbol_from_context(self, context):
         if not context:
             return None, context
 
         base_symbol_string = context[0]
         symbol = self.shell.user_ns.get(base_symbol_string, None)
         if symbol is None:
             symbol = __builtin__.__dict__.get(base_symbol_string, None)
         if symbol is None:
             return None, context
 
         context = context[1:]
         for i, name in enumerate(context):
             new_symbol = getattr(symbol, name, None)
             if new_symbol is None:
                 return symbol, context[i:]
             else:
                 symbol = new_symbol
 
         return symbol, []
 
     def _at_shutdown(self):
         """Actions taken at shutdown by the kernel, called by python's atexit.
         """
         # io.rprint("Kernel at_shutdown") # dbg
         if self._shutdown_message is not None:
             self.session.send(self.iopub_socket, self._shutdown_message, ident=self._topic('shutdown'))
             self.log.debug("%s", self._shutdown_message)
         [ s.flush(zmq.POLLOUT) for s in self.shell_streams ]
 
 #-----------------------------------------------------------------------------
 # Aliases and Flags for the IPKernelApp
 #-----------------------------------------------------------------------------
 
 flags = dict(kernel_flags)
 flags.update(shell_flags)
 
 addflag = lambda *args: flags.update(boolean_flag(*args))
 
 flags['pylab'] = (
     {'IPKernelApp' : {'pylab' : 'auto'}},
     """Pre-load matplotlib and numpy for interactive use with
     the default matplotlib backend."""
 )
 
 aliases = dict(kernel_aliases)
 aliases.update(shell_aliases)
 
 #-----------------------------------------------------------------------------
 # The IPKernelApp class
 #-----------------------------------------------------------------------------
 
 class IPKernelApp(KernelApp, InteractiveShellApp):
     name = 'ipkernel'
 
     aliases = Dict(aliases)
     flags = Dict(flags)
     classes = [Kernel, ZMQInteractiveShell, ProfileDir, Session]
 
     @catch_config_error
     def initialize(self, argv=None):
         super(IPKernelApp, self).initialize(argv)
         self.init_path()
         self.init_shell()
         self.init_gui_pylab()
         self.init_extensions()
         self.init_code()
 
     def init_kernel(self):
         
         shell_stream = ZMQStream(self.shell_socket)
 
         kernel = Kernel(config=self.config, session=self.session,
                                 shell_streams=[shell_stream],
                                 iopub_socket=self.iopub_socket,
                                 stdin_socket=self.stdin_socket,
                                 log=self.log,
                                 profile_dir=self.profile_dir,
         )
         self.kernel = kernel
         kernel.record_ports(self.ports)
         shell = kernel.shell
 
     def init_gui_pylab(self):
         """Enable GUI event loop integration, taking pylab into account."""
 
         # Provide a wrapper for :meth:`InteractiveShellApp.init_gui_pylab`
         # to ensure that any exception is printed straight to stderr.
         # Normally _showtraceback associates the reply with an execution,
         # which means frontends will never draw it, as this exception
         # is not associated with any execute request.
 
         shell = self.shell
         _showtraceback = shell._showtraceback
         try:
             # replace pyerr-sending traceback with stderr
             def print_tb(etype, evalue, stb):
                 print ("GUI event loop or pylab initialization failed",
                        file=io.stderr)
                 print (shell.InteractiveTB.stb2text(stb), file=io.stderr)
             shell._showtraceback = print_tb
             InteractiveShellApp.init_gui_pylab(self)
         finally:
             shell._showtraceback = _showtraceback
 
     def init_shell(self):
         self.shell = self.kernel.shell
         self.shell.configurables.append(self)
 
 
 #-----------------------------------------------------------------------------
 # Kernel main and launch functions
 #-----------------------------------------------------------------------------
 
 def launch_kernel(*args, **kwargs):
     """Launches a localhost IPython kernel, binding to the specified ports.
 
     This function simply calls entry_point.base_launch_kernel with the right
     first command to start an ipkernel.  See base_launch_kernel for arguments.
 
     Returns
     -------
     A tuple of form:
         (kernel_process, shell_port, iopub_port, stdin_port, hb_port)
     where kernel_process is a Popen object and the ports are integers.
     """
     return base_launch_kernel('from IPython.zmq.ipkernel import main; main()',
                               *args, **kwargs)
 
 
 def embed_kernel(module=None, local_ns=None, **kwargs):
     """Embed and start an IPython kernel in a given scope.
     
     Parameters
     ----------
     module : ModuleType, optional
         The module to load into IPython globals (default: caller)
     local_ns : dict, optional
         The namespace to load into IPython user namespace (default: caller)
     
     kwargs : various, optional
         Further keyword args are relayed to the KernelApp constructor,
         allowing configuration of the Kernel.  Will only have an effect
         on the first embed_kernel call for a given process.
     
     """
     # get the app if it exists, or set it up if it doesn't
     if IPKernelApp.initialized():
         app = IPKernelApp.instance()
     else:
         app = IPKernelApp.instance(**kwargs)
         app.initialize([])
         # Undo unnecessary sys module mangling from init_sys_modules.
         # This would not be necessary if we could prevent it
         # in the first place by using a different InteractiveShell
         # subclass, as in the regular embed case.
         main = app.kernel.shell._orig_sys_modules_main_mod
         if main is not None:
             sys.modules[app.kernel.shell._orig_sys_modules_main_name] = main
 
     # load the calling scope if not given
     (caller_module, caller_locals) = extract_module_locals(1)
     if module is None:
         module = caller_module
     if local_ns is None:
         local_ns = caller_locals
     
     app.kernel.user_module = module
     app.kernel.user_ns = local_ns
     app.shell.set_completer_frame()
     app.start()
 
 def main():
     """Run an IPKernel as an application"""
     app = IPKernelApp.instance()
     app.initialize()
     app.start()
 
 
 if __name__ == '__main__':
     main()

docs/source/development/messaging.txt

@@ -1,1002 +1,1002 b''
 .. _messaging:
 
 ======================
  Messaging in IPython
 ======================
 
 
 Introduction
 ============
 
 This document explains the basic communications design and messaging
 specification for how the various IPython objects interact over a network
 transport.  The current implementation uses the ZeroMQ_ library for messaging
 within and between hosts.
 
 .. Note::
 
    This document should be considered the authoritative description of the
    IPython messaging protocol, and all developers are strongly encouraged to
    keep it updated as the implementation evolves, so that we have a single
    common reference for all protocol details.
    
 The basic design is explained in the following diagram:
 
 .. image:: figs/frontend-kernel.png
    :width: 450px
    :alt: IPython kernel/frontend messaging architecture.
    :align: center
    :target: ../_images/frontend-kernel.png
 
 A single kernel can be simultaneously connected to one or more frontends.  The
 kernel has three sockets that serve the following functions:
 
 1. stdin: this ROUTER socket is connected to all frontends, and it allows
    the kernel to request input from the active frontend when :func:`raw_input` is called.
    The frontend that executed the code has a DEALER socket that acts as a 'virtual keyboard'
    for the kernel while this communication is happening (illustrated in the
    figure by the black outline around the central keyboard).  In practice,
    frontends may display such kernel requests using a special input widget or
    otherwise indicating that the user is to type input for the kernel instead
    of normal commands in the frontend.
 
 2. Shell: this single ROUTER socket allows multiple incoming connections from
    frontends, and this is the socket where requests for code execution, object
    information, prompts, etc. are made to the kernel by any frontend.  The
    communication on this socket is a sequence of request/reply actions from
    each frontend and the kernel.
 
 3. IOPub: this socket is the 'broadcast channel' where the kernel publishes all
    side effects (stdout, stderr, etc.) as well as the requests coming from any
    client over the shell socket and its own requests on the stdin socket.  There
    are a number of actions in Python which generate side effects: :func:`print`
    writes to ``sys.stdout``, errors generate tracebacks, etc.  Additionally, in
    a multi-client scenario, we want all frontends to be able to know what each
    other has sent to the kernel (this can be useful in collaborative scenarios,
    for example).  This socket allows both side effects and the information
    about communications taking place with one client over the shell channel
    to be made available to all clients in a uniform manner.
 
    All messages are tagged with enough information (details below) for clients
    to know which messages come from their own interaction with the kernel and
    which ones are from other clients, so they can display each type
    appropriately.
 
 The actual format of the messages allowed on each of these channels is
 specified below.  Messages are dicts of dicts with string keys and values that
 are reasonably representable in JSON.  Our current implementation uses JSON
 explicitly as its message format, but this shouldn't be considered a permanent
 feature.  As we've discovered that JSON has non-trivial performance issues due
 to excessive copying, we may in the future move to a pure pickle-based raw
 message format.  However, it should be possible to easily convert from the raw
 objects to JSON, since we may have non-python clients (e.g. a web frontend).
 As long as it's easy to make a JSON version of the objects that is a faithful
 representation of all the data, we can communicate with such clients.
 
 .. Note::
 
    Not all of these have yet been fully fleshed out, but the key ones are, see
    kernel and frontend files for actual implementation details.
 
 General Message Format
 ======================
 
 A message is defined by the following four-dictionary structure::
 
     {
       # The message header contains a pair of unique identifiers for the
       # originating session and the actual message id, in addition to the
       # username for the process that generated the message.  This is useful in
       # collaborative settings where multiple users may be interacting with the
       # same kernel simultaneously, so that frontends can label the various
       # messages in a meaningful way.
       'header' : {
                     'msg_id' : uuid,
                     'username' : str,
                     'session' : uuid
                     # All recognized message type strings are listed below.
                     'msg_type' : str,
          },
 
       # In a chain of messages, the header from the parent is copied so that
       # clients can track where messages come from.
       'parent_header' : dict,
 
       # The actual content of the message must be a dict, whose structure
       # depends on the message type.
       'content' : dict,
       
       # Any metadata associated with the message.
       'metadata' : dict,
     }
 
    
 Python functional API
 =====================
 
 As messages are dicts, they map naturally to a ``func(**kw)`` call form.  We
 should develop, at a few key points, functional forms of all the requests that
 take arguments in this manner and automatically construct the necessary dict
 for sending.
 
 In addition, the Python implementation of the message specification extends
 messages upon deserialization to the following form for convenience::
 
     {
       'header' : dict,
       # The msg's unique identifier and type are always stored in the header,
       # but the Python implementation copies them to the top level.
       'msg_id' : uuid,
       'msg_type' : str,
       'parent_header' : dict,
       'content' : dict,
       'metadata' : dict,
     }
 
 All messages sent to or received by any IPython process should have this
 extended structure.
 
 
 Messages on the shell ROUTER/DEALER sockets
 ===========================================
 
 .. _execute:
 
 Execute
 -------
 
 This message type is used by frontends to ask the kernel to execute code on
 behalf of the user, in a namespace reserved to the user's variables (and thus
 separate from the kernel's own internal code and variables).
 
 Message type: ``execute_request``::
 
     content = {
         # Source code to be executed by the kernel, one or more lines.
     'code' : str,
 
     # A boolean flag which, if True, signals the kernel to execute
     # this code as quietly as possible.  This means that the kernel
     # will compile the code with 'exec' instead of 'single' (so
     # sys.displayhook will not fire), forces store_history to be False, 
     # and will *not*:
     #   - broadcast exceptions on the PUB socket
     #   - do any logging
     #
     # The default is False.
     'silent' : bool,
 
     # A boolean flag which, if True, signals the kernel to populate history
     # The default is True if silent is False.  If silent is True, store_history
     # is forced to be False.
     'store_history' : bool,
 
     # A list of variable names from the user's namespace to be retrieved.  What
     # returns is a JSON string of the variable's repr(), not a python object.
     'user_variables' : list,
 
     # Similarly, a dict mapping names to expressions to be evaluated in the
     # user's dict.
     'user_expressions' : dict,
 
     # Some frontends (e.g. the Notebook) do not support stdin requests. If
     # raw_input is called from code executed from such a frontend, a
     # StdinNotImplementedError will be raised.
     'allow_stdin' : True,
 
     }
 
 The ``code`` field contains a single string (possibly multiline).  The kernel
 is responsible for splitting this into one or more independent execution blocks
 and deciding whether to compile these in 'single' or 'exec' mode (see below for
 detailed execution semantics).
 
 The ``user_`` fields deserve a detailed explanation.  In the past, IPython had
 the notion of a prompt string that allowed arbitrary code to be evaluated, and
 this was put to good use by many in creating prompts that displayed system
 status, path information, and even more esoteric uses like remote instrument
 status aqcuired over the network.  But now that IPython has a clean separation
 between the kernel and the clients, the kernel has no prompt knowledge; prompts
 are a frontend-side feature, and it should be even possible for different
 frontends to display different prompts while interacting with the same kernel.
 
 The kernel now provides the ability to retrieve data from the user's namespace
 after the execution of the main ``code``, thanks to two fields in the
 ``execute_request`` message:
 
 - ``user_variables``: If only variables from the user's namespace are needed, a
   list of variable names can be passed and a dict with these names as keys and
   their :func:`repr()` as values will be returned.
 
 - ``user_expressions``: For more complex expressions that require function
   evaluations, a dict can be provided with string keys and arbitrary python
   expressions as values.  The return message will contain also a dict with the
   same keys and the :func:`repr()` of the evaluated expressions as value.
 
 With this information, frontends can display any status information they wish
 in the form that best suits each frontend (a status line, a popup, inline for a
 terminal, etc).
 
 .. Note::
 
    In order to obtain the current execution counter for the purposes of
    displaying input prompts, frontends simply make an execution request with an
    empty code string and ``silent=True``.
 
 Execution semantics
 ~~~~~~~~~~~~~~~~~~~
 
 When the silent flag is false, the execution of use code consists of the
 following phases (in silent mode, only the ``code`` field is executed):
 
 1. Run the ``pre_runcode_hook``.
 
 2. Execute the ``code`` field, see below for details.
 
 3. If #2 succeeds, compute ``user_variables`` and ``user_expressions`` are
    computed.  This ensures that any error in the latter don't harm the main
    code execution.
 
 4. Call any method registered with :meth:`register_post_execute`.
 
 .. warning::
 
    The API for running code before/after the main code block is likely to
    change soon.  Both the ``pre_runcode_hook`` and the
    :meth:`register_post_execute` are susceptible to modification, as we find a
    consistent model for both.
 
 To understand how the ``code`` field is executed, one must know that Python
 code can be compiled in one of three modes (controlled by the ``mode`` argument
 to the :func:`compile` builtin):
 
 *single*
   Valid for a single interactive statement (though the source can contain
   multiple lines, such as a for loop).  When compiled in this mode, the
   generated bytecode contains special instructions that trigger the calling of
   :func:`sys.displayhook` for any expression in the block that returns a value.
   This means that a single statement can actually produce multiple calls to
   :func:`sys.displayhook`, if for example it contains a loop where each
   iteration computes an unassigned expression would generate 10 calls::
 
       for i in range(10):
           i**2
 
 *exec*
   An arbitrary amount of source code, this is how modules are compiled.
   :func:`sys.displayhook` is *never* implicitly called.
 
 *eval*
   A single expression that returns a value.  :func:`sys.displayhook` is *never*
   implicitly called.
 
 
 The ``code`` field is split into individual blocks each of which is valid for
 execution in 'single' mode, and then:
 
 - If there is only a single block: it is executed in 'single' mode.
 
 - If there is more than one block:
 
   * if the last one is a single line long, run all but the last in 'exec' mode
     and the very last one in 'single' mode.  This makes it easy to type simple
     expressions at the end to see computed values.
 
   * if the last one is no more than two lines long, run all but the last in
     'exec' mode and the very last one in 'single' mode.  This makes it easy to
     type simple expressions at the end to see computed values.  - otherwise
     (last one is also multiline), run all in 'exec' mode
 
   * otherwise (last one is also multiline), run all in 'exec' mode as a single
     unit.
 
 Any error in retrieving the ``user_variables`` or evaluating the
 ``user_expressions`` will result in a simple error message in the return fields
 of the form::
 
    [ERROR] ExceptionType: Exception message
 
 The user can simply send the same variable name or expression for evaluation to
 see a regular traceback.
 
 Errors in any registered post_execute functions are also reported similarly,
 and the failing function is removed from the post_execution set so that it does
 not continue triggering failures.
 
 Upon completion of the execution request, the kernel *always* sends a reply,
 with a status code indicating what happened and additional data depending on
 the outcome.  See :ref:`below <execution_results>` for the possible return
 codes and associated data.
 
 
 Execution counter (old prompt number)
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 The kernel has a single, monotonically increasing counter of all execution
 requests that are made with ``store_history=True``.  This counter is used to populate
 the ``In[n]``, ``Out[n]`` and ``_n`` variables, so clients will likely want to
 display it in some form to the user, which will typically (but not necessarily)
 be done in the prompts.  The value of this counter will be returned as the
 ``execution_count`` field of all ``execute_reply`` messages.
 
 .. _execution_results:
 
 Execution results
 ~~~~~~~~~~~~~~~~~
     
 Message type: ``execute_reply``::
 
     content = {
       # One of: 'ok' OR 'error' OR 'abort'
       'status' : str,
 
       # The global kernel counter that increases by one with each request that 
       # stores history.  This will typically be used by clients to display
       # prompt numbers to the user.  If the request did not store history, this will
       # be the current value of the counter in the kernel.
       'execution_count' : int,
     }
 
 When status is 'ok', the following extra fields are present::
 
     {
       # 'payload' will be a list of payload dicts.
       # Each execution payload is a dict with string keys that may have been
       # produced by the code being executed.  It is retrieved by the kernel at
       # the end of the execution and sent back to the front end, which can take
       # action on it as needed.  See main text for further details.
       'payload' : list(dict),
 
       # Results for the user_variables and user_expressions.
       'user_variables' : dict,
       'user_expressions' : dict,
     }
 
 .. admonition:: Execution payloads
     
    The notion of an 'execution payload' is different from a return value of a
    given set of code, which normally is just displayed on the pyout stream
    through the PUB socket.  The idea of a payload is to allow special types of
    code, typically magics, to populate a data container in the IPython kernel
    that will be shipped back to the caller via this channel.  The kernel
    has an API for this in the PayloadManager::
 
        ip.payload_manager.write_payload(payload_dict)
 
    which appends a dictionary to the list of payloads.
 
    
 When status is 'error', the following extra fields are present::
 
     {
       'ename' : str,   # Exception name, as a string
       'evalue' : str,  # Exception value, as a string
 
       # The traceback will contain a list of frames, represented each as a
       # string.  For now we'll stick to the existing design of ultraTB, which
       # controls exception level of detail statefully.  But eventually we'll
       # want to grow into a model where more information is collected and
       # packed into the traceback object, with clients deciding how little or
       # how much of it to unpack.  But for now, let's start with a simple list
       # of strings, since that requires only minimal changes to ultratb as
       # written.
       'traceback' : list,
     }
 
 
 When status is 'abort', there are for now no additional data fields.  This
 happens when the kernel was interrupted by a signal.
 
 Kernel attribute access
 -----------------------
 
 .. warning::
 
    This part of the messaging spec is not actually implemented in the kernel
    yet.
  
 While this protocol does not specify full RPC access to arbitrary methods of
 the kernel object, the kernel does allow read (and in some cases write) access
 to certain attributes.
 
 The policy for which attributes can be read is: any attribute of the kernel, or
 its sub-objects, that belongs to a :class:`Configurable` object and has been
 declared at the class-level with Traits validation, is in principle accessible
 as long as its name does not begin with a leading underscore.  The attribute
 itself will have metadata indicating whether it allows remote read and/or write
 access.  The message spec follows for attribute read and write requests.
 
 Message type: ``getattr_request``::
 
     content = {
         # The (possibly dotted) name of the attribute
 	'name' : str,
     }
 
 When a ``getattr_request`` fails, there are two possible error types:
 
 - AttributeError: this type of error was raised when trying to access the
   given name by the kernel itself.  This means that the attribute likely
   doesn't exist.
 
 - AccessError: the attribute exists but its value is not readable remotely.
 
 
 Message type: ``getattr_reply``::
 
     content = {
         # One of ['ok', 'AttributeError', 'AccessError'].
         'status' : str,
 	# If status is 'ok', a JSON object.
 	'value' : object,
     }
 
 Message type: ``setattr_request``::
 
     content = {
         # The (possibly dotted) name of the attribute
 	'name' : str,
 	
 	# A JSON-encoded object, that will be validated by the Traits
 	# information in the kernel
 	'value' : object,
     }
 
 When a ``setattr_request`` fails, there are also two possible error types with
 similar meanings  as those of the ``getattr_request`` case, but for writing.
     
 Message type: ``setattr_reply``::
 
     content = {
         # One of ['ok', 'AttributeError', 'AccessError'].
         'status' : str,
     }
 
 
     
 Object information
 ------------------
 
 One of IPython's most used capabilities is the introspection of Python objects
 in the user's namespace, typically invoked via the ``?`` and ``??`` characters
 (which in reality are shorthands for the ``%pinfo`` magic).  This is used often
 enough that it warrants an explicit message type, especially because frontends
 may want to get object information in response to user keystrokes (like Tab or
 F1) besides from the user explicitly typing code like ``x??``.
 
 Message type: ``object_info_request``::
 
     content = {
         # The (possibly dotted) name of the object to be searched in all
 	# relevant namespaces
         'name' : str,
 
     	# The level of detail desired.  The default (0) is equivalent to typing
 	# 'x?' at the prompt, 1 is equivalent to 'x??'.
 	'detail_level' : int,
     }
 
 The returned information will be a dictionary with keys very similar to the
 field names that IPython prints at the terminal.
     
 Message type: ``object_info_reply``::
 
     content = {
     # The name the object was requested under
     'name' : str,
     
     # Boolean flag indicating whether the named object was found or not.  If
     # it's false, all other fields will be empty.
     'found' : bool,
     
     # Flags for magics and system aliases
     'ismagic' : bool,
     'isalias' : bool,
 
     # The name of the namespace where the object was found ('builtin',
     # 'magics', 'alias', 'interactive', etc.)
     'namespace' : str,
 
     # The type name will be type.__name__ for normal Python objects, but it
     # can also be a string like 'Magic function' or 'System alias'
     'type_name' : str,
 
     # The string form of the object, possibly truncated for length if 
     # detail_level is 0
     'string_form' : str,
 
     # For objects with a __class__ attribute this will be set
     'base_class' : str,
 
     # For objects with a __len__ attribute this will be set
     'length' : int,
 
     # If the object is a function, class or method whose file we can find,
     # we give its full path
     'file' : str,
 
     # For pure Python callable objects, we can reconstruct the object
     # definition line which provides its call signature.  For convenience this
     # is returned as a single 'definition' field, but below the raw parts that
     # compose it are also returned as the argspec field.
     'definition' : str,
 
     # The individual parts that together form the definition string.  Clients
     # with rich display capabilities may use this to provide a richer and more
     # precise representation of the definition line (e.g. by highlighting
     # arguments based on the user's cursor position).  For non-callable
     # objects, this field is empty.
     'argspec' : { # The names of all the arguments
                   args : list,
 		  # The name of the varargs (*args), if any
                   varargs : str,
 		  # The name of the varkw (**kw), if any
 		  varkw : str,
 		  # The values (as strings) of all default arguments.  Note
 		  # that these must be matched *in reverse* with the 'args'
 		  # list above, since the first positional args have no default
 		  # value at all.
 		  defaults : list,
 		  },
 
     # For instances, provide the constructor signature (the definition of
     # the __init__ method):
     'init_definition' : str,
     
     # Docstrings: for any object (function, method, module, package) with a
     # docstring, we show it.  But in addition, we may provide additional
     # docstrings.  For example, for instances we will show the constructor
     # and class docstrings as well, if available.
     'docstring' : str,
 
     # For instances, provide the constructor and class docstrings
     'init_docstring' : str,
     'class_docstring' : str,
     
     # If it's a callable object whose call method has a separate docstring and
     # definition line:
     'call_def' : str,
     'call_docstring' : str,
     
     # If detail_level was 1, we also try to find the source code that
     # defines the object, if possible.  The string 'None' will indicate
     # that no source was found.
     'source' : str,
     }
 
     
 Complete
 --------
 
 Message type: ``complete_request``::
 
     content = {
         # The text to be completed, such as 'a.is'
     'text' : str,
 
     # The full line, such as 'print a.is'.  This allows completers to
     # make decisions that may require information about more than just the
     # current word.
     'line' : str,
 
     # The entire block of text where the line is.  This may be useful in the
     # case of multiline completions where more context may be needed.  Note: if
     # in practice this field proves unnecessary, remove it to lighten the
     # messages.
     
     'block' : str,
 
     # The position of the cursor where the user hit 'TAB' on the line.
     'cursor_pos' : int,
     }
 
 Message type: ``complete_reply``::
 
     content = {
         # The list of all matches to the completion request, such as
     # ['a.isalnum', 'a.isalpha'] for the above example.
     'matches' : list
     }
 
     
 History
 -------
 
 For clients to explicitly request history from a kernel.  The kernel has all
 the actual execution history stored in a single location, so clients can
 request it from the kernel when needed.
 
 Message type: ``history_request``::
 
     content = {
     
       # If True, also return output history in the resulting dict.
       'output' : bool,
 
       # If True, return the raw input history, else the transformed input.
       'raw' : bool,
 
       # So far, this can be 'range', 'tail' or 'search'.
       'hist_access_type' : str,
       
       # If hist_access_type is 'range', get a range of input cells. session can
       # be a positive session number, or a negative number to count back from
       # the current session.
       'session' : int,
       # start and stop are line numbers within that session.
       'start' : int,
       'stop' : int,
       
-      # If hist_access_type is 'tail', get the last n cells.
+      # If hist_access_type is 'tail' or 'search', get the last n cells.
       'n' : int,
       
       # If hist_access_type is 'search', get cells matching the specified glob
       # pattern (with * and ? as wildcards).
       'pattern' : str,
       
     }
 
 Message type: ``history_reply``::
 
     content = {
       # A list of 3 tuples, either:
       # (session, line_number, input) or
       # (session, line_number, (input, output)),
       # depending on whether output was False or True, respectively.
       'history' : list,
     }
 
 
 Connect
 -------
 
 When a client connects to the request/reply socket of the kernel, it can issue
 a connect request to get basic information about the kernel, such as the ports
 the other ZeroMQ sockets are listening on. This allows clients to only have
 to know about a single port (the shell channel) to connect to a kernel.
 
 Message type: ``connect_request``::
 
     content = {
     }
 
 Message type: ``connect_reply``::
 
     content = {
         'shell_port' : int  # The port the shell ROUTER socket is listening on.
         'iopub_port' : int   # The port the PUB socket is listening on.
         'stdin_port' : int   # The port the stdin ROUTER socket is listening on.
         'hb_port' : int    # The port the heartbeat socket is listening on.
     }
 
 
 
 Kernel shutdown
 ---------------
 
 The clients can request the kernel to shut itself down; this is used in
 multiple cases:
 
 - when the user chooses to close the client application via a menu or window
   control.
 - when the user types 'exit' or 'quit' (or their uppercase magic equivalents).
 - when the user chooses a GUI method (like the 'Ctrl-C' shortcut in the
   IPythonQt client) to force a kernel restart to get a clean kernel without
   losing client-side state like history or inlined figures.
 
 The client sends a shutdown request to the kernel, and once it receives the
 reply message (which is otherwise empty), it can assume that the kernel has
 completed shutdown safely.
 
 Upon their own shutdown, client applications will typically execute a last
 minute sanity check and forcefully terminate any kernel that is still alive, to
 avoid leaving stray processes in the user's machine.
 
 For both shutdown request and reply, there is no actual content that needs to
 be sent, so the content dict is empty.
 
 Message type: ``shutdown_request``::
 
     content = {
         'restart' : bool # whether the shutdown is final, or precedes a restart
     }
 
 Message type: ``shutdown_reply``::
 
     content = {
         'restart' : bool # whether the shutdown is final, or precedes a restart
     }
 
 .. Note::
 
    When the clients detect a dead kernel thanks to inactivity on the heartbeat
    socket, they simply send a forceful process termination signal, since a dead
    process is unlikely to respond in any useful way to messages.
     
 
 Messages on the PUB/SUB socket
 ==============================
 
 Streams (stdout,  stderr, etc)
 ------------------------------
 
 Message type: ``stream``::
 
     content = {
         # The name of the stream is one of 'stdin', 'stdout', 'stderr'
         'name' : str,
     
         # The data is an arbitrary string to be written to that stream
         'data' : str,
     }
 
 When a kernel receives a raw_input call, it should also broadcast it on the pub
 socket with the names 'stdin' and 'stdin_reply'.  This will allow other clients
 to monitor/display kernel interactions and possibly replay them to their user
 or otherwise expose them.
 
 Display Data
 ------------
 
 This type of message is used to bring back data that should be diplayed (text,
 html, svg, etc.) in the frontends. This data is published to all frontends.
 Each message can have multiple representations of the data; it is up to the
 frontend to decide which to use and how. A single message should contain all
 possible representations of the same information. Each representation should
 be a JSON'able data structure, and should be a valid MIME type.
 
 Some questions remain about this design:
 
 * Do we use this message type for pyout/displayhook? Probably not, because
   the displayhook also has to handle the Out prompt display. On the other hand
   we could put that information into the metadata secion.
 
 Message type: ``display_data``::
 
     content = {
 
         # Who create the data
         'source' : str,
 
         # The data dict contains key/value pairs, where the kids are MIME
         # types and the values are the raw data of the representation in that
         # format. The data dict must minimally contain the ``text/plain``
         # MIME type which is used as a backup representation.
         'data' : dict,
 
         # Any metadata that describes the data
         'metadata' : dict
     }
 
 
 Raw Data Publication
 --------------------
 
 ``display_data`` lets you publish *representations* of data, such as images and html.
 This ``data_pub`` message lets you publish *actual raw data*, sent via message buffers.
 
 data_pub messages are constructed via the :func:`IPython.lib.datapub.publish_data` function:
 
 .. sourcecode:: python
 
     from IPython.zmq.datapub import publish_data
     ns = dict(x=my_array)
     publish_data(ns)
 
 
 Message type: ``data_pub``::
 
     content = {
         # the keys of the data dict, after it has been unserialized
         keys = ['a', 'b']
     }
     # the namespace dict will be serialized in the message buffers,
     # which will have a length of at least one
     buffers = ['pdict', ...]
 
 
 The interpretation of a sequence of data_pub messages for a given parent request should be
 to update a single namespace with subsequent results.
 
 .. note::
 
     No frontends directly handle data_pub messages at this time.
     It is currently only used by the client/engines in :mod:`IPython.parallel`,
     where engines may publish *data* to the Client,
     of which the Client can then publish *representations* via ``display_data``
     to various frontends.
 
 Python inputs
 -------------
 
 These messages are the re-broadcast of the ``execute_request``.
 
 Message type: ``pyin``::
 
     content = {
         'code' : str,  # Source code to be executed, one or more lines
 
         # The counter for this execution is also provided so that clients can
         # display it, since IPython automatically creates variables called _iN
         # (for input prompt In[N]).
         'execution_count' : int
     }
 
 Python outputs
 --------------
 
 When Python produces output from code that has been compiled in with the
 'single' flag to :func:`compile`, any expression that produces a value (such as
 ``1+1``) is passed to ``sys.displayhook``, which is a callable that can do with
 this value whatever it wants.  The default behavior of ``sys.displayhook`` in
 the Python interactive prompt is to print to ``sys.stdout`` the :func:`repr` of
 the value as long as it is not ``None`` (which isn't printed at all).  In our
 case, the kernel instantiates as ``sys.displayhook`` an object which has
 similar behavior, but which instead of printing to stdout, broadcasts these
 values as ``pyout`` messages for clients to display appropriately.
 
 IPython's displayhook can handle multiple simultaneous formats depending on its
 configuration. The default pretty-printed repr text is always given with the
 ``data`` entry in this message. Any other formats are provided in the
 ``extra_formats`` list. Frontends are free to display any or all of these
 according to its capabilities. ``extra_formats`` list contains 3-tuples of an ID
 string, a type string, and the data. The ID is unique to the formatter
 implementation that created the data. Frontends will typically ignore the ID
 unless if it has requested a particular formatter. The type string tells the
 frontend how to interpret the data. It is often, but not always a MIME type.
 Frontends should ignore types that it does not understand. The data itself is
 any JSON object and depends on the format. It is often, but not always a string.
 
 Message type: ``pyout``::
 
     content = {
 
         # The counter for this execution is also provided so that clients can
         # display it, since IPython automatically creates variables called _N
         # (for prompt N).
         'execution_count' : int,
         
         # The data dict contains key/value pairs, where the kids are MIME
         # types and the values are the raw data of the representation in that
         # format. The data dict must minimally contain the ``text/plain``
         # MIME type which is used as a backup representation.
         'data' : dict,
 
     }
     
 Python errors
 -------------
 
 When an error occurs during code execution
 
 Message type: ``pyerr``::
 
     content = {
        # Similar content to the execute_reply messages for the 'error' case,
        # except the 'status' field is omitted.
     }
 
 Kernel status
 -------------
 
 This message type is used by frontends to monitor the status of the kernel.
 
 Message type: ``status``::
 
     content = {
         # When the kernel starts to execute code, it will enter the 'busy'
         # state and when it finishes, it will enter the 'idle' state.
         execution_state : ('busy', 'idle')
     }
 
 Kernel crashes
 --------------
 
 When the kernel has an unexpected exception, caught by the last-resort
 sys.excepthook, we should broadcast the crash handler's output before exiting.
 This will allow clients to notice that a kernel died, inform the user and
 propose further actions.
 
 Message type: ``crash``::
 
     content = {
        # Similarly to the 'error' case for execute_reply messages, this will
        # contain ename, etype and traceback fields.
 
        # An additional field with supplementary information such as where to
        # send the crash message
        'info' : str,
     }
 
 
 Future ideas
 ------------
     
 Other potential message types, currently unimplemented, listed below as ideas.
     
 Message type: ``file``::
 
     content = {
     'path' : 'cool.jpg',
     'mimetype' : str,
     'data' : str,
     }
 
 
 Messages on the stdin ROUTER/DEALER sockets
 ===========================================
 
 This is a socket where the request/reply pattern goes in the opposite direction:
 from the kernel to a *single* frontend, and its purpose is to allow
 ``raw_input`` and similar operations that read from ``sys.stdin`` on the kernel
 to be fulfilled by the client. The request should be made to the frontend that
 made the execution request that prompted ``raw_input`` to be called. For now we
 will keep these messages as simple as possible, since they only mean to convey
 the ``raw_input(prompt)`` call.
 
 Message type: ``input_request``::
 
     content = { 'prompt' : str }
 
 Message type: ``input_reply``::
 
     content = { 'value' : str }
     
 .. Note::
 
    We do not explicitly try to forward the raw ``sys.stdin`` object, because in
    practice the kernel should behave like an interactive program.  When a
    program is opened on the console, the keyboard effectively takes over the
    ``stdin`` file descriptor, and it can't be used for raw reading anymore.
    Since the IPython kernel effectively behaves like a console program (albeit
    one whose "keyboard" is actually living in a separate process and
    transported over the zmq connection), raw ``stdin`` isn't expected to be
    available.
 
    
 Heartbeat for kernels
 =====================
 
 Initially we had considered using messages like those above over ZMQ for a
 kernel 'heartbeat' (a way to detect quickly and reliably whether a kernel is
 alive at all, even if it may be busy executing user code).  But this has the
 problem that if the kernel is locked inside extension code, it wouldn't execute
 the python heartbeat code.  But it turns out that we can implement a basic
 heartbeat with pure ZMQ, without using any Python messaging at all.
 
 The monitor sends out a single zmq message (right now, it is a str of the
 monitor's lifetime in seconds), and gets the same message right back, prefixed
 with the zmq identity of the DEALER socket in the heartbeat process. This can be
 a uuid, or even a full message, but there doesn't seem to be a need for packing
 up a message when the sender and receiver are the exact same Python object.
 
 The model is this::
 
     monitor.send(str(self.lifetime)) # '1.2345678910'
 
 and the monitor receives some number of messages of the form::
 
     ['uuid-abcd-dead-beef', '1.2345678910']
 
 where the first part is the zmq.IDENTITY of the heart's DEALER on the engine, and
 the rest is the message sent by the monitor.  No Python code ever has any
 access to the message between the monitor's send, and the monitor's recv.
 
 
 ToDo
 ====
 
 Missing things include:
 
 * Important: finish thinking through the payload concept and API.
 
 * Important: ensure that we have a good solution for magics like %edit.  It's
   likely that with the payload concept we can build a full solution, but not
   100% clear yet.
 
 * Finishing the details of the heartbeat protocol.
 
 * Signal handling: specify what kind of information kernel should broadcast (or
   not) when it receives signals.
 
 .. include:: ../links.rst