Rework messaging to better conform to our spec.

Removed all explicit prompt handling, now only the execution_counter
field is given.  Added support for the user variables and expressions,
and for the silent field of the execution request.x

IPython/zmq/tests/test_message_spec.py

@@ -0,0 +1,40 b''
+"""Test suite for our zeromq-based messaging specification.
+"""
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2010  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING.txt, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+import sys
+import time
+
+import nose.tools as nt
+
+from ..blockingkernelmanager import BlockingKernelManager
+
+from IPython.utils import io
+
+def setup():
+    global KM
+    KM = BlockingKernelManager()
+
+    KM.start_kernel()
+    KM.start_channels()
+    # Give the kernel a chance to come up.
+    time.sleep(1)
+
+def teardown():
+    io.rprint('Entering teardown...')  # dbg
+    io.rprint('Stopping channels and kernel...')  # dbg
+    KM.stop_channels()
+    KM.kill_kernel()
+
+
+# Actual tests
+
+def test_execute():
+    KM.xreq_channel.execute(code='x=1')
+    KM.xreq_channel.execute(code='print 1')
+    

IPython/core/interactiveshell.py

@@ -1704,6 +1704,45 b' class InteractiveShell(Configurable, Magic):'
         self.prefilter = self.prefilter_manager.prefilter_lines
 
     #-------------------------------------------------------------------------
+    # Things related to extracting values/expressions from kernel and user_ns
+    #-------------------------------------------------------------------------
+
+    def _simple_error(self):
+        etype, value = sys.exc_info()[:2]
+        return u'[ERROR] {e.__name__}: {v}'.format(e=etype, v=value)
+
+    def get_user_variables(self, names):
+        """Get a list of variable names from the user's namespace.
+
+        The return value is a dict with the repr() of each value.
+        """
+        out = {}
+        user_ns = self.user_ns
+        for varname in names:
+            try:
+                value = repr(user_ns[varname])
+            except:
+                value = self._simple_error()
+            out[varname] = value
+        return out
+        
+    def eval_expressions(self, expressions):
+        """Evaluate a dict of expressions in the user's namespace.
+
+        The return value is a dict with the repr() of each value.
+        """
+        out = {}
+        user_ns = self.user_ns
+        global_ns = self.user_global_ns
+        for key, expr in expressions.iteritems():
+            try:
+                value = repr(eval(expr, global_ns, user_ns))
+            except:
+                value = self._simple_error()
+            out[key] = value
+        return out
+
+    #-------------------------------------------------------------------------
     # Things related to the running of code
     #-------------------------------------------------------------------------
 

IPython/frontend/qt/console/frontend_widget.py

@@ -131,10 +131,27 b' class FrontendWidget(HistoryConsoleWidget, BaseFrontendMixin):'
             complete = not self._input_splitter.push_accepts_more()
         return complete
 
-    def _execute(self, source, hidden):
+    def _execute(self, source, hidden, user_variables=None,
+                 user_expressions=None):
         """ Execute 'source'. If 'hidden', do not show any output.
-        """
-        self.kernel_manager.xreq_channel.execute(source, hidden)
+
+        See parent class :meth:`execute` docstring for full details.
+        """
+        # tmp code for testing, disable in real use with 'if 0'.  Only delete
+        # this code once we have automated tests for these fields.
+        if 0:
+            user_variables = ['x', 'y', 'z']
+            user_expressions = {'sum' : '1+1',
+                                'bad syntax' : 'klsdafj kasd f',
+                                'bad call' : 'range("hi")',
+                                'time' : 'time.time()',
+                                }
+        # /end tmp code
+        
+        # FIXME - user_variables/expressions are not visible in API above us.
+        self.kernel_manager.xreq_channel.execute(source, hidden,
+                                                 user_variables,
+                                                 user_expressions)
         self._hidden = hidden
         
     def _prompt_started_hook(self):

IPython/frontend/qt/console/ipython_widget.py

@@ -21,6 +21,7 b' from PyQt4 import QtCore, QtGui'
 # Local imports
 from IPython.core.inputsplitter import IPythonInputSplitter
 from IPython.core.usage import default_banner
+from IPython.utils import io
 from IPython.utils.traitlets import Bool, Str
 from frontend_widget import FrontendWidget
 
@@ -50,9 +51,13 b" default_dark_style_sheet = '''"
 '''
 default_dark_syntax_style = 'monokai'
 
-# Default prompts.
+# Default strings to build and display input and output prompts (and separators
+# in between)
 default_in_prompt = 'In [<span class="in-prompt-number">%i</span>]: '
 default_out_prompt = 'Out[<span class="out-prompt-number">%i</span>]: '
+default_input_sep = '\n'
+default_output_sep = ''
+default_output_sep2 = ''
 
 #-----------------------------------------------------------------------------
 # IPythonWidget class
@@ -92,6 +97,9 b' class IPythonWidget(FrontendWidget):'
     # Prompts.
     in_prompt = Str(default_in_prompt, config=True)
     out_prompt = Str(default_out_prompt, config=True)
+    input_sep = Str(default_input_sep, config=True)
+    output_sep = Str(default_output_sep, config=True)
+    output_sep2 = Str(default_output_sep2, config=True)
 
     # FrontendWidget protected class variables.
     _input_splitter_class = IPythonInputSplitter
@@ -155,20 +163,17 b' class IPythonWidget(FrontendWidget):'
         """ Implemented to handle prompt number replies, which are only
             supported by the IPython kernel.
         """
-        content = msg['content']
-        self._show_interpreter_prompt(content['prompt_number'], 
-                                      content['input_sep'])
+        self._show_interpreter_prompt(msg['content']['execution_count'])
 
     def _handle_pyout(self, msg):
         """ Reimplemented for IPython-style "display hook".
         """
         if not self._hidden and self._is_from_this_session(msg):
             content = msg['content']
-            prompt_number = content['prompt_number']
-            self._append_plain_text(content['output_sep'])
+            prompt_number = content['execution_count']
+            self._append_plain_text(self.output_sep)
             self._append_html(self._make_out_prompt(prompt_number))
-            self._append_plain_text(content['data'] + '\n' + 
-                                    content['output_sep2'])
+            self._append_plain_text(content['data']+self.output_sep2)
 
     def _started_channels(self):
         """ Reimplemented to make a history request.
@@ -244,17 +249,19 b' class IPythonWidget(FrontendWidget):'
         else:
             return False
 
-    def _show_interpreter_prompt(self, number=None, input_sep='\n'):
+    def _show_interpreter_prompt(self, number=None):
         """ Reimplemented for IPython-style prompts.
         """
         # If a number was not specified, make a prompt number request.
         if number is None:
-            self.kernel_manager.xreq_channel.prompt()
-            return
+            # FIXME - fperez: this should be a silent code request
+            number = 1
+            ##self.kernel_manager.xreq_channel.prompt()
+            ##return
 
         # Show a new prompt and save information about it so that it can be
         # updated later if the prompt number turns out to be wrong.
-        self._prompt_sep = input_sep
+        self._prompt_sep = self.input_sep
         self._show_prompt(self._make_in_prompt(number), html=True)
         block = self._control.document().lastBlock()
         length = len(self._prompt)
@@ -269,7 +276,8 b' class IPythonWidget(FrontendWidget):'
         """
         # Update the old prompt number if necessary.
         content = msg['content']
-        previous_prompt_number = content['prompt_number']
+        ##io.rprint('_show_interpreter_prompt_for_reply\n', content) # dbg
+        previous_prompt_number = content['execution_count']
         if self._previous_prompt_obj and \
                 self._previous_prompt_obj.number != previous_prompt_number:
             block = self._previous_prompt_obj.block
@@ -293,9 +301,7 b' class IPythonWidget(FrontendWidget):'
             self._previous_prompt_obj = None
 
         # Show a new prompt with the kernel's estimated prompt number.
-        next_prompt = content['next_prompt']
-        self._show_interpreter_prompt(next_prompt['prompt_number'], 
-                                      next_prompt['input_sep'])
+        self._show_interpreter_prompt(previous_prompt_number+1)
 
     #---------------------------------------------------------------------------
     # 'IPythonWidget' interface

IPython/utils/_process_common.py

@@ -63,7 +63,8 b' def process_handler(cmd, callback, stderr=subprocess.PIPE):'
     """
     sys.stdout.flush()
     sys.stderr.flush()
-    close_fds = False if sys.platform=='win32' else True
+    # On win32, close_fds can't be true when using pipes for stdin/out/err
+    close_fds = sys.platform != 'win32'
     p = subprocess.Popen(cmd, shell=True,
                          stdin=subprocess.PIPE,
                          stdout=subprocess.PIPE,

IPython/zmq/blockingkernelmanager.py

@@ -1,10 +1,32 b''
-from kernelmanager import SubSocketChannel
+"""Implement a fully blocking kernel manager.
+
+Useful for test suites and blocking terminal interfaces.
+"""
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2010  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING.txt, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+from __future__ import print_function
+
+# Stdlib
 from Queue import Queue, Empty
 
+# Our own
+from IPython.utils import io
+from IPython.utils.traitlets import Type
 
-class MsgNotReady(Exception):
-    pass
+from .kernelmanager import (KernelManager, SubSocketChannel, 
+                           XReqSocketChannel, RepSocketChannel, HBSocketChannel)
 
+#-----------------------------------------------------------------------------
+# Functions and classes
+#-----------------------------------------------------------------------------
 
 class BlockingSubSocketChannel(SubSocketChannel):
 
@@ -13,6 +35,7 b' class BlockingSubSocketChannel(SubSocketChannel):'
         self._in_queue = Queue()
 
     def call_handlers(self, msg):
+        io.rprint('[[Sub]]', msg)  # dbg
         self._in_queue.put(msg)
 
     def msg_ready(self):
@@ -24,21 +47,68 b' class BlockingSubSocketChannel(SubSocketChannel):'
 
     def get_msg(self, block=True, timeout=None):
         """Get a message if there is one that is ready."""
-        try:
-            msg = self.in_queue.get(block, timeout)
-        except Empty:
-            raise MsgNotReady('No message has been received.')
+        return self.in_queue.get(block, timeout)
+
+    def get_msgs(self):
+        """Get all messages that are currently ready."""
+        msgs = []
+        while True:
+            try:
+                msgs.append(self.get_msg(block=False))
+            except Empty:
+                break
+        return msgs
+
+    
+
+class BlockingXReqSocketChannel(XReqSocketChannel):
+
+    def __init__(self, context, session, address=None):
+        super(BlockingXReqSocketChannel, self).__init__(context, session, address)
+        self._in_queue = Queue()
+
+    def call_handlers(self, msg):
+        io.rprint('[[XReq]]', msg)  # dbg
+
+    def msg_ready(self):
+        """Is there a message that has been received?"""
+        if self._in_queue.qsize() == 0:
+            return False
         else:
-            return msg
+            return True
+
+    def get_msg(self, block=True, timeout=None):
+        """Get a message if there is one that is ready."""
+        return self.in_queue.get(block, timeout)
 
     def get_msgs(self):
         """Get all messages that are currently ready."""
         msgs = []
         while True:
             try:
-                msg = self.get_msg(block=False)
-            except MsgNotReady:
+                msgs.append(self.get_msg(block=False))
+            except Empty:
                 break
-            else:
-                msgs.append(msg)
-        return msgs
  No newline at end of file
+        return msgs
+
+class BlockingRepSocketChannel(RepSocketChannel):
+    def call_handlers(self, msg):
+        io.rprint('[[Rep]]', msg)  # dbg
+
+
+class BlockingHBSocketChannel(HBSocketChannel):
+    # This kernel needs rapid monitoring capabilities
+    time_to_dead = 0.2
+
+    def call_handlers(self, since_last_heartbeat):
+        io.rprint('[[Heart]]', since_last_heartbeat) # dbg
+    
+
+class BlockingKernelManager(KernelManager):
+    
+    # The classes to use for the various channels.
+    xreq_channel_class = Type(BlockingXReqSocketChannel)
+    sub_channel_class = Type(BlockingSubSocketChannel)
+    rep_channel_class = Type(BlockingRepSocketChannel)
+    hb_channel_class = Type(BlockingHBSocketChannel)
+  

IPython/zmq/ipkernel.py

@@ -35,6 +35,7 b' from iostream import OutStream'
 from session import Session, Message
 from zmqshell import ZMQInteractiveShell
 
+
 #-----------------------------------------------------------------------------
 # Main kernel class
 #-----------------------------------------------------------------------------
@@ -64,8 +65,7 b' class Kernel(Configurable):'
 
         # Build dict of handlers for message types
         msg_types = [ 'execute_request', 'complete_request', 
-                      'object_info_request', 'prompt_request',
-                      'history_request' ]
+                      'object_info_request', 'history_request' ]
         self.handlers = {}
         for msg_type in msg_types:
             self.handlers[msg_type] = getattr(self, msg_type)
@@ -81,14 +81,21 b' class Kernel(Configurable):'
         # FIXME: Bug in pyzmq/zmq?
         # assert self.reply_socket.rcvmore(), "Missing message part."
         msg = self.reply_socket.recv_json()
-        omsg = Message(msg)
-        io.raw_print('\n')
-        io.raw_print(omsg)
-        handler = self.handlers.get(omsg.msg_type, None)
+        
+        # Print some info about this message and leave a '--->' marker, so it's
+        # easier to trace visually the message chain when debugging.  Each
+        # handler prints its message at the end.
+        # Eventually we'll move these from stdout to a logger.
+        io.raw_print('\n*** MESSAGE TYPE:', msg['msg_type'], '***')
+        io.raw_print('   Content: ', msg['content'],
+                     '\n   --->\n   ', sep='', end='')
+
+        # Find and call actual handler for message
+        handler = self.handlers.get(msg['msg_type'], None)
         if handler is None:
-            io.raw_print_err("UNKNOWN MESSAGE TYPE:", omsg)
+            io.raw_print_err("UNKNOWN MESSAGE TYPE:", msg)
         else:
-            handler(ident, omsg)
+            handler(ident, msg)
 
     def start(self):
         """ Start the kernel main loop.
@@ -97,37 +104,56 b' class Kernel(Configurable):'
             time.sleep(0.05)
             self.do_one_iteration()
 
-
     #---------------------------------------------------------------------------
     # Kernel request handlers
     #---------------------------------------------------------------------------
 
+    def _publish_pyin(self, code, parent):
+        """Publish the code request on the pyin stream."""
+
+        pyin_msg = self.session.msg(u'pyin',{u'code':code}, parent=parent)
+        self.pub_socket.send_json(pyin_msg)
+
     def execute_request(self, ident, parent):
         try:
-            code = parent[u'content'][u'code']
+            content = parent[u'content']
+            code = content[u'code']
+            silent = content[u'silent'] 
         except:
             io.raw_print_err("Got bad msg: ")
             io.raw_print_err(Message(parent))
             return
-        pyin_msg = self.session.msg(u'pyin',{u'code':code}, parent=parent)
-        self.pub_socket.send_json(pyin_msg)
 
+        shell = self.shell # we'll need this a lot here
+
+        # Replace raw_input. Note that is not sufficient to replace 
+        # raw_input in the user namespace.
+        raw_input = lambda prompt='': self._raw_input(prompt, ident, parent)
+        __builtin__.raw_input = raw_input
+
+        # Set the parent message of the display hook and out streams.
+        shell.displayhook.set_parent(parent)
+        sys.stdout.set_parent(parent)
+        sys.stderr.set_parent(parent)
+
+        # Re-broadcast our input for the benefit of listening clients, and
+        # start computing output
+        if not silent:
+            self._publish_pyin(code, parent)
+
+        reply_content = {}
         try:
-            # Replace raw_input. Note that is not sufficient to replace 
-            # raw_input in the user namespace.
-            raw_input = lambda prompt='': self._raw_input(prompt, ident, parent)
-            __builtin__.raw_input = raw_input
-
-            # Set the parent message of the display hook and out streams.
-            self.shell.displayhook.set_parent(parent)
-            sys.stdout.set_parent(parent)
-            sys.stderr.set_parent(parent)
-
-            # FIXME: runlines calls the exception handler itself.  We should
-            # clean this up.
-            self.shell._reply_content = None
-            self.shell.runlines(code)
+            if silent:
+                # runcode uses 'exec' mode, so no displayhook will fire, and it
+                # doesn't call logging or history manipulations.  Print
+                # statements in that code will obviously still execute.
+                shell.runcode(code)
+            else:
+                # FIXME: runlines calls the exception handler itself.
+                shell._reply_content = None
+                shell.runlines(code)
         except:
+            status = u'error'
             # FIXME: this code right now isn't being used yet by default,
             # because the runlines() call above directly fires off exception
             # reporting.  This code, therefore, is only active in the scenario
@@ -136,35 +162,39 b' class Kernel(Configurable):'
             # single location in the codbase.
             etype, evalue, tb = sys.exc_info()
             tb_list = traceback.format_exception(etype, evalue, tb)
-            reply_content = self.shell._showtraceback(etype, evalue, tb_list)
+            reply_content.update(shell._showtraceback(etype, evalue, tb_list))
         else:
-            payload = self.shell.payload_manager.read_payload()
+            status = u'ok'
+            reply_content[u'payload'] = shell.payload_manager.read_payload()
             # Be agressive about clearing the payload because we don't want
             # it to sit in memory until the next execute_request comes in.
-            self.shell.payload_manager.clear_payload()
-            reply_content = { 'status' : 'ok', 'payload' : payload }
-
-        # Compute the prompt information
-        prompt_number = self.shell.displayhook.prompt_count
-        reply_content['prompt_number'] = prompt_number        
-        prompt_string = self.shell.displayhook.prompt1.peek_next_prompt()
-        next_prompt = {'prompt_string' : prompt_string,
-                       'prompt_number' : prompt_number+1,
-                       'input_sep'     : self.shell.displayhook.input_sep}
-        reply_content['next_prompt'] = next_prompt
-
-        # TMP - fish exception info out of shell, possibly left there by
-        # runlines
-        if self.shell._reply_content is not None:
-            reply_content.update(self.shell._reply_content)
-
-        # Flush output before sending the reply.
-        sys.stderr.flush()
-        sys.stdout.flush()
-
+            shell.payload_manager.clear_payload()
+
+        reply_content[u'status'] = status
+        # Compute the execution counter so clients can display prompts
+        reply_content['execution_count'] = shell.displayhook.prompt_count
+
+        # FIXME - fish exception info out of shell, possibly left there by
+        # runlines.  We'll need to clean up this logic later.
+        if shell._reply_content is not None:
+            reply_content.update(shell._reply_content)
+
+        # At this point, we can tell whether the main code execution succeeded
+        # or not.  If it did, we proceed to evaluate user_variables/expressions
+        if reply_content['status'] == 'ok':
+            reply_content[u'user_variables'] = \
+                         shell.get_user_variables(content[u'user_variables'])
+            reply_content[u'user_expressions'] = \
+                         shell.eval_expressions(content[u'user_expressions'])
+        else:
+            # If there was an error, don't even try to compute variables or
+            # expressions
+            reply_content[u'user_variables'] = {}
+            reply_content[u'user_expressions'] = {}
+            
         # Send the reply.
         reply_msg = self.session.msg(u'execute_reply', reply_content, parent)
-        io.raw_print(Message(reply_msg))
+        io.raw_print(reply_msg)
         self.reply_socket.send(ident, zmq.SNDMORE)
         self.reply_socket.send_json(reply_msg)
         if reply_msg['content']['status'] == u'error':
@@ -186,16 +216,6 b' class Kernel(Configurable):'
                                 object_info, parent, ident)
         io.raw_print(msg)
 
-    def prompt_request(self, ident, parent):
-        prompt_number = self.shell.displayhook.prompt_count
-        prompt_string = self.shell.displayhook.prompt1.peek_next_prompt()
-        content = {'prompt_string' : prompt_string,
-                   'prompt_number' : prompt_number+1,
-                   'input_sep'     : self.shell.displayhook.input_sep}
-        msg = self.session.send(self.reply_socket, 'prompt_reply',
-                                content, parent, ident)
-        io.raw_print(msg)
-
     def history_request(self, ident, parent):
         output = parent['content']['output']
         index = parent['content']['index']
@@ -218,13 +238,14 b' class Kernel(Configurable):'
                 if e.errno == zmq.EAGAIN:
                     break
             else:
-                assert self.reply_socket.rcvmore(), "Unexpected missing message part."
+                assert self.reply_socket.rcvmore(), \
+                       "Unexpected missing message part."
                 msg = self.reply_socket.recv_json()
             io.raw_print("Aborting:\n", Message(msg))
             msg_type = msg['msg_type']
             reply_type = msg_type.split('_')[0] + '_reply'
             reply_msg = self.session.msg(reply_type, {'status' : 'aborted'}, msg)
-            io.raw_print(Message(reply_msg))
+            io.raw_print(reply_msg)
             self.reply_socket.send(ident,zmq.SNDMORE)
             self.reply_socket.send_json(reply_msg)
             # We need to wait a bit for requests to come in. This can probably
@@ -312,6 +333,7 b' class QtKernel(Kernel):'
         self.timer.start(50)
         start_event_loop_qt4(self.app)
 
+
 class WxKernel(Kernel):
     """A Kernel subclass with Wx support."""
 
@@ -421,6 +443,7 b' def launch_kernel(xrep_port=0, pub_port=0, req_port=0, hb_port=0,'
                               xrep_port, pub_port, req_port, hb_port, 
                               independent, extra_arguments)
 
+
 def main():
     """ The IPython kernel main entry point.
     """
@@ -458,5 +481,6 b" given, the GUI backend is matplotlib's, otherwise use one of: \\"
 
     start_kernel(namespace, kernel)
 
+
 if __name__ == '__main__':
     main()

IPython/zmq/kernelmanager.py

@@ -29,6 +29,7 b' from zmq import POLLIN, POLLOUT, POLLERR'
 from zmq.eventloop import ioloop
 
 # Local imports.
+from IPython.utils import io
 from IPython.utils.traitlets import HasTraits, Any, Instance, Type, TCPAddress
 from session import Session
 
@@ -42,6 +43,35 b' class InvalidPortNumber(Exception):'
     pass
 
 #-----------------------------------------------------------------------------
+# Utility functions
+#-----------------------------------------------------------------------------
+
+# some utilities to validate message structure, these might get moved elsewhere
+# if they prove to have more generic utility
+
+def validate_string_list(lst):
+    """Validate that the input is a list of strings.
+
+    Raises ValueError if not."""
+    if not isinstance(lst, list):
+        raise ValueError('input %r must be a list' % lst)
+    for x in lst:
+        if not isinstance(x, basestring):
+            raise ValueError('element %r in list must be a string' % x)
+
+
+def validate_string_dict(dct):
+    """Validate that the input is a dict with string keys and values.
+
+    Raises ValueError if not."""
+    for k,v in dct.iteritems():
+        if not isinstance(k, basestring):
+            raise ValueError('key %r in dict must be a string' % k)
+        if not isinstance(v, basestring):
+            raise ValueError('value %r in dict must be a string' % v)
+
+
+#-----------------------------------------------------------------------------
 # ZMQ Socket Channel classes
 #-----------------------------------------------------------------------------
 
@@ -163,23 +193,49 b' class XReqSocketChannel(ZmqSocketChannel):'
         """
         raise NotImplementedError('call_handlers must be defined in a subclass.')
 
-    def execute(self, code, silent=False):
+    def execute(self, code, silent=False,
+                user_variables=None, user_expressions=None):
         """Execute code in the kernel.
 
         Parameters
         ----------
         code : str
             A string of Python code.
+            
         silent : bool, optional (default False)
             If set, the kernel will execute the code as quietly possible.
 
+        user_variables : list, optional
+        
+            A list of variable names to pull from the user's namespace.  They
+            will come back as a dict with these names as keys and their
+            :func:`repr` as values.
+            
+        user_expressions : dict, optional
+            A dict with string keys and  to pull from the user's
+            namespace.  They will come back as a dict with these names as keys
+            and their :func:`repr` as values.
+
         Returns
         -------
         The msg_id of the message sent.
         """
+        if user_variables is None:
+            user_variables = []
+        if user_expressions is None:
+            user_expressions = {}
+            
+        # Don't waste network traffic if inputs are invalid
+        if not isinstance(code, basestring):
+            raise ValueError('code %r must be a string' % code)
+        validate_string_list(user_variables)
+        validate_string_dict(user_expressions)
+
         # Create class for content/msg creation. Related to, but possibly
         # not in Session.
-        content = dict(code=code, silent=silent)
+        content = dict(code=code, silent=silent,
+                       user_variables=user_variables,
+                       user_expressions=user_expressions)
         msg = self.session.msg('execute_request', content)
         self._queue_request(msg)
         return msg['header']['msg_id']
@@ -249,17 +305,6 b' class XReqSocketChannel(ZmqSocketChannel):'
         self._queue_request(msg)
         return msg['header']['msg_id']
 
-    def prompt(self):
-        """Requests a prompt number from the kernel.
-
-        Returns
-        -------
-        The msg_id of the message sent.
-        """
-        msg = self.session.msg('prompt_request')
-        self._queue_request(msg)
-        return msg['header']['msg_id']
-
     def _handle_events(self, socket, events):
         if events & POLLERR:
             self._handle_err()
@@ -479,9 +524,12 b' class HBSocketChannel(ZmqSocketChannel):'
             since_last_heartbeat = 0.0
             request_time = time.time()
             try:
+                #io.rprint('Ping from HB channel') # dbg
                 self.socket.send_json('ping')
             except zmq.ZMQError, e:
+                #io.rprint('*** HB Error:', e) # dbg
                 if e.errno == zmq.EFSM:
+                    #io.rprint('sleep...', self.time_to_dead) # dbg
                     time.sleep(self.time_to_dead)
                     self._create_socket()
                 else:
@@ -489,13 +537,21 b' class HBSocketChannel(ZmqSocketChannel):'
             else:
                 while True:
                     try:
-                        reply = self.socket.recv_json(zmq.NOBLOCK)
+                        self.socket.recv_json(zmq.NOBLOCK)
                     except zmq.ZMQError, e:
+                        #io.rprint('*** HB Error 2:', e) # dbg
                         if e.errno == zmq.EAGAIN:
-                            until_dead = self.time_to_dead - (time.time() -
+                            before_poll = time.time()
+                            until_dead = self.time_to_dead - (before_poll -
                                                               request_time)
-                            # poll timeout is in milliseconds.
-                            poll_result = self.poller.poll(1000*until_dead)
+
+                            # When the return value of poll() is an empty list,
+                            # that is when things have gone wrong (zeromq bug).
+                            # As long as it is not an empty list, poll is
+                            # working correctly even if it returns quickly.
+                            # Note: poll timeout is in milliseconds.
+                            self.poller.poll(1000*until_dead)
+                            
                             since_last_heartbeat = time.time() - request_time
                             if since_last_heartbeat > self.time_to_dead:
                                 self.call_handlers(since_last_heartbeat)
@@ -507,6 +563,7 b' class HBSocketChannel(ZmqSocketChannel):'
                         until_dead = self.time_to_dead - (time.time() -
                                                           request_time)
                         if until_dead > 0.0:
+                            #io.rprint('sleep...', self.time_to_dead) # dbg
                             time.sleep(until_dead)
                         break
 

IPython/zmq/zmqshell.py

@@ -61,10 +61,7 b' class ZMQDisplayHook(DisplayHook):'
     def write_output_prompt(self):
         """Write the output prompt."""
         if self.do_full_cache:
-            self.msg['content']['output_sep'] = self.output_sep
-            self.msg['content']['prompt_string'] = str(self.prompt_out)
-            self.msg['content']['prompt_number'] = self.prompt_count
-            self.msg['content']['output_sep2'] = self.output_sep2
+            self.msg['content']['execution_count'] = self.prompt_count
 
     def write_result_repr(self, result_repr):
         self.msg['content']['data'] = result_repr
@@ -383,7 +380,6 b' class ZMQInteractiveShell(InteractiveShell):'
     def _showtraceback(self, etype, evalue, stb):
 
         exc_content = {
-            u'status' : u'error',
             u'traceback' : stb,
             u'ename' : unicode(etype.__name__),
             u'evalue' : unicode(evalue)
@@ -397,7 +393,10 b' class ZMQInteractiveShell(InteractiveShell):'
 
         # FIXME - Hack: store exception info in shell object.  Right now, the
         # caller is reading this info after the fact, we need to fix this logic
-        # to remove this hack.
+        # to remove this hack.  Even uglier, we need to store the error status
+        # here, because in the main loop, the logic that sets it is being
+        # skipped because runlines swallows the exceptions.
+        exc_content[u'status'] = u'error'
         self._reply_content = exc_content
         # /FIXME
         

docs/source/development/messaging.txt

@@ -242,28 +242,27 b' Message type: ``execute_reply``::'
       # prompt numbers to the user.  If the request was a silent one, this will
       # be the current value of the counter in the kernel.
       'execution_count' : int,
-
-      # If the state_template was provided, this will contain the evaluated
-      # form of the template.
-      'state' : str,
     }
 
 When status is 'ok', the following extra fields are present::
 
     {
-      # The kernel will often transform the input provided to it.  If the
-      # '---->' transform had been applied, this is filled, otherwise it's the
-      # empty string.  So transformations like magics don't appear here, only
-      # autocall ones.
-    
-      'transformed_code' : str,
-
       # The execution payload is a dict with string keys that may have been
       # produced by the code being executed.  It is retrieved by the kernel at
       # the end of the execution and sent back to the front end, which can take
       # action on it as needed.  See main text for further details.
       'payload' : dict,
-    }
+
+      # Results for the user_variables and user_expressions.
+      'user_variables' : dict,
+      'user_expressions' : dict,
+
+      # The kernel will often transform the input provided to it.  If the
+      # '---->' transform had been applied, this is filled, otherwise it's the
+      # empty string.  So transformations like magics don't appear here, only
+      # autocall ones.
+      'transformed_code' : str,
+      }
 
 .. admonition:: Execution payloads