Added kernel shutdown support: messaging spec, zmq and client code ready.

This isn't the cleanest code in the world because shutdown has a
synchronous flavor to it that is hard to meld with a purely async
framework.  But the key pieces are reasonably well in place.  Tested
both with 'exit' calls and Ctrl-. interruption.

IPython/frontend/qt/console/frontend_widget.py

@@ -206,7 +206,7 b' class FrontendWidget(HistoryConsoleWidget, BaseFrontendMixin):'
                 return True
             elif key == QtCore.Qt.Key_Period:
                 message = 'Are you sure you want to restart the kernel?'
-                self.restart_kernel(message)
+                self.restart_kernel(message, instant_death=False)
                 return True
         return super(FrontendWidget, self)._event_filter_console_keypress(event)
 
@@ -279,7 +279,7 b' class FrontendWidget(HistoryConsoleWidget, BaseFrontendMixin):'
         if self.custom_restart:
             self.custom_restart_kernel_died.emit(since_last_heartbeat)
         else:
-            self.restart_kernel(message)
+            self.restart_kernel(message, instant_death=True)
 
     def _handle_object_info_reply(self, rep):
         """ Handle replies for call tips.
@@ -341,9 +341,16 b' class FrontendWidget(HistoryConsoleWidget, BaseFrontendMixin):'
             self._append_plain_text('Kernel process is either remote or '
                                     'unspecified. Cannot interrupt.\n')
 
-    def restart_kernel(self, message):
+    def restart_kernel(self, message, instant_death=False):
         """ Attempts to restart the running kernel.
         """
+        # FIXME: instant_death should be configurable via a checkbox in the
+        # dialog.  Right now at least the heartbeat path sets it to True and
+        # the manual restart to False.  But those should just be the
+        # pre-selected states of a checkbox that the user could override if so
+        # desired.  But I don't know enough Qt to go implementing the checkbox
+        # now.
+
         # We want to make sure that if this dialog is already happening, that
         # other signals don't trigger it again. This can happen when the 
         # kernel_died heartbeat signal is emitted and the user is slow to
@@ -360,7 +367,8 b' class FrontendWidget(HistoryConsoleWidget, BaseFrontendMixin):'
                                                     message, buttons)
                 if result == QtGui.QMessageBox.Yes:
                     try:
-                        self.kernel_manager.restart_kernel()
+                        self.kernel_manager.restart_kernel(
+                            instant_death=instant_death)
                     except RuntimeError:
                         message = 'Kernel started externally. Cannot restart.\n'
                         self._append_plain_text(message)

IPython/zmq/ipkernel.py

@@ -17,6 +17,7 b' from __future__ import print_function'
 
 # Standard library imports.
 import __builtin__
+import atexit
 import sys
 import time
 import traceback
@@ -30,13 +31,12 b' from IPython.utils import io'
 from IPython.utils.jsonutil import json_clean
 from IPython.lib import pylabtools
 from IPython.utils.traitlets import Instance, Float
-from entry_point import base_launch_kernel, make_argument_parser, make_kernel, \
-    start_kernel
+from entry_point import (base_launch_kernel, make_argument_parser, make_kernel,
+                         start_kernel)
 from iostream import OutStream
 from session import Session, Message
 from zmqshell import ZMQInteractiveShell
 
-
 #-----------------------------------------------------------------------------
 # Main kernel class
 #-----------------------------------------------------------------------------
@@ -68,10 +68,21 b' class Kernel(Configurable):'
     # Units are in seconds, kernel subclasses for GUI toolkits may need to
     # adapt to milliseconds.
     _poll_interval = Float(0.05, config=True)
+
+    # If the shutdown was requested over the network, we leave here the
+    # necessary reply message so it can be sent by our registered atexit
+    # handler.  This ensures that the reply is only sent to clients truly at
+    # the end of our shutdown process (which happens after the underlying
+    # IPython shell's own shutdown).
+    _shutdown_message = None
     
     def __init__(self, **kwargs):
         super(Kernel, self).__init__(**kwargs)
 
+        # Before we even start up the shell, register *first* our exit handlers
+        # so they come before the shell's
+        atexit.register(self._at_shutdown)
+
         # Initialize the InteractiveShell subclass
         self.shell = ZMQInteractiveShell.instance()
         self.shell.displayhook.session = self.session
@@ -82,7 +93,8 b' class Kernel(Configurable):'
 
         # Build dict of handlers for message types
         msg_types = [ 'execute_request', 'complete_request', 
-                      'object_info_request', 'history_request' ]
+                      'object_info_request', 'history_request',
+                      'shutdown_request']
         self.handlers = {}
         for msg_type in msg_types:
             self.handlers[msg_type] = getattr(self, msg_type)
@@ -271,6 +283,11 b' class Kernel(Configurable):'
                                 content, parent, ident)
         io.raw_print(msg)
         
+    def shutdown_request(self, ident, parent):
+        self.shell.exit_now = True
+        self._shutdown_message = self.session.msg(u'shutdown_reply', {}, parent)
+        sys.exit(0)
+        
     #---------------------------------------------------------------------------
     # Protected interface
     #---------------------------------------------------------------------------
@@ -360,6 +377,17 b' class Kernel(Configurable):'
 
         return symbol, []
 
+    def _at_shutdown(self):
+        """Actions taken at shutdown by the kernel, called by python's atexit.
+        """
+        # io.rprint("Kernel at_shutdown") # dbg
+        if self._shutdown_message is not None:
+            self.reply_socket.send_json(self._shutdown_message)
+            io.raw_print(self._shutdown_message)
+            # A very short sleep to give zmq time to flush its message buffers
+            # before Python truly shuts down.
+            time.sleep(0.01)
+
 
 class QtKernel(Kernel):
     """A Kernel subclass with Qt support."""
@@ -367,10 +395,9 b' class QtKernel(Kernel):'
     def start(self):
         """Start a kernel with QtPy4 event loop integration."""
 
-        from PyQt4 import QtGui, QtCore
-        from IPython.lib.guisupport import (
-            get_app_qt4, start_event_loop_qt4
-        )
+        from PyQt4 import QtCore
+        from IPython.lib.guisupport import get_app_qt4, start_event_loop_qt4
+
         self.app = get_app_qt4([" "])
         self.app.setQuitOnLastWindowClosed(False)
         self.timer = QtCore.QTimer()
@@ -388,6 +415,7 b' class WxKernel(Kernel):'
 
         import wx
         from IPython.lib.guisupport import start_event_loop_wx
+
         doi = self.do_one_iteration
          # Wx uses milliseconds
         poll_interval = int(1000*self._poll_interval)

IPython/zmq/kernelmanager.py

@@ -304,6 +304,23 b' class XReqSocketChannel(ZmqSocketChannel):'
         self._queue_request(msg)
         return msg['header']['msg_id']
 
+    def shutdown(self):
+        """Request an immediate kernel shutdown.
+
+        Upon receipt of the (empty) reply, client code can safely assume that
+        the kernel has shut down and it's safe to forcefully terminate it if
+        it's still alive.
+
+        The kernel will send the reply via a function registered with Python's
+        atexit module, ensuring it's truly done as the kernel is done with all
+        normal operation.
+        """
+        # Send quit message to kernel. Once we implement kernel-side setattr,
+        # this should probably be done that way, but for now this will do.
+        msg = self.session.msg('shutdown_request', {})
+        self._queue_request(msg)
+        return msg['header']['msg_id']
+
     def _handle_events(self, socket, events):
         if events & POLLERR:
             self._handle_err()
@@ -700,14 +717,11 b' class KernelManager(HasTraits):'
         """ Attempts to the stop the kernel process cleanly. If the kernel
         cannot be stopped, it is killed, if possible.
         """
-        # Send quit message to kernel. Once we implement kernel-side setattr,
-        # this should probably be done that way, but for now this will do.
-        self.xreq_channel.execute('get_ipython().exit_now=True', silent=True)
-
+        self.xreq_channel.shutdown()
         # Don't send any additional kernel kill messages immediately, to give
         # the kernel a chance to properly execute shutdown actions. Wait for at
-        # most 2s, checking every 0.1s.
-        for i in range(20):
+        # most 1s, checking every 0.1s.
+        for i in range(10):
             if self.is_alive:
                 time.sleep(0.1)
             else:
@@ -716,18 +730,31 b' class KernelManager(HasTraits):'
             # OK, we've waited long enough.
             if self.has_kernel:
                 self.kill_kernel()
-
-    def restart_kernel(self):
+    
+    def restart_kernel(self, instant_death=False):
         """Restarts a kernel with the same arguments that were used to launch
         it. If the old kernel was launched with random ports, the same ports
         will be used for the new kernel.
+
+        Parameters
+        ----------
+        instant_death : bool, optional
+          If True, the kernel is forcefully restarted *immediately*, without
+          having a chance to do any cleanup action.  Otherwise the kernel is
+          given 1s to clean up before a forceful restart is issued.
+
+          In all cases the kernel is restarted, the only difference is whether
+          it is given a chance to perform a clean shutdown or not.
         """
         if self._launch_args is None:
             raise RuntimeError("Cannot restart the kernel. "
                                "No previous call to 'start_kernel'.")
         else:
             if self.has_kernel:
-                self.kill_kernel()
+                if instant_death:
+                    self.kill_kernel()
+                else:
+                    self.shutdown_kernel()
             self.start_kernel(**self._launch_args)
 
     @property
@@ -755,6 +782,8 b' class KernelManager(HasTraits):'
     @property
     def is_alive(self):
         """Is the kernel process still running?"""
+        # FIXME: not using a heartbeat means this method is broken for any
+        # remote kernel, it's only capable of handling local kernels.
         if self.kernel is not None:
             if self.kernel.poll() is None:
                 return True

docs/source/development/messaging.txt

@@ -534,7 +534,49 b' Message type: ``history_reply``::'
       # respectively.
       'history' : dict,
     }
+
+
+Kernel shutdown
+---------------
+
+The clients can request the kernel to shut itself down; this is used in
+multiple cases:
+
+- when the user chooses to close the client application via a menu or window
+  control.
+- when the user types 'exit' or 'quit' (or their uppercase magic equivalents).
+- when the user chooses a GUI method (like the 'Ctrl-C' shortcut in the
+  IPythonQt client) to force a kernel restart to get a clean kernel without
+  losing client-side state like history or inlined figures.
+
+The client sends a shutdown request to the kernel, and once it receives the
+reply message (which is otherwise empty), it can assume that the kernel has
+completed shutdown safely.
+
+Upon their own shutdown, client applications will typically execute a last
+minute sanity check and forcefully terminate any kernel that is still alive, to
+avoid leaving stray processes in the user's machine.
+
+For both shutdown request and reply, there is no actual content that needs to
+be sent, so the content dict is empty.
+
+Message type: ``shutdown_request``::
+
+    content = {
+    }
+
+Message type: ``shutdown_reply``::
+
+    content = {
+    }
+
+.. Note::
+
+   When the clients detect a dead kernel thanks to inactivity on the heartbeat
+   socket, they simply send a forceful process termination signal, since a dead
+   process is unlikely to respond in any useful way to messages.
     
+
 Messages on the PUB/SUB socket
 ==============================