Merge pull request #689 from minrk/auth

Protect ipkernel from bad messages: now the kernel logs bad messages as errors instead of crashing.

IPython/zmq/ipkernel.py

@@ -1,686 +1,700 b''
 #!/usr/bin/env python
 """A simple interactive kernel that talks to a frontend over 0MQ.
 
 Things to do:
 
 * Implement `set_parent` logic. Right before doing exec, the Kernel should
   call set_parent on all the PUB objects with the message about to be executed.
 * Implement random port and security key logic.
 * Implement control messages.
 * Implement event loop and poll version.
 """
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 from __future__ import print_function
 
 # Standard library imports.
 import __builtin__
 import atexit
 import sys
 import time
 import traceback
 import logging
 # System library imports.
 import zmq
 
 # Local imports.
 from IPython.config.configurable import Configurable
 from IPython.config.application import boolean_flag
 from IPython.core.application import ProfileDir
 from IPython.core.shellapp import (
     InteractiveShellApp, shell_flags, shell_aliases
 )
 from IPython.utils import io
 from IPython.utils.jsonutil import json_clean
 from IPython.lib import pylabtools
 from IPython.utils.traitlets import (
     List, Instance, Float, Dict, Bool, Int, Unicode, CaselessStrEnum
 )
 
 from entry_point import base_launch_kernel
 from kernelapp import KernelApp, kernel_flags, kernel_aliases
 from iostream import OutStream
 from session import Session, Message
 from zmqshell import ZMQInteractiveShell
 
 
 
 #-----------------------------------------------------------------------------
 # Main kernel class
 #-----------------------------------------------------------------------------
 
 class Kernel(Configurable):
 
     #---------------------------------------------------------------------------
     # Kernel interface
     #---------------------------------------------------------------------------
 
     shell = Instance('IPython.core.interactiveshell.InteractiveShellABC')
     session = Instance(Session)
     shell_socket = Instance('zmq.Socket')
     iopub_socket = Instance('zmq.Socket')
     stdin_socket = Instance('zmq.Socket')
     log = Instance(logging.Logger)
 
     # Private interface
 
     # Time to sleep after flushing the stdout/err buffers in each execute
     # cycle.  While this introduces a hard limit on the minimal latency of the
     # execute cycle, it helps prevent output synchronization problems for
     # clients.
     # Units are in seconds.  The minimum zmq latency on local host is probably
     # ~150 microseconds, set this to 500us for now.  We may need to increase it
     # a little if it's not enough after more interactive testing.
     _execute_sleep = Float(0.0005, config=True)
 
     # Frequency of the kernel's event loop.
     # Units are in seconds, kernel subclasses for GUI toolkits may need to
     # adapt to milliseconds.
     _poll_interval = Float(0.05, config=True)
 
     # If the shutdown was requested over the network, we leave here the
     # necessary reply message so it can be sent by our registered atexit
     # handler.  This ensures that the reply is only sent to clients truly at
     # the end of our shutdown process (which happens after the underlying
     # IPython shell's own shutdown).
     _shutdown_message = None
 
     # This is a dict of port number that the kernel is listening on. It is set
     # by record_ports and used by connect_request.
     _recorded_ports = Dict()
 
 
 
     def __init__(self, **kwargs):
         super(Kernel, self).__init__(**kwargs)
 
         # Before we even start up the shell, register *first* our exit handlers
         # so they come before the shell's
         atexit.register(self._at_shutdown)
 
         # Initialize the InteractiveShell subclass
         self.shell = ZMQInteractiveShell.instance(config=self.config)
         self.shell.displayhook.session = self.session
         self.shell.displayhook.pub_socket = self.iopub_socket
         self.shell.display_pub.session = self.session
         self.shell.display_pub.pub_socket = self.iopub_socket
 
         # TMP - hack while developing
         self.shell._reply_content = None
 
         # Build dict of handlers for message types
         msg_types = [ 'execute_request', 'complete_request', 
                       'object_info_request', 'history_request',
                       'connect_request', 'shutdown_request']
         self.handlers = {}
         for msg_type in msg_types:
             self.handlers[msg_type] = getattr(self, msg_type)
 
     def do_one_iteration(self):
         """Do one iteration of the kernel's evaluation loop.
         """
-        ident,msg = self.session.recv(self.shell_socket, zmq.NOBLOCK)
+        try:
+            ident,msg = self.session.recv(self.shell_socket, zmq.NOBLOCK)
+        except Exception:
+            self.log.warn("Invalid Message:", exc_info=True)
+            return
         if msg is None:
             return
 
         msg_type = msg['header']['msg_type']
 
         # This assert will raise in versions of zeromq 2.0.7 and lesser.
         # We now require 2.0.8 or above, so we can uncomment for safety.
         # print(ident,msg, file=sys.__stdout__)
         assert ident is not None, "Missing message part."
         
         # Print some info about this message and leave a '--->' marker, so it's
         # easier to trace visually the message chain when debugging.  Each
         # handler prints its message at the end.
         self.log.debug('\n*** MESSAGE TYPE:'+str(msg_type)+'***')
         self.log.debug('   Content: '+str(msg['content'])+'\n   --->\n   ')
 
         # Find and call actual handler for message
         handler = self.handlers.get(msg_type, None)
         if handler is None:
             self.log.error("UNKNOWN MESSAGE TYPE:" +str(msg))
         else:
             handler(ident, msg)
             
         # Check whether we should exit, in case the incoming message set the
         # exit flag on
         if self.shell.exit_now:
             self.log.debug('\nExiting IPython kernel...')
             # We do a normal, clean exit, which allows any actions registered
             # via atexit (such as history saving) to take place.
             sys.exit(0)
 
 
     def start(self):
         """ Start the kernel main loop.
         """
         poller = zmq.Poller()
         poller.register(self.shell_socket, zmq.POLLIN)
         while True:
             try:
                 # scale by extra factor of 10, because there is no
                 # reason for this to be anything less than ~ 0.1s
                 # since it is a real poller and will respond
                 # to events immediately
                 
                 # double nested try/except, to properly catch KeyboardInterrupt
                 # due to pyzmq Issue #130
                 try:
                     poller.poll(10*1000*self._poll_interval)
                     self.do_one_iteration()
                 except:
                     raise
             except KeyboardInterrupt:
                 # Ctrl-C shouldn't crash the kernel
                 io.raw_print("KeyboardInterrupt caught in kernel")
 
     def record_ports(self, ports):
         """Record the ports that this kernel is using.
 
         The creator of the Kernel instance must call this methods if they
         want the :meth:`connect_request` method to return the port numbers.
         """
         self._recorded_ports = ports
 
     #---------------------------------------------------------------------------
     # Kernel request handlers
     #---------------------------------------------------------------------------
 
     def _publish_pyin(self, code, parent):
         """Publish the code request on the pyin stream."""
 
         pyin_msg = self.session.send(self.iopub_socket, u'pyin',{u'code':code}, parent=parent)
 
     def execute_request(self, ident, parent):
         
         status_msg = self.session.send(self.iopub_socket,
             u'status',
             {u'execution_state':u'busy'},
             parent=parent
         )
         
         try:
             content = parent[u'content']
             code = content[u'code']
             silent = content[u'silent'] 
         except:
             self.log.error("Got bad msg: ")
             self.log.error(str(Message(parent)))
             return
 
         shell = self.shell # we'll need this a lot here
 
         # Replace raw_input. Note that is not sufficient to replace 
         # raw_input in the user namespace.
         raw_input = lambda prompt='': self._raw_input(prompt, ident, parent)
         __builtin__.raw_input = raw_input
 
         # Set the parent message of the display hook and out streams.
         shell.displayhook.set_parent(parent)
         shell.display_pub.set_parent(parent)
         sys.stdout.set_parent(parent)
         sys.stderr.set_parent(parent)
 
         # Re-broadcast our input for the benefit of listening clients, and
         # start computing output
         if not silent:
             self._publish_pyin(code, parent)
 
         reply_content = {}
         try:
             if silent:
                 # run_code uses 'exec' mode, so no displayhook will fire, and it
                 # doesn't call logging or history manipulations.  Print
                 # statements in that code will obviously still execute.
                 shell.run_code(code)
             else:
                 # FIXME: the shell calls the exception handler itself.
                 shell.run_cell(code)
         except:
             status = u'error'
             # FIXME: this code right now isn't being used yet by default,
             # because the run_cell() call above directly fires off exception
             # reporting.  This code, therefore, is only active in the scenario
             # where runlines itself has an unhandled exception.  We need to
             # uniformize this, for all exception construction to come from a
             # single location in the codbase.
             etype, evalue, tb = sys.exc_info()
             tb_list = traceback.format_exception(etype, evalue, tb)
             reply_content.update(shell._showtraceback(etype, evalue, tb_list))
         else:
             status = u'ok'
 
         reply_content[u'status'] = status
         
         # Return the execution counter so clients can display prompts
         reply_content['execution_count'] = shell.execution_count -1
 
         # FIXME - fish exception info out of shell, possibly left there by
         # runlines.  We'll need to clean up this logic later.
         if shell._reply_content is not None:
             reply_content.update(shell._reply_content)
             # reset after use
             shell._reply_content = None
 
         # At this point, we can tell whether the main code execution succeeded
         # or not.  If it did, we proceed to evaluate user_variables/expressions
         if reply_content['status'] == 'ok':
             reply_content[u'user_variables'] = \
                          shell.user_variables(content[u'user_variables'])
             reply_content[u'user_expressions'] = \
                          shell.user_expressions(content[u'user_expressions'])
         else:
             # If there was an error, don't even try to compute variables or
             # expressions
             reply_content[u'user_variables'] = {}
             reply_content[u'user_expressions'] = {}
 
         # Payloads should be retrieved regardless of outcome, so we can both
         # recover partial output (that could have been generated early in a
         # block, before an error) and clear the payload system always.
         reply_content[u'payload'] = shell.payload_manager.read_payload()
         # Be agressive about clearing the payload because we don't want
         # it to sit in memory until the next execute_request comes in.
         shell.payload_manager.clear_payload()
 
         # Flush output before sending the reply.
         sys.stdout.flush()
         sys.stderr.flush()
         # FIXME: on rare occasions, the flush doesn't seem to make it to the
         # clients... This seems to mitigate the problem, but we definitely need
         # to better understand what's going on.
         if self._execute_sleep:
             time.sleep(self._execute_sleep)
         
         # Send the reply.
         reply_msg = self.session.send(self.shell_socket, u'execute_reply',
                                       reply_content, parent, ident=ident)
         self.log.debug(str(reply_msg))
 
         if reply_msg['content']['status'] == u'error':
             self._abort_queue()
 
         status_msg = self.session.send(self.iopub_socket,
             u'status',
             {u'execution_state':u'idle'},
             parent=parent
         )
 
     def complete_request(self, ident, parent):
         txt, matches = self._complete(parent)
         matches = {'matches' : matches,
                    'matched_text' : txt,
                    'status' : 'ok'}
         completion_msg = self.session.send(self.shell_socket, 'complete_reply',
                                            matches, parent, ident)
         self.log.debug(str(completion_msg))
 
     def object_info_request(self, ident, parent):
         object_info = self.shell.object_inspect(parent['content']['oname'])
         # Before we send this object over, we scrub it for JSON usage
         oinfo = json_clean(object_info)
         msg = self.session.send(self.shell_socket, 'object_info_reply',
                                 oinfo, parent, ident)
         self.log.debug(msg)
 
     def history_request(self, ident, parent):
         # We need to pull these out, as passing **kwargs doesn't work with
         # unicode keys before Python 2.6.5.
         hist_access_type = parent['content']['hist_access_type']
         raw = parent['content']['raw']
         output = parent['content']['output']
         if hist_access_type == 'tail':
             n = parent['content']['n']
             hist = self.shell.history_manager.get_tail(n, raw=raw, output=output,
                                                             include_latest=True)
         
         elif hist_access_type == 'range':
             session = parent['content']['session']
             start = parent['content']['start']
             stop = parent['content']['stop']
             hist = self.shell.history_manager.get_range(session, start, stop,
                                                         raw=raw, output=output)
         
         elif hist_access_type == 'search':
             pattern = parent['content']['pattern']
             hist = self.shell.history_manager.search(pattern, raw=raw, output=output)
         
         else:
             hist = []
         content = {'history' : list(hist)}
         msg = self.session.send(self.shell_socket, 'history_reply',
                                 content, parent, ident)
         self.log.debug(str(msg))
 
     def connect_request(self, ident, parent):
         if self._recorded_ports is not None:
             content = self._recorded_ports.copy()
         else:
             content = {}
         msg = self.session.send(self.shell_socket, 'connect_reply',
                                 content, parent, ident)
         self.log.debug(msg)
 
     def shutdown_request(self, ident, parent):
         self.shell.exit_now = True
         self._shutdown_message = self.session.msg(u'shutdown_reply', parent['content'], parent)
         sys.exit(0)
 
     #---------------------------------------------------------------------------
     # Protected interface
     #---------------------------------------------------------------------------
 
     def _abort_queue(self):
         while True:
-            ident,msg = self.session.recv(self.shell_socket, zmq.NOBLOCK)
+            try:
+                ident,msg = self.session.recv(self.shell_socket, zmq.NOBLOCK)
+            except Exception:
+                self.log.warn("Invalid Message:", exc_info=True)
+                continue
             if msg is None:
                 break
             else:
                 assert ident is not None, \
                        "Unexpected missing message part."
 
             self.log.debug("Aborting:\n"+str(Message(msg)))
             msg_type = msg['header']['msg_type']
             reply_type = msg_type.split('_')[0] + '_reply'
             reply_msg = self.session.send(self.shell_socket, reply_type,
                     {'status' : 'aborted'}, msg, ident=ident)
             self.log.debug(reply_msg)
             # We need to wait a bit for requests to come in. This can probably
             # be set shorter for true asynchronous clients.
             time.sleep(0.1)
 
     def _raw_input(self, prompt, ident, parent):
         # Flush output before making the request.
         sys.stderr.flush()
         sys.stdout.flush()
 
         # Send the input request.
         content = dict(prompt=prompt)
         msg = self.session.send(self.stdin_socket, u'input_request', content, parent)
 
         # Await a response.
-        ident, reply = self.session.recv(self.stdin_socket, 0)
+        while True:
+            try:
+                ident, reply = self.session.recv(self.stdin_socket, 0)
+            except Exception:
+                self.log.warn("Invalid Message:", exc_info=True)
+            else:
+                break
         try:
             value = reply['content']['value']
         except:
             self.log.error("Got bad raw_input reply: ")
             self.log.error(str(Message(parent)))
             value = ''
         return value
     
     def _complete(self, msg):
         c = msg['content']
         try:
             cpos = int(c['cursor_pos'])
         except:
             # If we don't get something that we can convert to an integer, at
             # least attempt the completion guessing the cursor is at the end of
             # the text, if there's any, and otherwise of the line
             cpos = len(c['text'])
             if cpos==0:
                 cpos = len(c['line'])
         return self.shell.complete(c['text'], c['line'], cpos)
 
     def _object_info(self, context):
         symbol, leftover = self._symbol_from_context(context)
         if symbol is not None and not leftover:
             doc = getattr(symbol, '__doc__', '')
         else:
             doc = ''
         object_info = dict(docstring = doc)
         return object_info
 
     def _symbol_from_context(self, context):
         if not context:
             return None, context
 
         base_symbol_string = context[0]
         symbol = self.shell.user_ns.get(base_symbol_string, None)
         if symbol is None:
             symbol = __builtin__.__dict__.get(base_symbol_string, None)
         if symbol is None:
             return None, context
 
         context = context[1:]
         for i, name in enumerate(context):
             new_symbol = getattr(symbol, name, None)
             if new_symbol is None:
                 return symbol, context[i:]
             else:
                 symbol = new_symbol
 
         return symbol, []
 
     def _at_shutdown(self):
         """Actions taken at shutdown by the kernel, called by python's atexit.
         """
         # io.rprint("Kernel at_shutdown") # dbg
         if self._shutdown_message is not None:
             self.session.send(self.shell_socket, self._shutdown_message)
             self.session.send(self.iopub_socket, self._shutdown_message)
             self.log.debug(str(self._shutdown_message))
             # A very short sleep to give zmq time to flush its message buffers
             # before Python truly shuts down.
             time.sleep(0.01)
 
 
 class QtKernel(Kernel):
     """A Kernel subclass with Qt support."""
 
     def start(self):
         """Start a kernel with QtPy4 event loop integration."""
 
         from IPython.external.qt_for_kernel import QtCore
         from IPython.lib.guisupport import get_app_qt4, start_event_loop_qt4
 
         self.app = get_app_qt4([" "])
         self.app.setQuitOnLastWindowClosed(False)
         self.timer = QtCore.QTimer()
         self.timer.timeout.connect(self.do_one_iteration)
         # Units for the timer are in milliseconds
         self.timer.start(1000*self._poll_interval)
         start_event_loop_qt4(self.app)
 
 
 class WxKernel(Kernel):
     """A Kernel subclass with Wx support."""
 
     def start(self):
         """Start a kernel with wx event loop support."""
 
         import wx
         from IPython.lib.guisupport import start_event_loop_wx
 
         doi = self.do_one_iteration
          # Wx uses milliseconds
         poll_interval = int(1000*self._poll_interval)
 
         # We have to put the wx.Timer in a wx.Frame for it to fire properly.
         # We make the Frame hidden when we create it in the main app below.
         class TimerFrame(wx.Frame):
             def __init__(self, func):
                 wx.Frame.__init__(self, None, -1)
                 self.timer = wx.Timer(self)
                 # Units for the timer are in milliseconds
                 self.timer.Start(poll_interval)
                 self.Bind(wx.EVT_TIMER, self.on_timer)
                 self.func = func
 
             def on_timer(self, event):
                 self.func()
 
         # We need a custom wx.App to create our Frame subclass that has the
         # wx.Timer to drive the ZMQ event loop.
         class IPWxApp(wx.App):
             def OnInit(self):
                 self.frame = TimerFrame(doi)
                 self.frame.Show(False)
                 return True
 
         # The redirect=False here makes sure that wx doesn't replace
         # sys.stdout/stderr with its own classes.
         self.app = IPWxApp(redirect=False)
         start_event_loop_wx(self.app)
 
 
 class TkKernel(Kernel):
     """A Kernel subclass with Tk support."""
 
     def start(self):
         """Start a Tk enabled event loop."""
 
         import Tkinter
         doi = self.do_one_iteration
         # Tk uses milliseconds
         poll_interval = int(1000*self._poll_interval)
         # For Tkinter, we create a Tk object and call its withdraw method.
         class Timer(object):
             def __init__(self, func):
                 self.app = Tkinter.Tk()
                 self.app.withdraw()
                 self.func = func
                 
             def on_timer(self):
                 self.func()
                 self.app.after(poll_interval, self.on_timer)
                 
             def start(self):
                 self.on_timer()  # Call it once to get things going.
                 self.app.mainloop()
 
         self.timer = Timer(doi)
         self.timer.start()
 
 
 class GTKKernel(Kernel):
     """A Kernel subclass with GTK support."""
     
     def start(self):
         """Start the kernel, coordinating with the GTK event loop"""
         from .gui.gtkembed import GTKEmbed
         
         gtk_kernel = GTKEmbed(self)
         gtk_kernel.start()
 
 
 #-----------------------------------------------------------------------------
 # Aliases and Flags for the IPKernelApp
 #-----------------------------------------------------------------------------
 
 flags = dict(kernel_flags)
 flags.update(shell_flags)
 
 addflag = lambda *args: flags.update(boolean_flag(*args))
 
 flags['pylab'] = (
     {'IPKernelApp' : {'pylab' : 'auto'}},
     """Pre-load matplotlib and numpy for interactive use with
     the default matplotlib backend."""
 )
 
 aliases = dict(kernel_aliases)
 aliases.update(shell_aliases)
 
 # it's possible we don't want short aliases for *all* of these:
 aliases.update(dict(
     pylab='IPKernelApp.pylab',
 ))
 
 #-----------------------------------------------------------------------------
 # The IPKernelApp class
 #-----------------------------------------------------------------------------
 
 class IPKernelApp(KernelApp, InteractiveShellApp):
     name = 'ipkernel'
 
     aliases = Dict(aliases)
     flags = Dict(flags)
     classes = [Kernel, ZMQInteractiveShell, ProfileDir, Session]
     # configurables
     pylab = CaselessStrEnum(['tk', 'qt', 'wx', 'gtk', 'osx', 'inline', 'auto'],
         config=True,
         help="""Pre-load matplotlib and numpy for interactive use,
         selecting a particular matplotlib backend and loop integration.
         """
     )
     pylab_import_all = Bool(True, config=True,
         help="""If true, an 'import *' is done from numpy and pylab,
         when using pylab"""
     )
     def initialize(self, argv=None):
         super(IPKernelApp, self).initialize(argv)
         self.init_shell()
         self.init_extensions()
         self.init_code()
 
     def init_kernel(self):
         kernel_factory = Kernel
 
         kernel_map = {
             'qt' : QtKernel,
             'qt4': QtKernel,
             'inline': Kernel,
             'osx': TkKernel,
             'wx' : WxKernel,
             'tk' : TkKernel,
             'gtk': GTKKernel,
         }
 
         if self.pylab:
             key = None if self.pylab == 'auto' else self.pylab
             gui, backend = pylabtools.find_gui_and_backend(key)
             kernel_factory = kernel_map.get(gui)
             if kernel_factory is None:
                 raise ValueError('GUI is not supported: %r' % gui)
             pylabtools.activate_matplotlib(backend)
 
         kernel = kernel_factory(config=self.config, session=self.session,
                                 shell_socket=self.shell_socket,
                                 iopub_socket=self.iopub_socket,
                                 stdin_socket=self.stdin_socket,
                                 log=self.log
         )
         self.kernel = kernel
         kernel.record_ports(self.ports)
 
         if self.pylab:
             import_all = self.pylab_import_all
             pylabtools.import_pylab(kernel.shell.user_ns, backend, import_all,
                                     shell=kernel.shell)
     
     def init_shell(self):
         self.shell = self.kernel.shell
 
 
 #-----------------------------------------------------------------------------
 # Kernel main and launch functions
 #-----------------------------------------------------------------------------
 
 def launch_kernel(*args, **kwargs):
     """Launches a localhost IPython kernel, binding to the specified ports.
 
     This function simply calls entry_point.base_launch_kernel with the right first
     command to start an ipkernel.  See base_launch_kernel for arguments.
 
     Returns
     -------
     A tuple of form:
         (kernel_process, shell_port, iopub_port, stdin_port, hb_port)
     where kernel_process is a Popen object and the ports are integers.
     """
     return base_launch_kernel('from IPython.zmq.ipkernel import main; main()',
                               *args, **kwargs)
 
 
 def main():
     """Run an IPKernel as an application"""
     app = IPKernelApp.instance()
     app.initialize()
     app.start()
 
 
 if __name__ == '__main__':
     main()

IPython/zmq/session.py

@@ -1,697 +1,698 b''
 #!/usr/bin/env python
 """Session object for building, serializing, sending, and receiving messages in
 IPython. The Session object supports serialization, HMAC signatures, and
 metadata on messages.
 
 Also defined here are utilities for working with Sessions:
 * A SessionFactory to be used as a base class for configurables that work with
 Sessions.
 * A Message object for convenience that allows attribute-access to the msg dict.
 
 Authors:
 
 * Min RK
 * Brian Granger
 * Fernando Perez
 """
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2010-2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 import hmac
 import logging
 import os
 import pprint
 import uuid
 from datetime import datetime
 
 try:
     import cPickle
     pickle = cPickle
 except:
     cPickle = None
     import pickle
 
 import zmq
 from zmq.utils import jsonapi
 from zmq.eventloop.ioloop import IOLoop
 from zmq.eventloop.zmqstream import ZMQStream
 
 from IPython.config.configurable import Configurable, LoggingConfigurable
 from IPython.utils.importstring import import_item
 from IPython.utils.jsonutil import extract_dates, squash_dates, date_default
 from IPython.utils.traitlets import (CBytes, Unicode, Bool, Any, Instance, Set,
                                         DottedObjectName)
 
 #-----------------------------------------------------------------------------
 # utility functions
 #-----------------------------------------------------------------------------
 
 def squash_unicode(obj):
     """coerce unicode back to bytestrings."""
     if isinstance(obj,dict):
         for key in obj.keys():
             obj[key] = squash_unicode(obj[key])
             if isinstance(key, unicode):
                 obj[squash_unicode(key)] = obj.pop(key)
     elif isinstance(obj, list):
         for i,v in enumerate(obj):
             obj[i] = squash_unicode(v)
     elif isinstance(obj, unicode):
         obj = obj.encode('utf8')
     return obj
 
 #-----------------------------------------------------------------------------
 # globals and defaults
 #-----------------------------------------------------------------------------
 key = 'on_unknown' if jsonapi.jsonmod.__name__ == 'jsonlib' else 'default'
 json_packer = lambda obj: jsonapi.dumps(obj, **{key:date_default})
 json_unpacker = lambda s: extract_dates(jsonapi.loads(s))
 
 pickle_packer = lambda o: pickle.dumps(o,-1)
 pickle_unpacker = pickle.loads
 
 default_packer = json_packer
 default_unpacker = json_unpacker
 
 
 DELIM=b"<IDS|MSG>"
 
 #-----------------------------------------------------------------------------
 # Classes
 #-----------------------------------------------------------------------------
 
 class SessionFactory(LoggingConfigurable):
     """The Base class for configurables that have a Session, Context, logger,
     and IOLoop.
     """
     
     logname = Unicode('')
     def _logname_changed(self, name, old, new):
         self.log = logging.getLogger(new)
     
     # not configurable:
     context = Instance('zmq.Context')
     def _context_default(self):
         return zmq.Context.instance()
     
     session = Instance('IPython.zmq.session.Session')
     
     loop = Instance('zmq.eventloop.ioloop.IOLoop', allow_none=False)
     def _loop_default(self):
         return IOLoop.instance()
     
     def __init__(self, **kwargs):
         super(SessionFactory, self).__init__(**kwargs)
         
         if self.session is None:
             # construct the session
             self.session = Session(**kwargs)
     
 
 class Message(object):
     """A simple message object that maps dict keys to attributes.
 
     A Message can be created from a dict and a dict from a Message instance
     simply by calling dict(msg_obj)."""
     
     def __init__(self, msg_dict):
         dct = self.__dict__
         for k, v in dict(msg_dict).iteritems():
             if isinstance(v, dict):
                 v = Message(v)
             dct[k] = v
 
     # Having this iterator lets dict(msg_obj) work out of the box.
     def __iter__(self):
         return iter(self.__dict__.iteritems())
     
     def __repr__(self):
         return repr(self.__dict__)
 
     def __str__(self):
         return pprint.pformat(self.__dict__)
 
     def __contains__(self, k):
         return k in self.__dict__
 
     def __getitem__(self, k):
         return self.__dict__[k]
 
 
 def msg_header(msg_id, msg_type, username, session):
     date = datetime.now()
     return locals()
 
 def extract_header(msg_or_header):
     """Given a message or header, return the header."""
     if not msg_or_header:
         return {}
     try:
         # See if msg_or_header is the entire message.
         h = msg_or_header['header']
     except KeyError:
         try:
             # See if msg_or_header is just the header
             h = msg_or_header['msg_id']
         except KeyError:
             raise
         else:
             h = msg_or_header
     if not isinstance(h, dict):
         h = dict(h)
     return h
 
 class Session(Configurable):
     """Object for handling serialization and sending of messages.
     
     The Session object handles building messages and sending them
     with ZMQ sockets or ZMQStream objects.  Objects can communicate with each
     other over the network via Session objects, and only need to work with the
     dict-based IPython message spec. The Session will handle 
     serialization/deserialization, security, and metadata.
     
     Sessions support configurable serialiization via packer/unpacker traits,
     and signing with HMAC digests via the key/keyfile traits.
     
     Parameters
     ----------
     
     debug : bool
         whether to trigger extra debugging statements
     packer/unpacker : str : 'json', 'pickle' or import_string
         importstrings for methods to serialize message parts.  If just
         'json' or 'pickle', predefined JSON and pickle packers will be used.
         Otherwise, the entire importstring must be used.
         
         The functions must accept at least valid JSON input, and output *bytes*.
         
         For example, to use msgpack:
         packer = 'msgpack.packb', unpacker='msgpack.unpackb'
     pack/unpack : callables
         You can also set the pack/unpack callables for serialization directly.
     session : bytes
         the ID of this Session object.  The default is to generate a new UUID.
     username : unicode
         username added to message headers.  The default is to ask the OS.
     key : bytes
         The key used to initialize an HMAC signature.  If unset, messages
         will not be signed or checked.
     keyfile : filepath
         The file containing a key.  If this is set, `key` will be initialized
         to the contents of the file.
     
     """
     
     debug=Bool(False, config=True, help="""Debug output in the Session""")
     
     packer = DottedObjectName('json',config=True,
             help="""The name of the packer for serializing messages.
             Should be one of 'json', 'pickle', or an import name
             for a custom callable serializer.""")
     def _packer_changed(self, name, old, new):
         if new.lower() == 'json':
             self.pack = json_packer
             self.unpack = json_unpacker
         elif new.lower() == 'pickle':
             self.pack = pickle_packer
             self.unpack = pickle_unpacker
         else:
             self.pack = import_item(str(new))
 
     unpacker = DottedObjectName('json', config=True,
         help="""The name of the unpacker for unserializing messages.
         Only used with custom functions for `packer`.""")
     def _unpacker_changed(self, name, old, new):
         if new.lower() == 'json':
             self.pack = json_packer
             self.unpack = json_unpacker
         elif new.lower() == 'pickle':
             self.pack = pickle_packer
             self.unpack = pickle_unpacker
         else:
             self.unpack = import_item(str(new))
         
     session = CBytes(b'', config=True,
         help="""The UUID identifying this session.""")
     def _session_default(self):
         return bytes(uuid.uuid4())
     
     username = Unicode(os.environ.get('USER',u'username'), config=True,
         help="""Username for the Session. Default is your system username.""")
     
     # message signature related traits:
     key = CBytes(b'', config=True,
         help="""execution key, for extra authentication.""")
     def _key_changed(self, name, old, new):
         if new:
             self.auth = hmac.HMAC(new)
         else:
             self.auth = None
     auth = Instance(hmac.HMAC)
     digest_history = Set()
     
     keyfile = Unicode('', config=True,
         help="""path to file containing execution key.""")
     def _keyfile_changed(self, name, old, new):
         with open(new, 'rb') as f:
             self.key = f.read().strip()
 
     pack = Any(default_packer) # the actual packer function
     def _pack_changed(self, name, old, new):
         if not callable(new):
             raise TypeError("packer must be callable, not %s"%type(new))
     
     unpack = Any(default_unpacker) # the actual packer function
     def _unpack_changed(self, name, old, new):
         # unpacker is not checked - it is assumed to be 
         if not callable(new):
             raise TypeError("unpacker must be callable, not %s"%type(new))
 
     def __init__(self, **kwargs):
         """create a Session object
         
         Parameters
         ----------
 
         debug : bool
             whether to trigger extra debugging statements
         packer/unpacker : str : 'json', 'pickle' or import_string
             importstrings for methods to serialize message parts.  If just
             'json' or 'pickle', predefined JSON and pickle packers will be used.
             Otherwise, the entire importstring must be used.
 
             The functions must accept at least valid JSON input, and output
             *bytes*.
 
             For example, to use msgpack:
             packer = 'msgpack.packb', unpacker='msgpack.unpackb'
         pack/unpack : callables
             You can also set the pack/unpack callables for serialization
             directly.
         session : bytes
             the ID of this Session object.  The default is to generate a new 
             UUID.
         username : unicode
             username added to message headers.  The default is to ask the OS.
         key : bytes
             The key used to initialize an HMAC signature.  If unset, messages
             will not be signed or checked.
         keyfile : filepath
             The file containing a key.  If this is set, `key` will be 
             initialized to the contents of the file.
         """
         super(Session, self).__init__(**kwargs)
         self._check_packers()
         self.none = self.pack({})
 
     @property
     def msg_id(self):
         """always return new uuid"""
         return str(uuid.uuid4())
 
     def _check_packers(self):
         """check packers for binary data and datetime support."""
         pack = self.pack
         unpack = self.unpack
         
         # check simple serialization
         msg = dict(a=[1,'hi'])
         try:
             packed = pack(msg)
         except Exception:
             raise ValueError("packer could not serialize a simple message")
         
         # ensure packed message is bytes
         if not isinstance(packed, bytes):
             raise ValueError("message packed to %r, but bytes are required"%type(packed))
         
         # check that unpack is pack's inverse
         try:
             unpacked = unpack(packed)
         except Exception:
             raise ValueError("unpacker could not handle the packer's output")
         
         # check datetime support
         msg = dict(t=datetime.now())
         try:
             unpacked = unpack(pack(msg))
         except Exception:
             self.pack = lambda o: pack(squash_dates(o))
             self.unpack = lambda s: extract_dates(unpack(s))
     
     def msg_header(self, msg_type):
         return msg_header(self.msg_id, msg_type, self.username, self.session)
 
     def msg(self, msg_type, content=None, parent=None, subheader=None, header=None):
         """Return the nested message dict.
 
         This format is different from what is sent over the wire. The
         serialize/unserialize methods converts this nested message dict to the wire
         format, which is a list of message parts.
         """
         msg = {}
         msg['header'] = self.msg_header(msg_type) if header is None else header
         msg['parent_header'] = {} if parent is None else extract_header(parent)
         msg['content'] = {} if content is None else content
         sub = {} if subheader is None else subheader
         msg['header'].update(sub)
         return msg
 
     def sign(self, msg_list):
         """Sign a message with HMAC digest. If no auth, return b''.
 
         Parameters
         ----------
         msg_list : list
             The [p_header,p_parent,p_content] part of the message list. 
         """
         if self.auth is None:
             return b''
         h = self.auth.copy()
         for m in msg_list:
             h.update(m)
         return h.hexdigest()
     
     def serialize(self, msg, ident=None):
         """Serialize the message components to bytes.
 
         This is roughly the inverse of unserialize. The serialize/unserialize
         methods work with full message lists, whereas pack/unpack work with
         the individual message parts in the message list.
 
         Parameters
         ----------
         msg : dict or Message
             The nexted message dict as returned by the self.msg method.
 
         Returns
         -------
         msg_list : list
             The list of bytes objects to be sent with the format:
             [ident1,ident2,...,DELIM,HMAC,p_header,p_parent,p_content,
              buffer1,buffer2,...]. In this list, the p_* entities are
             the packed or serialized versions, so if JSON is used, these
             are uft8 encoded JSON strings.
         """
         content = msg.get('content', {})
         if content is None:
             content = self.none
         elif isinstance(content, dict):
             content = self.pack(content)
         elif isinstance(content, bytes):
             # content is already packed, as in a relayed message
             pass
         elif isinstance(content, unicode):
             # should be bytes, but JSON often spits out unicode
             content = content.encode('utf8')
         else:
             raise TypeError("Content incorrect type: %s"%type(content))
         
         real_message = [self.pack(msg['header']), 
                         self.pack(msg['parent_header']), 
                         content
         ]
         
         to_send = []
 
         if isinstance(ident, list):
             # accept list of idents
             to_send.extend(ident)
         elif ident is not None:
             to_send.append(ident)
         to_send.append(DELIM)
         
         signature = self.sign(real_message)
         to_send.append(signature)
         
         to_send.extend(real_message)
 
         return to_send
         
     def send(self, stream, msg_or_type, content=None, parent=None, ident=None,
              buffers=None, subheader=None, track=False, header=None):
         """Build and send a message via stream or socket.
 
         The message format used by this function internally is as follows:
 
         [ident1,ident2,...,DELIM,HMAC,p_header,p_parent,p_content,
          buffer1,buffer2,...] 
 
         The serialize/unserialize methods convert the nested message dict into this
         format.
 
         Parameters
         ----------
         
         stream : zmq.Socket or ZMQStream
             The socket-like object used to send the data.
         msg_or_type : str or Message/dict
             Normally, msg_or_type will be a msg_type unless a message is being 
             sent more than once. If a header is supplied, this can be set to
             None and the msg_type will be pulled from the header.
         
         content : dict or None
             The content of the message (ignored if msg_or_type is a message).
         header : dict or None
             The header dict for the message (ignores if msg_to_type is a message).
         parent : Message or dict or None
             The parent or parent header describing the parent of this message
             (ignored if msg_or_type is a message).
         ident : bytes or list of bytes
             The zmq.IDENTITY routing path.
         subheader : dict or None
             Extra header keys for this message's header (ignored if msg_or_type
             is a message).
         buffers : list or None
             The already-serialized buffers to be appended to the message.
         track : bool
             Whether to track.  Only for use with Sockets, because ZMQStream
             objects cannot track messages.
         
         Returns
         -------
         msg : dict
             The constructed message.
         (msg,tracker) : (dict, MessageTracker)
             if track=True, then a 2-tuple will be returned, 
             the first element being the constructed
             message, and the second being the MessageTracker
             
         """
 
         if not isinstance(stream, (zmq.Socket, ZMQStream)):
             raise TypeError("stream must be Socket or ZMQStream, not %r"%type(stream))
         elif track and isinstance(stream, ZMQStream):
             raise TypeError("ZMQStream cannot track messages")
         
         if isinstance(msg_or_type, (Message, dict)):
             # We got a Message or message dict, not a msg_type so don't
             # build a new Message.
             msg = msg_or_type
         else:
             msg = self.msg(msg_or_type, content=content, parent=parent, 
                            subheader=subheader, header=header)
 
         buffers = [] if buffers is None else buffers
         to_send = self.serialize(msg, ident)
         flag = 0
         if buffers:
             flag = zmq.SNDMORE
             _track = False
         else:
             _track=track
         if track:
             tracker = stream.send_multipart(to_send, flag, copy=False, track=_track)
         else:
             tracker = stream.send_multipart(to_send, flag, copy=False)
         for b in buffers[:-1]:
             stream.send(b, flag, copy=False)
         if buffers:
             if track:
                 tracker = stream.send(buffers[-1], copy=False, track=track)
             else:
                 tracker = stream.send(buffers[-1], copy=False)
 
         # omsg = Message(msg)
         if self.debug:
             pprint.pprint(msg)
             pprint.pprint(to_send)
             pprint.pprint(buffers)
         
         msg['tracker'] = tracker
         
         return msg
 
     def send_raw(self, stream, msg_list, flags=0, copy=True, ident=None):
         """Send a raw message via ident path.
 
         This method is used to send a already serialized message.
 
         Parameters
         ----------
         stream : ZMQStream or Socket
             The ZMQ stream or socket to use for sending the message.
         msg_list : list
             The serialized list of messages to send. This only includes the
             [p_header,p_parent,p_content,buffer1,buffer2,...] portion of
             the message.
         ident : ident or list
             A single ident or a list of idents to use in sending.
         """
         to_send = []
         if isinstance(ident, bytes):
             ident = [ident]
         if ident is not None:
             to_send.extend(ident)
 
         to_send.append(DELIM)
         to_send.append(self.sign(msg_list))
         to_send.extend(msg_list)
         stream.send_multipart(msg_list, flags, copy=copy)
     
     def recv(self, socket, mode=zmq.NOBLOCK, content=True, copy=True):
         """Receive and unpack a message.
 
         Parameters
         ----------
         socket : ZMQStream or Socket
             The socket or stream to use in receiving.
 
         Returns
         -------
         [idents], msg
             [idents] is a list of idents and msg is a nested message dict of
             same format as self.msg returns.
         """
         if isinstance(socket, ZMQStream):
             socket = socket.socket
         try:
             msg_list = socket.recv_multipart(mode)
         except zmq.ZMQError as e:
             if e.errno == zmq.EAGAIN:
                 # We can convert EAGAIN to None as we know in this case
                 # recv_multipart won't return None.
                 return None,None
             else:
                 raise
         # split multipart message into identity list and message dict
         # invalid large messages can cause very expensive string comparisons
         idents, msg_list = self.feed_identities(msg_list, copy)
         try:
             return idents, self.unserialize(msg_list, content=content, copy=copy)
         except Exception as e:
-            print (idents, msg_list)
             # TODO: handle it
             raise e
     
     def feed_identities(self, msg_list, copy=True):
         """Split the identities from the rest of the message.
 
         Feed until DELIM is reached, then return the prefix as idents and
         remainder as msg_list. This is easily broken by setting an IDENT to DELIM,
         but that would be silly.
         
         Parameters
         ----------
         msg_list : a list of Message or bytes objects
             The message to be split.
         copy : bool
             flag determining whether the arguments are bytes or Messages
         
         Returns
         -------
         (idents, msg_list) : two lists
             idents will always be a list of bytes, each of which is a ZMQ
             identity. msg_list will be a list of bytes or zmq.Messages of the
             form [HMAC,p_header,p_parent,p_content,buffer1,buffer2,...] and
             should be unpackable/unserializable via self.unserialize at this
             point.
         """
         if copy:
             idx = msg_list.index(DELIM)
             return msg_list[:idx], msg_list[idx+1:]
         else:
             failed = True
             for idx,m in enumerate(msg_list):
                 if m.bytes == DELIM:
                     failed = False
                     break
             if failed:
                 raise ValueError("DELIM not in msg_list")
             idents, msg_list = msg_list[:idx], msg_list[idx+1:]
             return [m.bytes for m in idents], msg_list
     
     def unserialize(self, msg_list, content=True, copy=True):
         """Unserialize a msg_list to a nested message dict.
 
         This is roughly the inverse of serialize. The serialize/unserialize
         methods work with full message lists, whereas pack/unpack work with
         the individual message parts in the message list.
 
         Parameters:
         -----------
         msg_list : list of bytes or Message objects
             The list of message parts of the form [HMAC,p_header,p_parent,
             p_content,buffer1,buffer2,...].
         content : bool (True)
             Whether to unpack the content dict (True), or leave it packed
             (False).
         copy : bool (True)
             Whether to return the bytes (True), or the non-copying Message
             object in each place (False).
 
         Returns
         -------
         msg : dict
             The nested message dict with top-level keys [header, parent_header,
             content, buffers].
         """
         minlen = 4
         message = {}
         if not copy:
             for i in range(minlen):
                 msg_list[i] = msg_list[i].bytes
         if self.auth is not None:
             signature = msg_list[0]
+            if not signature:
+                raise ValueError("Unsigned Message")
             if signature in self.digest_history:
                 raise ValueError("Duplicate Signature: %r"%signature)
             self.digest_history.add(signature)
             check = self.sign(msg_list[1:4])
             if not signature == check:
                 raise ValueError("Invalid Signature: %r"%signature)
         if not len(msg_list) >= minlen:
             raise TypeError("malformed message, must have at least %i elements"%minlen)
         message['header'] = self.unpack(msg_list[1])
         message['parent_header'] = self.unpack(msg_list[2])
         if content:
             message['content'] = self.unpack(msg_list[3])
         else:
             message['content'] = msg_list[3]
         
         message['buffers'] = msg_list[4:]
         return message
 
 def test_msg2obj():
     am = dict(x=1)
     ao = Message(am)
     assert ao.x == am['x']
 
     am['y'] = dict(z=1)
     ao = Message(am)
     assert ao.y.z == am['y']['z']
     
     k1, k2 = 'y', 'z'
     assert ao[k1][k2] == am[k1][k2]
     
     am2 = dict(ao)
     assert am['x'] == am2['x']
     assert am['y']['z'] == am2['y']['z']
 

docs/source/parallel/parallel_security.txt

@@ -1,255 +1,254 b''
 .. _parallelsecurity:
 
 ===========================
 Security details of IPython
 ===========================
 
 .. note::
 
     This section is not thorough, and IPython.zmq needs a thorough security
     audit.
 
 IPython's :mod:`IPython.zmq` package exposes the full power of the
 Python interpreter over a TCP/IP network for the purposes of parallel
 computing. This feature brings up the important question of IPython's security
 model. This document gives details about this model and how it is implemented
 in IPython's architecture.
 
 Process and network topology
 ============================
 
 To enable parallel computing, IPython has a number of different processes that
 run. These processes are discussed at length in the IPython documentation and
 are summarized here:
 
 * The IPython *engine*.  This process is a full blown Python
   interpreter in which user code is executed.  Multiple
   engines are started to make parallel computing possible.
 * The IPython *hub*.  This process monitors a set of 
   engines and schedulers, and keeps track of the state of the processes. It listens
   for registration connections from engines and clients, and monitor connections
   from schedulers.
 * The IPython *schedulers*. This is a set of processes that relay commands and results
   between clients and engines. They are typically on the same machine as the controller,
   and listen for connections from engines and clients, but connect to the Hub.
 * The IPython *client*.  This process is typically an
   interactive Python process that is used to coordinate the
   engines to get a parallel computation done.
 
 Collectively, these processes are called the IPython *cluster*, and the hub and schedulers
 together are referred to as the *controller*.
 
 
 These processes communicate over any transport supported by ZeroMQ (tcp,pgm,infiniband,ipc)
 with a well defined topology. The IPython hub and schedulers listen on sockets. Upon
 starting, an engine connects to a hub and registers itself, which then informs the engine
 of the connection information for the schedulers, and the engine then connects to the
 schedulers. These engine/hub and engine/scheduler connections persist for the
 lifetime of each engine.
 
 The IPython client also connects to the controller processes using a number of socket
 connections. As of writing, this is one socket per scheduler (4), and 3 connections to the
 hub for a total of 7. These connections persist for the lifetime of the client only.
 
 A given IPython controller and set of engines engines typically has a relatively
 short lifetime. Typically this lifetime corresponds to the duration of a single parallel
 simulation performed by a single user. Finally, the hub, schedulers, engines, and client
 processes typically execute with the permissions of that same user. More specifically, the
 controller and engines are *not* executed as root or with any other superuser permissions.
 
 Application logic
 =================
 
 When running the IPython kernel to perform a parallel computation, a user
 utilizes the IPython client to send Python commands and data through the
 IPython schedulers to the IPython engines, where those commands are executed
 and the data processed. The design of IPython ensures that the client is the
 only access point for the capabilities of the engines. That is, the only way
 of addressing the engines is through a client.
 
 A user can utilize the client to instruct the IPython engines to execute
 arbitrary Python commands. These Python commands can include calls to the
 system shell, access the filesystem, etc., as required by the user's
 application code. From this perspective, when a user runs an IPython engine on
 a host, that engine has the same capabilities and permissions as the user
 themselves (as if they were logged onto the engine's host with a terminal).
 
 Secure network connections
 ==========================
 
 Overview
 --------
 
 ZeroMQ provides exactly no security. For this reason, users of IPython must be very
 careful in managing connections, because an open TCP/IP socket presents access to
 arbitrary execution as the user on the engine machines. As a result, the default behavior
 of controller processes is to only listen for clients on the loopback interface, and the
 client must establish SSH tunnels to connect to the controller processes.
 
 .. warning:: 
 
     If the controller's loopback interface is untrusted, then IPython should be considered
     vulnerable, and this extends to the loopback of all connected clients, which have
     opened a loopback port that is redirected to the controller's loopback port.
 
 
 SSH
 ---
 
 Since ZeroMQ provides no security, SSH tunnels are the primary source of secure
 connections. A connector file, such as `ipcontroller-client.json`, will contain
 information for connecting to the controller, possibly including the address of an
 ssh-server through with the client is to tunnel. The Client object then creates tunnels
 using either [OpenSSH]_ or [Paramiko]_, depending on the platform. If users do not wish to
 use OpenSSH or Paramiko, or the tunneling utilities are insufficient, then they may
 construct the tunnels themselves, and simply connect clients and engines as if the
 controller were on loopback on the connecting machine.
 
 .. note::
 
     There is not currently tunneling available for engines.
 
 Authentication
 --------------
 
 To protect users of shared machines, [HMAC]_ digests are used to sign messages, using a
 shared key.
 
 The Session object that handles the message protocol uses a unique key to verify valid
 messages. This can be any value specified by the user, but the default behavior is a
 pseudo-random 128-bit number, as generated by `uuid.uuid4()`. This key is used to
 initialize an HMAC object, which digests all messages, and includes that digest as a
 signature and part of the message. Every message that is unpacked (on Controller, Engine,
 and Client) will also be digested by the receiver, ensuring that the sender's key is the
 same as the receiver's. No messages that do not contain this key are acted upon in any
 way. The key itself is never sent over the network.
 
 There is exactly one shared key per cluster - it must be the same everywhere. Typically,
 the controller creates this key, and stores it in the private connection files
 `ipython-{engine|client}.json`. These files are typically stored in the
 `~/.ipython/profile_<name>/security` directory, and are maintained as readable only by the
 owner, just as is common practice with a user's keys in their `.ssh` directory.
 
 .. warning::
 
-    It is important to note that the key authentication, as emphasized by the use of
-    a uuid rather than generating a key with a cryptographic library, provides a 
-    defense against *accidental* messages more than it does against malicious attacks.
-    If loopback is compromised, it would be trivial for an attacker to intercept messages
-    and deduce the key, as there is no encryption.
+    It is important to note that the signatures protect against unauthorized messages,
+    but, as there is no encryption, provide exactly no protection of data privacy.  It is
+    possible, however, to use a custom serialization scheme (via Session.packer/unpacker
+    traits) that does incorporate your own encryption scheme.
 
 
 
 Specific security vulnerabilities
 =================================
 
 There are a number of potential security vulnerabilities present in IPython's
 architecture. In this section we discuss those vulnerabilities and detail how
 the security architecture described above prevents them from being exploited.
 
 Unauthorized clients
 --------------------
 
 The IPython client can instruct the IPython engines to execute arbitrary
 Python code with the permissions of the user who started the engines. If an
 attacker were able to connect their own hostile IPython client to the IPython
 controller, they could instruct the engines to execute code.
 
 
 On the first level, this attack is prevented by requiring access to the controller's
 ports, which are recommended to only be open on loopback if the controller is on an
 untrusted local network. If the attacker does have access to the Controller's ports, then
 the attack is prevented by the capabilities based client authentication of the execution
 key. The relevant authentication information is encoded into the JSON file that clients
 must present to gain access to the IPython controller. By limiting the distribution of
 those keys, a user can grant access to only authorized persons, just as with SSH keys.
 
 It is highly unlikely that an execution key could be guessed by an attacker
 in a brute force guessing attack. A given instance of the IPython controller
 only runs for a relatively short amount of time (on the order of hours). Thus
 an attacker would have only a limited amount of time to test a search space of
 size 2**128.  For added security, users can have arbitrarily long keys.
 
 .. warning::
 
     If the attacker has gained enough access to intercept loopback connections on *either* the
     controller or client, then a duplicate message can be sent. To protect against this,
     recipients only allow each signature once, and consider duplicates invalid.  However,
     the duplicate message could be sent to *another* recipient using the same key,
     and it would be considered valid.
 
 
 Unauthorized engines
 --------------------
 
 If an attacker were able to connect a hostile engine to a user's controller,
 the user might unknowingly send sensitive code or data to the hostile engine.
 This attacker's engine would then have full access to that code and data.
 
 This type of attack is prevented in the same way as the unauthorized client
 attack, through the usage of the capabilities based authentication scheme.
 
 Unauthorized controllers
 ------------------------
 
 It is also possible that an attacker could try to convince a user's IPython
 client or engine to connect to a hostile IPython controller. That controller
 would then have full access to the code and data sent between the IPython
 client and the IPython engines.
 
 Again, this attack is prevented through the capabilities in a connection file, which
 ensure that a client or engine connects to the correct controller. It is also important to
 note that the connection files also encode the IP address and port that the controller is
 listening on, so there is little chance of mistakenly connecting to a controller running
 on a different IP address and port.
 
 When starting an engine or client, a user must specify the key to use
 for that connection. Thus, in order to introduce a hostile controller, the
 attacker must convince the user to use the key associated with the
 hostile controller. As long as a user is diligent in only using keys from
 trusted sources, this attack is not possible.
 
 .. note::
 
     I may be wrong, the unauthorized controller may be easier to fake than this.
 
 Other security measures
 =======================
 
 A number of other measures are taken to further limit the security risks
 involved in running the IPython kernel.
 
 First, by default, the IPython controller listens on random port numbers.
 While this can be overridden by the user, in the default configuration, an
 attacker would have to do a port scan to even find a controller to attack.
 When coupled with the relatively short running time of a typical controller
 (on the order of hours), an attacker would have to work extremely hard and
 extremely *fast* to even find a running controller to attack.
 
 Second, much of the time, especially when run on supercomputers or clusters,
 the controller is running behind a firewall. Thus, for engines or client to
 connect to the controller:
 
 * The different processes have to all be behind the firewall.
 
 or:
 
 * The user has to use SSH port forwarding to tunnel the
   connections through the firewall.
    
 In either case, an attacker is presented with additional barriers that prevent
 attacking or even probing the system.
 
 Summary
 =======
 
 IPython's architecture has been carefully designed with security in mind. The
 capabilities based authentication model, in conjunction with SSH tunneled
 TCP/IP channels, address the core potential vulnerabilities in the system,
 while still enabling user's to use the system in open networks.
 
 .. [RFC5246] <http://tools.ietf.org/html/rfc5246>
 
 .. [OpenSSH] <http://www.openssh.com/>
 .. [Paramiko] <http://www.lag.net/paramiko/>
 .. [HMAC] <http://tools.ietf.org/html/rfc2104.html>