Merge pull request #3756 from minrk/wiredoc

document the wire protocol

in the messaging doc

also update the digest scheme to use sha256 and make this configurable,
since md5 is the previous default and has been shown to be bad.

More messaging docs to update here:

- [x] remove get/setattr on kernel
- [x] remove crash messages
- [x] completion requests do not behave as documented
- [x] object_info is misdocumented (`name` is actually `oname` in object_info_request)

IPython/consoleapp.py

@@ -265,6 +265,8 b' class IPythonConsoleApp(Configurable):'
                 setattr(self, name, cfg[name])
         if 'key' in cfg:
             self.config.Session.key = str_to_bytes(cfg['key'])
+        if 'signature_scheme' in cfg:
+            self.config.Session.signature_scheme = cfg['signature_scheme']
     
     def init_ssh(self):
         """set up ssh tunnels, if needed."""

IPython/kernel/connect.py

@@ -50,7 +50,9 b' from IPython.utils.traitlets import ('
 #-----------------------------------------------------------------------------
 
 def write_connection_file(fname=None, shell_port=0, iopub_port=0, stdin_port=0, hb_port=0,
-                         control_port=0, ip=LOCALHOST, key=b'', transport='tcp'):
+                         control_port=0, ip=LOCALHOST, key=b'', transport='tcp',
+                         signature_scheme='hmac-sha256',
+                         ):
     """Generates a JSON config file, including the selection of random ports.
     
     Parameters
@@ -78,7 +80,15 b' def write_connection_file(fname=None, shell_port=0, iopub_port=0, stdin_port=0, '
         The ip address the kernel will bind to.
 
     key : str, optional
-        The Session key used for HMAC authentication.
+        The Session key used for message authentication.
+    
+    signature_scheme : str, optional
+        The scheme used for message authentication.
+        This has the form 'digest-hash', where 'digest'
+        is the scheme used for digests, and 'hash' is the name of the hash function
+        used by the digest scheme.
+        Currently, 'hmac' is the only supported digest scheme,
+        and 'sha256' is the default hash function.
 
     """
     # default to temporary connector file
@@ -129,6 +139,7 b' def write_connection_file(fname=None, shell_port=0, iopub_port=0, stdin_port=0, '
     cfg['ip'] = ip
     cfg['key'] = bytes_to_str(key)
     cfg['transport'] = transport
+    cfg['signature_scheme'] = signature_scheme
     
     with open(fname, 'w') as f:
         f.write(json.dumps(cfg, indent=2))
@@ -380,6 +391,7 b' class ConnectionFileMixin(HasTraits):'
     _connection_file_written = Bool(False)
 
     transport = CaselessStrEnum(['tcp', 'ipc'], default_value='tcp', config=True)
+    signature_scheme = Unicode('')
 
     ip = Unicode(LOCALHOST, config=True,
         help="""Set the kernel\'s IP address [default localhost].
@@ -427,6 +439,7 b' class ConnectionFileMixin(HasTraits):'
             stdin_port=self.stdin_port,
             hb_port=self.hb_port,
             control_port=self.control_port,
+            signature_scheme=self.signature_scheme,
         )
 
     def cleanup_connection_file(self):
@@ -463,6 +476,7 b' class ConnectionFileMixin(HasTraits):'
             stdin_port=self.stdin_port, iopub_port=self.iopub_port,
             shell_port=self.shell_port, hb_port=self.hb_port,
             control_port=self.control_port,
+            signature_scheme=self.signature_scheme,
         )
         # write_connection_file also sets default ports:
         for name in port_names:
@@ -479,7 +493,10 b' class ConnectionFileMixin(HasTraits):'
         self.ip = cfg['ip']
         for name in port_names:
             setattr(self, name, cfg[name])
-        self.session.key = str_to_bytes(cfg['key'])
+        if 'key' in cfg:
+            self.session.key = str_to_bytes(cfg['key'])
+        if cfg.get('signature_scheme'):
+            self.session.signature_scheme = cfg['signature_scheme']
 
     #--------------------------------------------------------------------------
     # Creating connected sockets

IPython/kernel/zmq/displayhook.py

@@ -10,7 +10,7 b' from session import extract_header, Session'
 class ZMQDisplayHook(object):
     """A simple displayhook that publishes the object's repr over a ZeroMQ
     socket."""
-    topic=None
+    topic=b'pyout'
 
     def __init__(self, session, pub_socket):
         self.session = session

IPython/kernel/zmq/iostream.py

@@ -44,6 +44,7 b' class OutStream(object):'
         self.session = session
         self.pub_socket = pub_socket
         self.name = name
+        self.topic = b'stream.' + py3compat.cast_bytes(name)
         self.parent_header = {}
         self._new_buffer()
         self._buffer_lock = threading.Lock()

IPython/kernel/zmq/session.py

@@ -24,6 +24,7 b' Authors:'
 # Imports
 #-----------------------------------------------------------------------------
 
+import hashlib
 import hmac
 import logging
 import os
@@ -50,7 +51,9 b' from IPython.utils.importstring import import_item'
 from IPython.utils.jsonutil import extract_dates, squash_dates, date_default
 from IPython.utils.py3compat import str_to_bytes, str_to_unicode
 from IPython.utils.traitlets import (CBytes, Unicode, Bool, Any, Instance, Set,
-                                        DottedObjectName, CUnicode, Dict, Integer)
+                                        DottedObjectName, CUnicode, Dict, Integer,
+                                        TraitError,
+)
 from IPython.kernel.zmq.serialize import MAX_ITEMS, MAX_BYTES
 
 #-----------------------------------------------------------------------------
@@ -308,10 +311,26 b' class Session(Configurable):'
         help="""execution key, for extra authentication.""")
     def _key_changed(self, name, old, new):
         if new:
-            self.auth = hmac.HMAC(new)
+            self.auth = hmac.HMAC(new, digestmod=self.digest_mod)
         else:
             self.auth = None
     
+    signature_scheme = Unicode('hmac-sha256', config=True,
+        help="""The digest scheme used to construct the message signatures.
+        Must have the form 'hmac-HASH'.""")
+    def _signature_scheme_changed(self, name, old, new):
+        if not new.startswith('hmac-'):
+            raise TraitError("signature_scheme must start with 'hmac-', got %r" % new)
+        hash_name = new.split('-', 1)[1]
+        try:
+            self.digest_mod = getattr(hashlib, hash_name)
+        except AttributeError:
+            raise TraitError("hashlib has no such attribute: %s" % hash_name)
+    
+    digest_mod = Any()
+    def _digest_mod_default(self):
+        return hashlib.sha256
+    
     auth = Instance(hmac.HMAC)
     
     digest_history = Set()
@@ -387,6 +406,11 b' class Session(Configurable):'
         key : bytes
             The key used to initialize an HMAC signature.  If unset, messages
             will not be signed or checked.
+        signature_scheme : str
+            The message digest scheme. Currently must be of the form 'hmac-HASH',
+            where 'HASH' is a hashing function available in Python's hashlib.
+            The default is 'hmac-sha256'.
+            This is ignored if 'key' is empty.
         keyfile : filepath
             The file containing a key.  If this is set, `key` will be
             initialized to the contents of the file.

IPython/kernel/zmq/zmqshell.py

@@ -61,7 +61,7 b' class ZMQDisplayPublisher(DisplayPublisher):'
     session = Instance(Session)
     pub_socket = Instance(SocketABC)
     parent_header = Dict({})
-    topic = CBytes(b'displaypub')
+    topic = CBytes(b'display_data')
 
     def set_parent(self, parent):
         """Set the parent for outbound messages."""

IPython/parallel/apps/ipcontrollerapp.py

@@ -251,7 +251,7 b' class IPControllerApp(BaseParallelApplication):'
             ecfg = json.loads(f.read())
         
         # json gives unicode, Session.key wants bytes
-        c.Session.key = ecfg['exec_key'].encode('ascii')
+        c.Session.key = ecfg['key'].encode('ascii')
         
         xport,ip = ecfg['interface'].split('://')
         
@@ -269,7 +269,7 b' class IPControllerApp(BaseParallelApplication):'
         with open(fname) as f:
             ccfg = json.loads(f.read())
         
-        for key in ('exec_key', 'registration', 'pack', 'unpack'):
+        for key in ('key', 'registration', 'pack', 'unpack', 'signature_scheme'):
             assert ccfg[key] == ecfg[key], "mismatch between engine and client info: %r" % key
         
         xport,addr = ccfg['interface'].split('://')
@@ -338,10 +338,11 b' class IPControllerApp(BaseParallelApplication):'
             # save to new json config files
             f = self.factory
             base = {
-                'exec_key'  : f.session.key.decode('ascii'),
+                'key'  : f.session.key.decode('ascii'),
                 'location'  : self.location,
                 'pack'      : f.session.packer,
                 'unpack'    : f.session.unpacker,
+                'signature_scheme' : f.session.signature_scheme,
             }
             
             cdict = {'ssh' : self.ssh_server}

IPython/parallel/apps/ipengineapp.py

@@ -227,9 +227,10 b' class IPEngineApp(BaseParallelApplication):'
         ip = disambiguate_ip_address(ip, location)
         d['interface'] = '%s://%s' % (proto, ip)
         
-        # DO NOT allow override of basic URLs, serialization, or exec_key
+        # DO NOT allow override of basic URLs, serialization, or key
         # JSON file takes top priority there
-        config.Session.key = cast_bytes(d['exec_key'])
+        config.Session.key = cast_bytes(d['key'])
+        config.Session.signature_scheme = d['signature_scheme']
         
         config.EngineFactory.url = d['interface'] + ':%i' % d['registration']
         

IPython/parallel/client/client.py

@@ -462,7 +462,8 b' class Client(HasTraits):'
         # configure and construct the session
         extra_args['packer'] = cfg['pack']
         extra_args['unpacker'] = cfg['unpack']
-        extra_args['key'] = cast_bytes(cfg['exec_key'])
+        extra_args['key'] = cast_bytes(cfg['key'])
+        extra_args['signature_scheme'] = cfg['signature_scheme']
         
         self.session = Session(**extra_args)
 

docs/source/development/messaging.txt

@@ -93,7 +93,7 b' A message is defined by the following four-dictionary structure::'
       'header' : {
                     'msg_id' : uuid,
                     'username' : str,
-                    'session' : uuid
+                    'session' : uuid,
                     # All recognized message type strings are listed below.
                     'msg_type' : str,
          },
@@ -101,16 +101,101 b' A message is defined by the following four-dictionary structure::'
       # In a chain of messages, the header from the parent is copied so that
       # clients can track where messages come from.
       'parent_header' : dict,
-
-      # The actual content of the message must be a dict, whose structure
-      # depends on the message type.
-      'content' : dict,
       
       # Any metadata associated with the message.
       'metadata' : dict,
+      
+      # The actual content of the message must be a dict, whose structure
+      # depends on the message type.
+      'content' : dict,
     }
 
-   
+The Wire Protocol
+=================
+
+
+This message format exists at a high level,
+but does not describe the actual *implementation* at the wire level in zeromq.
+The canonical implementation of the message spec is our :class:`~IPython.kernel.zmq.session.Session` class.
+
+.. note::
+
+    This section should only be relevant to non-Python consumers of the protocol.
+    Python consumers should simply import and use IPython's own implementation of the wire protocol
+    in the :class:`IPython.kernel.zmq.session.Session` object.
+
+Every message is serialized to a sequence of at least six blobs of bytes:
+
+.. sourcecode:: python
+
+    [
+      b'u-u-i-d',         # zmq identity(ies)
+      b'<IDS|MSG>',       # delimiter
+      b'baddad42',        # HMAC signature
+      b'{header}',        # serialized header dict
+      b'{parent_header}', # serialized parent header dict
+      b'{metadata}',      # serialized metadata dict
+      b'{content},        # serialized content dict
+      b'blob',            # extra raw data buffer(s)
+      ...
+    ]
+
+The front of the message is the ZeroMQ routing prefix,
+which can be zero or more socket identities.
+This is every piece of the message prior to the delimiter key ``<IDS|MSG>``.
+In the case of IOPub, there should be just one prefix component,
+which is the topic for IOPub subscribers, e.g. ``pyout``, ``display_data``.
+
+.. note::
+
+    In most cases, the IOPub topics are irrelevant and completely ignored,
+    because frontends just subscribe to all topics.
+    The convention used in the IPython kernel is to use the msg_type as the topic,
+    and possibly extra information about the message, e.g. ``pyout`` or ``stream.stdout``
+
+After the delimiter is the `HMAC`_ signature of the message, used for authentication.
+If authentication is disabled, this should be an empty string.
+By default, the hashing function used for computing these signatures is sha256.
+
+.. _HMAC: http://en.wikipedia.org/wiki/HMAC
+
+.. note::
+
+    To disable authentication and signature checking,
+    set the `key` field of a connection file to an empty string.
+
+The signature is the HMAC hex digest of the concatenation of:
+
+- A shared key (typically the ``key`` field of a connection file)
+- The serialized header dict
+- The serialized parent header dict
+- The serialized metadata dict
+- The serialized content dict
+
+In Python, this is implemented via:
+
+.. sourcecode:: python
+
+    # once:
+    digester = HMAC(key, digestmod=hashlib.sha256)
+    
+    # for each message
+    d = digester.copy()
+    for serialized_dict in (header, parent, metadata, content):
+        d.update(serialized_dict)
+    signature = d.hexdigest()
+
+After the signature is the actual message, always in four frames of bytes.
+The four dictionaries that compose a message are serialized separately,
+in the order of header, parent header, metadata, and content.
+These can be serialized by any function that turns a dict into bytes.
+The default and most common serialization is JSON, but msgpack and pickle
+are common alternatives.
+
+After the serialized dicts are zero to many raw data buffers,
+which can be used by message types that support binary data (mainly apply and data_pub).
+
+
 Python functional API
 =====================
 
@@ -388,73 +473,7 b" When status is 'error', the following extra fields are present::"
 When status is 'abort', there are for now no additional data fields.  This
 happens when the kernel was interrupted by a signal.
 
-Kernel attribute access
------------------------
-
-.. warning::
-
-   This part of the messaging spec is not actually implemented in the kernel
-   yet.
- 
-While this protocol does not specify full RPC access to arbitrary methods of
-the kernel object, the kernel does allow read (and in some cases write) access
-to certain attributes.
-
-The policy for which attributes can be read is: any attribute of the kernel, or
-its sub-objects, that belongs to a :class:`Configurable` object and has been
-declared at the class-level with Traits validation, is in principle accessible
-as long as its name does not begin with a leading underscore.  The attribute
-itself will have metadata indicating whether it allows remote read and/or write
-access.  The message spec follows for attribute read and write requests.
-
-Message type: ``getattr_request``::
-
-    content = {
-        # The (possibly dotted) name of the attribute
-	'name' : str,
-    }
 
-When a ``getattr_request`` fails, there are two possible error types:
-
-- AttributeError: this type of error was raised when trying to access the
-  given name by the kernel itself.  This means that the attribute likely
-  doesn't exist.
-
-- AccessError: the attribute exists but its value is not readable remotely.
-
-
-Message type: ``getattr_reply``::
-
-    content = {
-        # One of ['ok', 'AttributeError', 'AccessError'].
-        'status' : str,
-	# If status is 'ok', a JSON object.
-	'value' : object,
-    }
-
-Message type: ``setattr_request``::
-
-    content = {
-        # The (possibly dotted) name of the attribute
-	'name' : str,
-	
-	# A JSON-encoded object, that will be validated by the Traits
-	# information in the kernel
-	'value' : object,
-    }
-
-When a ``setattr_request`` fails, there are also two possible error types with
-similar meanings  as those of the ``getattr_request`` case, but for writing.
-    
-Message type: ``setattr_reply``::
-
-    content = {
-        # One of ['ok', 'AttributeError', 'AccessError'].
-        'status' : str,
-    }
-
-
-    
 Object information
 ------------------
 
@@ -469,12 +488,12 b' Message type: ``object_info_request``::'
 
     content = {
         # The (possibly dotted) name of the object to be searched in all
-	# relevant namespaces
-        'name' : str,
+        # relevant namespaces
+        'oname' : str,
 
-    	# The level of detail desired.  The default (0) is equivalent to typing
-	# 'x?' at the prompt, 1 is equivalent to 'x??'.
-	'detail_level' : int,
+        # The level of detail desired.  The default (0) is equivalent to typing
+        # 'x?' at the prompt, 1 is equivalent to 'x??'.
+        'detail_level' : int,
     }
 
 The returned information will be a dictionary with keys very similar to the
@@ -529,16 +548,16 b' Message type: ``object_info_reply``::'
     # objects, this field is empty.
     'argspec' : { # The names of all the arguments
                   args : list,
-		  # The name of the varargs (*args), if any
-                  varargs : str,
-		  # The name of the varkw (**kw), if any
-		  varkw : str,
-		  # The values (as strings) of all default arguments.  Note
-		  # that these must be matched *in reverse* with the 'args'
-		  # list above, since the first positional args have no default
-		  # value at all.
-		  defaults : list,
-		  },
+        # The name of the varargs (*args), if any
+                      varargs : str,
+        # The name of the varkw (**kw), if any
+        varkw : str,
+        # The values (as strings) of all default arguments.  Note
+        # that these must be matched *in reverse* with the 'args'
+        # list above, since the first positional args have no default
+        # value at all.
+        defaults : list,
+    },
 
     # For instances, provide the constructor signature (the definition of
     # the __init__ method):
@@ -573,30 +592,44 b' Message type: ``complete_request``::'
 
     content = {
         # The text to be completed, such as 'a.is'
-    'text' : str,
-
-    # The full line, such as 'print a.is'.  This allows completers to
-    # make decisions that may require information about more than just the
-    # current word.
-    'line' : str,
-
-    # The entire block of text where the line is.  This may be useful in the
-    # case of multiline completions where more context may be needed.  Note: if
-    # in practice this field proves unnecessary, remove it to lighten the
-    # messages.
+        # this may be an empty string if the frontend does not do any lexing,
+        # in which case the kernel must figure out the completion
+        # based on 'line' and 'cursor_pos'.
+        'text' : str,
+
+        # The full line, such as 'print a.is'.  This allows completers to
+        # make decisions that may require information about more than just the
+        # current word.
+        'line' : str,
+
+        # The entire block of text where the line is.  This may be useful in the
+        # case of multiline completions where more context may be needed.  Note: if
+        # in practice this field proves unnecessary, remove it to lighten the
+        # messages.
     
-    'block' : str,
+        'block' : str or null/None,
 
-    # The position of the cursor where the user hit 'TAB' on the line.
-    'cursor_pos' : int,
+        # The position of the cursor where the user hit 'TAB' on the line.
+        'cursor_pos' : int,
     }
 
 Message type: ``complete_reply``::
 
     content = {
-        # The list of all matches to the completion request, such as
+    # The list of all matches to the completion request, such as
     # ['a.isalnum', 'a.isalpha'] for the above example.
-    'matches' : list
+    'matches' : list,
+    
+    # the substring of the matched text
+    # this is typically the common prefix of the matches,
+    # and the text that is already in the block that would be replaced by the full completion.
+    # This would be 'a.is' in the above example.
+    'text' : str,
+    
+    # status should be 'ok' unless an exception was raised during the request,
+    # in which case it should be 'error', along with the usual error message content
+    # in other messages.
+    'status' : 'ok'
     }
 
     
@@ -671,21 +704,21 b' Message type: ``connect_request``::'
 Message type: ``connect_reply``::
 
     content = {
-        'shell_port' : int  # The port the shell ROUTER socket is listening on.
-        'iopub_port' : int   # The port the PUB socket is listening on.
-        'stdin_port' : int   # The port the stdin ROUTER socket is listening on.
-        'hb_port' : int    # The port the heartbeat socket is listening on.
+        'shell_port' : int,   # The port the shell ROUTER socket is listening on.
+        'iopub_port' : int,   # The port the PUB socket is listening on.
+        'stdin_port' : int,   # The port the stdin ROUTER socket is listening on.
+        'hb_port' : int,      # The port the heartbeat socket is listening on.
     }
 
 
 Kernel info
 -----------
 
-If a client needs to know what protocol the kernel supports, it can
-ask version number of the messaging protocol supported by the kernel.
-This message can be used to fetch other core information of the
+If a client needs to know information about the kernel, it can
+make a request of the kernel's information.
+This message can be used to fetch core information of the
 kernel, including language (e.g., Python), language version number and
-IPython version number.
+IPython version number, and the IPython message spec version number.
 
 Message type: ``kernel_info_request``::
 
@@ -741,9 +774,6 b' Upon their own shutdown, client applications will typically execute a last'
 minute sanity check and forcefully terminate any kernel that is still alive, to
 avoid leaving stray processes in the user's machine.
 
-For both shutdown request and reply, there is no actual content that needs to
-be sent, so the content dict is empty.
-
 Message type: ``shutdown_request``::
 
     content = {
@@ -772,18 +802,13 b' Streams (stdout, stderr, etc)'
 Message type: ``stream``::
 
     content = {
-        # The name of the stream is one of 'stdin', 'stdout', 'stderr'
+        # The name of the stream is one of 'stdout', 'stderr'
         'name' : str,
     
         # The data is an arbitrary string to be written to that stream
         'data' : str,
     }
 
-When a kernel receives a raw_input call, it should also broadcast it on the pub
-socket with the names 'stdin' and 'stdin_reply'.  This will allow other clients
-to monitor/display kernel interactions and possibly replay them to their user
-or otherwise expose them.
-
 Display Data
 ------------
 
@@ -956,39 +981,6 b' Message type: ``status``::'
         execution_state : ('busy', 'idle', 'starting')
     }
 
-Kernel crashes
---------------
-
-When the kernel has an unexpected exception, caught by the last-resort
-sys.excepthook, we should broadcast the crash handler's output before exiting.
-This will allow clients to notice that a kernel died, inform the user and
-propose further actions.
-
-Message type: ``crash``::
-
-    content = {
-       # Similarly to the 'error' case for execute_reply messages, this will
-       # contain ename, evalue and traceback fields.
-
-       # An additional field with supplementary information such as where to
-       # send the crash message
-       'info' : str,
-    }
-
-
-Future ideas
-------------
-    
-Other potential message types, currently unimplemented, listed below as ideas.
-    
-Message type: ``file``::
-
-    content = {
-    'path' : 'cool.jpg',
-    'mimetype' : str,
-    'data' : str,
-    }
-
 
 Messages on the stdin ROUTER/DEALER sockets
 ===========================================