diff --git a/IPython/html/base/zmqhandlers.py b/IPython/html/base/zmqhandlers.py
index 99b7077..0287502 100644
--- a/IPython/html/base/zmqhandlers.py
+++ b/IPython/html/base/zmqhandlers.py
@@ -23,7 +23,7 @@ from tornado import web
 from tornado import websocket
 
 from IPython.kernel.zmq.session import Session
-from IPython.utils.jsonutil import date_default
+from IPython.utils.jsonutil import date_default, extract_dates
 from IPython.utils.py3compat import PY3, cast_unicode
 
 from .handlers import IPythonHandler
@@ -45,7 +45,9 @@ def serialize_binary_message(msg):
     The message serialized to bytes.
 
     """
-    buffers = msg.pop('buffers')
+    # don't modify msg or buffer list in-place
+    msg = msg.copy()
+    buffers = list(msg.pop('buffers'))
     bmsg = json.dumps(msg, default=date_default).encode('utf8')
     buffers.insert(0, bmsg)
     nbufs = len(buffers)
@@ -72,13 +74,15 @@ def deserialize_binary_message(bmsg):
 
     message dictionary
     """
-    nbufs = struct.unpack('i', bmsg[:4])[0]
+    nbufs = struct.unpack('!i', bmsg[:4])[0]
     offsets = list(struct.unpack('!' + 'i' * nbufs, bmsg[4:4*(nbufs+1)]))
     offsets.append(None)
     bufs = []
     for start, stop in zip(offsets[:-1], offsets[1:]):
         bufs.append(bmsg[start:stop])
-    msg = json.loads(bufs[0])
+    msg = json.loads(bufs[0].decode('utf8'))
+    msg['header'] = extract_dates(msg['header'])
+    msg['parent_header'] = extract_dates(msg['parent_header'])
     msg['buffers'] = bufs[1:]
     return msg
 
@@ -139,14 +143,6 @@ class ZMQStreamHandler(websocket.WebSocketHandler):
         """
         idents, msg_list = self.session.feed_identities(msg_list)
         msg = self.session.deserialize(msg_list)
-        try:
-            msg['header'].pop('date')
-        except KeyError:
-            pass
-        try:
-            msg['parent_header'].pop('date')
-        except KeyError:
-            pass
         if msg['buffers']:
             buf = serialize_binary_message(msg)
             return buf
diff --git a/IPython/html/tests/test_serialize.py b/IPython/html/tests/test_serialize.py
new file mode 100644
index 0000000..7a88b29
--- /dev/null
+++ b/IPython/html/tests/test_serialize.py
@@ -0,0 +1,26 @@
+"""Test serialize/deserialize messages with buffers"""
+
+import os
+
+import nose.tools as nt
+
+from IPython.kernel.zmq.session import Session
+from ..base.zmqhandlers import (
+    serialize_binary_message,
+    deserialize_binary_message,
+)
+
+def test_serialize_binary():
+    s = Session()
+    msg = s.msg('data_pub', content={'a': 'b'})
+    msg['buffers'] = [ os.urandom(3) for i in range(3) ]
+    bmsg = serialize_binary_message(msg)
+    nt.assert_is_instance(bmsg, bytes)
+
+def test_deserialize_binary():
+    s = Session()
+    msg = s.msg('data_pub', content={'a': 'b'})
+    msg['buffers'] = [ os.urandom(2) for i in range(3) ]
+    bmsg = serialize_binary_message(msg)
+    msg2 = deserialize_binary_message(bmsg)
+    nt.assert_equal(msg2, msg)