From d467b8c567c9508dbea7c9bd90aa4c8eae4e55c4 2015-03-24 15:15:30
From: Jason Grout <jgrout6@bloomberg.net>
Date: 2015-03-24 15:15:30
Subject: [PATCH] work-in-progress for custom js serializers

---

diff --git a/IPython/html/static/widgets/js/types.js b/IPython/html/static/widgets/js/types.js
new file mode 100644
index 0000000..5cc10b3
--- /dev/null
+++ b/IPython/html/static/widgets/js/types.js
@@ -0,0 +1,50 @@
+// Copyright (c) IPython Development Team.
+// Distributed under the terms of the Modified BSD License.
+
+define([
+    "base/js/utils"
+], function(utils){
+
+    return {
+        widget_serialization: {
+            deserialize: function deserialize_models(value, model) {
+                /**
+                 * Replace model ids with models recursively.
+                 */
+                var unpacked;
+                if ($.isArray(value)) {
+                    unpacked = [];
+                    _.each(value, function(sub_value, key) {
+                        unpacked.push(deserialize_models(sub_value, model));
+                    });
+                    return Promise.all(unpacked);
+                } else if (value instanceof Object) {
+                    unpacked = {};
+                    _.each(value, function(sub_value, key) {
+                        unpacked[key] = deserialize_models(sub_value, model);
+                    });
+                    return utils.resolve_promises_dict(unpacked);
+                } else if (typeof value === 'string' && value.slice(0,10) === "IPY_MODEL_") {
+                    // get_model returns a promise already
+                    return model.widget_manager.get_model(value.slice(10, value.length));
+                } else {
+                    return Promise.resolve(value);
+                }
+            },
+        },
+        
+        list_of_numbers: {
+            deserialize: function (value, model) {
+                /* value is a DataView */
+                /* create a float64 typed array */
+                return new Float64Array(value.buffer)
+            },
+            serialize: function (value, model) {
+                return value;
+            },
+        }
+    }
+
+    
+
+});
diff --git a/IPython/html/static/widgets/js/widget.js b/IPython/html/static/widgets/js/widget.js
index e5f758d..b837f57 100644
--- a/IPython/html/static/widgets/js/widget.js
+++ b/IPython/html/static/widgets/js/widget.js
@@ -32,6 +32,7 @@ define(["widgets/js/manager",
             this.state_lock = null;
             this.id = model_id;
             this.views = {};
+            this.serializers = {};
             this._resolve_received_state = {};
 
             if (comm !== undefined) {
@@ -62,13 +63,13 @@ define(["widgets/js/manager",
             return Backbone.Model.apply(this);
         },
 
-        send: function (content, callbacks) {
+        send: function (content, callbacks, buffers) {
             /**
              * Send a custom msg over the comm.
              */
             if (this.comm !== undefined) {
                 var data = {method: 'custom', content: content};
-                this.comm.send(data, callbacks);
+                this.comm.send(data, callbacks, {}, buffers);
                 this.pending_msgs++;
             }
         },
@@ -136,12 +137,37 @@ define(["widgets/js/manager",
              * Handle incoming comm msg.
              */
             var method = msg.content.data.method;
+            
             var that = this;
             switch (method) {
                 case 'update':
                     this.state_change = this.state_change
                         .then(function() {
-                            return that.set_state(msg.content.data.state);
+                            var state = msg.content.data.state || {};
+                            var buffer_keys = msg.content.data.buffers || [];
+                            var buffers = msg.buffers || [];
+                            var metadata = msg.content.data.metadata || {};
+                            var i,k;
+                            for (var i=0; i<buffer_keys.length; i++) {
+                                k = buffer_keys[i];
+                                state[k] = buffers[i];
+                            }
+
+                            // for any metadata specifying a deserializer, set the
+                            // state to a promise that resolves to the deserialized version
+                            // also, store the serialization function for the attribute
+                            var keys = Object.keys(metadata);
+                            for (var i=0; i<keys.length; i++) {
+                                k = keys[i];
+                                if (metadata[k] && metadata[k].serialization) {
+                                    that.serializers[k] = utils.load_class.apply(that,
+                                                                                 metadata[k].serialization);
+                                    state[k] = that.deserialize(that.serializers[k], state[k]);
+                                }
+                            }
+                            return utils.resolve_promises_dict(state);
+                        }).then(function(state) {
+                            return that.set_state(state);
                         }).catch(utils.reject("Couldn't process update msg for model id '" + String(that.id) + "'", true))
                         .then(function() {
                             var parent_id = msg.parent_header.msg_id;
@@ -152,7 +178,7 @@ define(["widgets/js/manager",
                         }).catch(utils.reject("Couldn't resolve state request promise", true));
                     break;
                 case 'custom':
-                    this.trigger('msg:custom', msg.content.data.content);
+                    this.trigger('msg:custom', msg.content.data.content, msg.buffers);
                     break;
                 case 'display':
                     this.state_change = this.state_change.then(function() {
@@ -162,30 +188,39 @@ define(["widgets/js/manager",
             }
         },
 
+        deserialize: function(serializer, value) {
+            // given a serializer dict and a value,
+            // return a promise for the deserialized value
+            var that = this;
+            return serializer.then(function(s) {
+                if (s.deserialize) {
+                    return s.deserialize(value, that);
+                } else {
+                    return value;
+                }
+            });
+        },
+
         set_state: function (state) {
             var that = this;
             // Handle when a widget is updated via the python side.
-            return this._unpack_models(state).then(function(state) {
+            return new Promise(function(resolve, reject) {
                 that.state_lock = state;
                 try {
                     WidgetModel.__super__.set.call(that, state);
                 } finally {
                     that.state_lock = null;
                 }
+                resolve();
             }).catch(utils.reject("Couldn't set model state", true));
         },
 
         get_state: function() {
             // Get the serializable state of the model.
-            var state = this.toJSON();
-            for (var key in state) {
-                if (state.hasOwnProperty(key)) {
-                    state[key] = this._pack_models(state[key]);
-                }
-            }
-            return state;
+            // Equivalent to Backbone.Model.toJSON()
+            return _.clone(this.attributes);
         },
-
+        
         _handle_status: function (msg, callbacks) {
             /**
              * Handle status msgs.
@@ -243,6 +278,19 @@ define(["widgets/js/manager",
              * Handle sync to the back-end.  Called when a model.save() is called.
              *
              * Make sure a comm exists.
+
+             * Parameters
+             * ----------
+             * method : create, update, patch, delete, read
+             *   create/update always send the full attribute set
+             *   patch - only send attributes listed in options.attrs, and if we are queuing
+             *     up messages, combine with previous messages that have not been sent yet
+             * model : the model we are syncing
+             *   will normally be the same as `this`
+             * options : dict
+             *   the `attrs` key, if it exists, gives an {attr: value} dict that should be synced,
+             *   otherwise, sync all attributes
+             * 
              */
             var error = options.error || function() {
                 console.error('Backbone sync error:', arguments);
@@ -252,8 +300,11 @@ define(["widgets/js/manager",
                 return false;
             }
 
-            // Delete any key value pairs that the back-end already knows about.
-            var attrs = (method === 'patch') ? options.attrs : model.toJSON(options);
+            var attrs = (method === 'patch') ? options.attrs : model.get_state(options);
+
+            // the state_lock lists attributes that are currently be changed right now from a kernel message
+            // we don't want to send these non-changes back to the kernel, so we delete them out of attrs
+            // (but we only delete them if the value hasn't changed from the value stored in the state_lock
             if (this.state_lock !== null) {
                 var keys = Object.keys(this.state_lock);
                 for (var i=0; i<keys.length; i++) {
@@ -263,9 +314,7 @@ define(["widgets/js/manager",
                     }
                 }
             }
-
-            // Only sync if there are attributes to send to the back-end.
-            attrs = this._pack_models(attrs);
+            
             if (_.size(attrs) > 0) {
 
                 // If this message was sent via backbone itself, it will not
@@ -297,9 +346,7 @@ define(["widgets/js/manager",
                 } else {
                     // We haven't exceeded the throttle, send the message like 
                     // normal.
-                    var data = {method: 'backbone', sync_data: attrs};
-                    this.comm.send(data, callbacks);
-                    this.pending_msgs++;
+                    this.send_sync_message(attrs, callbacks);
                 }
             }
             // Since the comm is a one-way communication, assume the message 
@@ -308,68 +355,71 @@ define(["widgets/js/manager",
             this._buffered_state_diff = {};
         },
 
-        save_changes: function(callbacks) {
-            /**
-             * Push this model's state to the back-end
-             *
-             * This invokes a Backbone.Sync.
-             */
-            this.save(this._buffered_state_diff, {patch: true, callbacks: callbacks});
-        },
 
-        _pack_models: function(value) {
-            /**
-             * Replace models with model ids recursively.
-             */
+        send_sync_message: function(attrs, callbacks) {
+            // prepare and send a comm message syncing attrs
             var that = this;
-            var packed;
-            if (value instanceof Backbone.Model) {
-                return "IPY_MODEL_" + value.id;
-
-            } else if ($.isArray(value)) {
-                packed = [];
-                _.each(value, function(sub_value, key) {
-                    packed.push(that._pack_models(sub_value));
-                });
-                return packed;
-            } else if (value instanceof Date || value instanceof String) {
-                return value;
-            } else if (value instanceof Object) {
-                packed = {};
-                _.each(value, function(sub_value, key) {
-                    packed[key] = that._pack_models(sub_value);
-                });
-                return packed;
-
-            } else {
-                return value;
+            // first, build a state dictionary with key=the attribute and the value
+            // being the value or the promise of the serialized value
+            var state_promise_dict = {};
+            var keys = Object.keys(attrs);
+            for (var i=0; i<keys.length; i++) {
+                // bind k and v locally; needed since we have an inner async function using v
+                (function(k,v) {
+                    if (that.serializers[k]) {
+                        state_promise_dict[k] = that.serializers[k].then(function(f) {
+                            if (f.serialize) {
+                                return f.serialize(v, that);
+                            } else {
+                                return v;
+                            }
+                        })
+                    } else {
+                        state_promise_dict[k] = v;
+                    }
+                })(keys[i], attrs[keys[i]])
+            }
+            utils.resolve_promises_dict(state_promise_dict).then(function(state) {
+                // get binary values, then send
+                var keys = Object.keys(state);
+                var buffers = [];
+                var buffer_keys = [];
+                for (var i=0; i<keys.length; i++) {
+                    var key = keys[i];
+                    var value = state[key];
+                    if (value.buffer instanceof ArrayBuffer
+                        || value instanceof ArrayBuffer) {
+                        buffers.push(value);
+                        buffer_keys.push(key);
+                        delete state[key];
+                    }
+                }
+                that.comm.send({method: 'backbone', sync_data: state, buffer_keys: buffer_keys}, callbacks, {}, buffers);
+                that.pending_msgs++;
+            })
+        },
+        
+        serialize: function(model, attrs) {
+            // Serialize the attributes into a sync message
+            var keys = Object.keys(attrs);            
+            var key, value;
+            var buffers, metadata, buffer_keys, serialize;
+            for (var i=0; i<keys.length; i++) {
+                key = keys[i];
+                serialize = model.serializers[key];
+                if (serialize && serialize.serialize) {
+                    attrs[key] = serialize.serialize(attrs[key]);
+                }
             }
         },
 
-        _unpack_models: function(value) {
+        save_changes: function(callbacks) {
             /**
-             * Replace model ids with models recursively.
+             * Push this model's state to the back-end
+             *
+             * This invokes a Backbone.Sync.
              */
-            var that = this;
-            var unpacked;
-            if ($.isArray(value)) {
-                unpacked = [];
-                _.each(value, function(sub_value, key) {
-                    unpacked.push(that._unpack_models(sub_value));
-                });
-                return Promise.all(unpacked);
-            } else if (value instanceof Object) {
-                unpacked = {};
-                _.each(value, function(sub_value, key) {
-                    unpacked[key] = that._unpack_models(sub_value);
-                });
-                return utils.resolve_promises_dict(unpacked);
-            } else if (typeof value === 'string' && value.slice(0,10) === "IPY_MODEL_") {
-                // get_model returns a promise already
-                return this.widget_manager.get_model(value.slice(10, value.length));
-            } else {
-                return Promise.resolve(value);
-            }
+            this.save(this._buffered_state_diff, {patch: true, callbacks: callbacks});
         },
 
         on_some_change: function(keys, callback, context) {
@@ -386,7 +436,7 @@ define(["widgets/js/manager",
                 }
             }, this);
 
-       },
+        },
     });
     widgetmanager.WidgetManager.register_widget_model('WidgetModel', WidgetModel);
 
@@ -444,11 +494,11 @@ define(["widgets/js/manager",
              */
         },
 
-        send: function (content) {
+        send: function (content, buffers) {
             /**
              * Send a custom msg associated with this view.
              */
-            this.model.send(content, this.callbacks());
+            this.model.send(content, this.callbacks(), buffers);
         },
 
         touch: function () {
diff --git a/IPython/html/widgets/widget.py b/IPython/html/widgets/widget.py
index 456133d..4dfa7d2 100644
--- a/IPython/html/widgets/widget.py
+++ b/IPython/html/widgets/widget.py
@@ -89,6 +89,47 @@ def register(key=None):
     return wrap
 
 
+def _widget_to_json(x):
+    if isinstance(x, dict):
+        return {k: _widget_to_json(v) for k, v in x.items()}
+    elif isinstance(x, (list, tuple)):
+        return [_widget_to_json(v) for v in x]
+    elif isinstance(x, Widget):
+        return "IPY_MODEL_" + x.model_id
+    else:
+        return x
+
+def _json_to_widget(x):
+    if isinstance(x, dict):
+        return {k: _json_to_widget(v) for k, v in x.items()}
+    elif isinstance(x, (list, tuple)):
+        return [_json_to_widget(v) for v in x]
+    elif isinstance(x, string_types) and x.startswith('IPY_MODEL_') and x[10:] in Widget.widgets:
+        return Widget.widgets[x[10:]]
+    else:
+        return x
+
+widget_serialization = {
+    'from_json': _json_to_widget,
+    'to_json': lambda x: (_widget_to_json(x), {'serialization': ('widget_serialization', 'widgets/js/types')})
+}
+
+def _to_binary_list(x):
+    import numpy
+    return memoryview(numpy.array(x, dtype=float)), {'serialization': ('list_of_numbers', 'widgets/js/types')}
+
+def _from_binary_list(x):
+    import numpy
+    a = numpy.frombuffer(x.tobytes(), dtype=float)
+    return list(a)
+
+list_of_numbers = {
+    'from_json': _from_binary_list,
+    'to_json': _to_binary_list
+}
+
+
+
 class Widget(LoggingConfigurable):
     #-------------------------------------------------------------------------
     # Class attributes
@@ -216,10 +257,13 @@ class Widget(LoggingConfigurable):
         key : unicode, or iterable (optional)
             A single property's name or iterable of property names to sync with the front-end.
         """
-        self._send({
-            "method" : "update",
-            "state"  : self.get_state(key=key)
-        })
+        state, buffer_keys, buffers, metadata = self.get_state(key=key)
+        msg = {"method": "update", "state": state}
+        if buffer_keys:
+            msg['buffers'] = buffer_keys
+        if metadata:
+            msg['metadata'] = metadata
+        self._send(msg, buffers=buffers)
 
     def get_state(self, key=None):
         """Gets the widget state, or a piece of it.
@@ -228,6 +272,16 @@ class Widget(LoggingConfigurable):
         ----------
         key : unicode or iterable (optional)
             A single property's name or iterable of property names to get.
+
+        Returns
+        -------
+        state : dict of states
+        buffer_keys : list of strings
+            the values that are stored in buffers
+        buffers : list of binary memoryviews
+            values to transmit in binary
+        metadata : dict
+            metadata for each field: {key: metadata}
         """
         if key is None:
             keys = self.keys
@@ -238,11 +292,21 @@ class Widget(LoggingConfigurable):
         else:
             raise ValueError("key must be a string, an iterable of keys, or None")
         state = {}
+        buffers = []
+        buffer_keys = []
+        metadata = {}
         for k in keys:
             f = self.trait_metadata(k, 'to_json', self._trait_to_json)
             value = getattr(self, k)
-            state[k] = f(value)
-        return state
+            serialized, md = f(value)
+            if isinstance(serialized, memoryview):
+                buffers.append(serialized)
+                buffer_keys.append(k)
+            else:
+                state[k] = serialized
+            if md is not None:
+                metadata[k] = md
+        return state, buffer_keys, buffers, metadata
 
     def set_state(self, sync_data):
         """Called when a state is received from the front-end."""
@@ -253,15 +317,17 @@ class Widget(LoggingConfigurable):
                 with self._lock_property(name, json_value):
                     setattr(self, name, from_json(json_value))
     
-    def send(self, content):
+    def send(self, content, buffers=None):
         """Sends a custom msg to the widget model in the front-end.
 
         Parameters
         ----------
         content : dict
             Content of the message to send.
+        buffers : list of binary buffers
+            Binary buffers to send with message
         """
-        self._send({"method": "custom", "content": content})
+        self._send({"method": "custom", "content": content}, buffers=buffers)
 
     def on_msg(self, callback, remove=False):
         """(Un)Register a custom msg receive callback.
@@ -269,9 +335,9 @@ class Widget(LoggingConfigurable):
         Parameters
         ----------
         callback: callable
-            callback will be passed two arguments when a message arrives::
+            callback will be passed three arguments when a message arrives::
             
-                callback(widget, content)
+                callback(widget, content, buffers)
             
         remove: bool
             True if the callback should be unregistered."""
@@ -346,7 +412,10 @@ class Widget(LoggingConfigurable):
         # Handle backbone sync methods CREATE, PATCH, and UPDATE all in one.
         if method == 'backbone':
             if 'sync_data' in data:
+                # get binary buffers too
                 sync_data = data['sync_data']
+                for i,k in enumerate(data.get('buffer_keys', [])):
+                    sync_data[k] = msg['buffers'][i]
                 self.set_state(sync_data) # handles all methods
 
         # Handle a state request.
@@ -356,15 +425,15 @@ class Widget(LoggingConfigurable):
         # Handle a custom msg from the front-end.
         elif method == 'custom':
             if 'content' in data:
-                self._handle_custom_msg(data['content'])
+                self._handle_custom_msg(data['content'], msg['buffers'])
 
         # Catch remainder.
         else:
             self.log.error('Unknown front-end to back-end widget msg with method "%s"' % method)
 
-    def _handle_custom_msg(self, content):
+    def _handle_custom_msg(self, content, buffers):
         """Called when a custom msg is received."""
-        self._msg_callbacks(self, content)
+        self._msg_callbacks(self, content, buffers)
 
     def _notify_trait(self, name, old_value, new_value):
         """Called when a property has been changed."""
@@ -391,30 +460,14 @@ class Widget(LoggingConfigurable):
         Traverse lists/tuples and dicts and serialize their values as well.
         Replace any widgets with their model_id
         """
-        if isinstance(x, dict):
-            return {k: self._trait_to_json(v) for k, v in x.items()}
-        elif isinstance(x, (list, tuple)):
-            return [self._trait_to_json(v) for v in x]
-        elif isinstance(x, Widget):
-            return "IPY_MODEL_" + x.model_id
-        else:
-            return x # Value must be JSON-able
+        return x, None
 
     def _trait_from_json(self, x):
         """Convert json values to objects
 
         Replace any strings representing valid model id values to Widget references.
         """
-        if isinstance(x, dict):
-            return {k: self._trait_from_json(v) for k, v in x.items()}
-        elif isinstance(x, (list, tuple)):
-            return [self._trait_from_json(v) for v in x]
-        elif isinstance(x, string_types) and x.startswith('IPY_MODEL_') and x[10:] in Widget.widgets:
-            # we want to support having child widgets at any level in a hierarchy
-            # trusting that a widget UUID will not appear out in the wild
-            return Widget.widgets[x[10:]]
-        else:
-            return x
+        return x
 
     def _ipython_display_(self, **kwargs):
         """Called when `IPython.display.display` is called on the widget."""
@@ -423,9 +476,9 @@ class Widget(LoggingConfigurable):
             self._send({"method": "display"})
             self._handle_displayed(**kwargs)
 
-    def _send(self, msg):
+    def _send(self, msg, buffers=None):
         """Sends a message to the model in the front-end."""
-        self.comm.send(msg)
+        self.comm.send(data=msg, buffers=buffers)
 
 
 class DOMWidget(Widget):
diff --git a/IPython/html/widgets/widget_box.py b/IPython/html/widgets/widget_box.py
index ca73e0d..4ad55fe 100644
--- a/IPython/html/widgets/widget_box.py
+++ b/IPython/html/widgets/widget_box.py
@@ -6,7 +6,7 @@ Represents a container that can be used to group other widgets.
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
-from .widget import DOMWidget, register
+from .widget import DOMWidget, register, widget_serialization
 from IPython.utils.traitlets import Unicode, Tuple, TraitError, Int, CaselessStrEnum
 from IPython.utils.warn import DeprecatedClass
 
@@ -18,7 +18,9 @@ class Box(DOMWidget):
     # Child widgets in the container.
     # Using a tuple here to force reassignment to update the list.
     # When a proper notifying-list trait exists, that is what should be used here.
-    children = Tuple(sync=True)
+    # TODO: make this tuple serialize models
+    # TODO: enforce that tuples here have a single datatype
+    children = Tuple(sync=True, **widget_serialization)
     
     _overflow_values = ['visible', 'hidden', 'scroll', 'auto', 'initial', 'inherit', '']
     overflow_x = CaselessStrEnum(
diff --git a/IPython/html/widgets/widget_button.py b/IPython/html/widgets/widget_button.py
index 4d5549c..3b3ad66 100644
--- a/IPython/html/widgets/widget_button.py
+++ b/IPython/html/widgets/widget_button.py
@@ -67,7 +67,7 @@ class Button(DOMWidget):
             Set to true to remove the callback from the list of callbacks."""
         self._click_handlers.register_callback(callback, remove=remove)
 
-    def _handle_button_msg(self, _, content):
+    def _handle_button_msg(self, _, content, buffers):
         """Handle a msg from the front-end.
 
         Parameters