Merge pull request #3319 from minrk/user-expressions

IPEP 13: user-expressions and user-variables

The user-expressions and user-variables side effects in execute_requests are updated to use the rich-repr display data identical to a display_data message.

closes  #2654

IPython/core/interactiveshell.py

@@ -2363,9 +2363,37 b' class InteractiveShell(SingletonConfigurable):'
     # Things related to extracting values/expressions from kernel and user_ns
     #-------------------------------------------------------------------------
 
-    def _simple_error(self):
-        etype, value = sys.exc_info()[:2]
-        return u'[ERROR] {e.__name__}: {v}'.format(e=etype, v=value)
+    def _user_obj_error(self):
+        """return simple exception dict
+        
+        for use in user_variables / expressions
+        """
+        
+        etype, evalue, tb = self._get_exc_info()
+        stb = self.InteractiveTB.get_exception_only(etype, evalue)
+        
+        exc_info = {
+            u'status' : 'error',
+            u'traceback' : stb,
+            u'ename' : unicode(etype.__name__),
+            u'evalue' : py3compat.safe_unicode(evalue),
+        }
+
+        return exc_info
+    
+    def _format_user_obj(self, obj):
+        """format a user object to display dict
+        
+        for use in user_expressions / variables
+        """
+        
+        data, md = self.display_formatter.format(obj)
+        value = {
+            'status' : 'ok',
+            'data' : data,
+            'metadata' : md,
+        }
+        return value
 
     def user_variables(self, names):
         """Get a list of variable names from the user's namespace.
@@ -2377,15 +2405,17 b' class InteractiveShell(SingletonConfigurable):'
 
         Returns
         -------
-        A dict, keyed by the input names and with the repr() of each value.
+        A dict, keyed by the input names and with the rich mime-type repr(s) of each value.
+        Each element will be a sub-dict of the same form as a display_data message.
         """
         out = {}
         user_ns = self.user_ns
+        
         for varname in names:
             try:
-                value = repr(user_ns[varname])
+                value = self._format_user_obj(user_ns[varname])
             except:
-                value = self._simple_error()
+                value = self._user_obj_error()
             out[varname] = value
         return out
 
@@ -2401,17 +2431,18 b' class InteractiveShell(SingletonConfigurable):'
 
         Returns
         -------
-        A dict, keyed like the input expressions dict, with the repr() of each
-        value.
+        A dict, keyed like the input expressions dict, with the rich mime-typed
+        display_data of each value.
         """
         out = {}
         user_ns = self.user_ns
         global_ns = self.user_global_ns
+        
         for key, expr in expressions.iteritems():
             try:
-                value = repr(eval(expr, global_ns, user_ns))
+                value = self._format_user_obj(eval(expr, global_ns, user_ns))
             except:
-                value = self._simple_error()
+                value = self._user_obj_error()
             out[key] = value
         return out
 

IPython/core/magic.py

@@ -15,6 +15,7 b''
 # Imports
 #-----------------------------------------------------------------------------
 # Stdlib
+import json
 import os
 import re
 import sys
@@ -326,16 +327,6 b' class MagicsManager(Configurable):'
         """Return descriptive string with automagic status."""
         return self._auto_status[self.auto_magic]
     
-    def lsmagic_info(self):
-        magic_list = []
-        for m_type in self.magics :
-            for m_name,mgc in self.magics[m_type].items():
-                try :
-                    magic_list.append({'name':m_name,'type':m_type,'class':mgc.im_class.__name__})
-                except AttributeError :
-                    magic_list.append({'name':m_name,'type':m_type,'class':'Other'})
-        return magic_list
-
     def lsmagic(self):
         """Return a dict of currently available magic functions.
 

IPython/core/magics/basic.py

@@ -15,6 +15,7 b' from __future__ import print_function'
 
 # Stdlib
 import io
+import json
 import sys
 from pprint import pformat
 
@@ -33,6 +34,55 b' from IPython.utils.warn import warn, error'
 # Magics class implementation
 #-----------------------------------------------------------------------------
 
+class MagicsDisplay(object):
+    def __init__(self, magics_manager):
+        self.magics_manager = magics_manager
+    
+    def _lsmagic(self):
+        """The main implementation of the %lsmagic"""
+        mesc = magic_escapes['line']
+        cesc = magic_escapes['cell']
+        mman = self.magics_manager
+        magics = mman.lsmagic()
+        out = ['Available line magics:',
+               mesc + ('  '+mesc).join(sorted(magics['line'])),
+               '',
+               'Available cell magics:',
+               cesc + ('  '+cesc).join(sorted(magics['cell'])),
+               '',
+               mman.auto_status()]
+        return '\n'.join(out)
+
+    def _repr_pretty_(self, p, cycle):
+        p.text(self._lsmagic())
+    
+    def __str__(self):
+        return self._lsmagic()
+    
+    def _jsonable(self):
+        """turn magics dict into jsonable dict of the same structure
+        
+        replaces object instances with their class names as strings
+        """
+        magic_dict = {}
+        mman = self.magics_manager
+        magics = mman.lsmagic()
+        for key, subdict in magics.items():
+            d = {}
+            magic_dict[key] = d
+            for name, obj in subdict.items():
+                try:
+                    classname = obj.im_class.__name__
+                except AttributeError:
+                    classname = 'Other'
+                
+                d[name] = classname
+        return magic_dict
+        
+    def _repr_json_(self):
+        return json.dumps(self._jsonable())
+
+
 @magics_class
 class BasicMagics(Magics):
     """Magics that provide central IPython functionality.
@@ -124,24 +174,10 b' class BasicMagics(Magics):'
                 magic_escapes['cell'], name,
                 magic_escapes['cell'], target))
 
-    def _lsmagic(self):
-        mesc = magic_escapes['line']
-        cesc = magic_escapes['cell']
-        mman = self.shell.magics_manager
-        magics = mman.lsmagic()
-        out = ['Available line magics:',
-               mesc + ('  '+mesc).join(sorted(magics['line'])),
-               '',
-               'Available cell magics:',
-               cesc + ('  '+cesc).join(sorted(magics['cell'])),
-               '',
-               mman.auto_status()]
-        return '\n'.join(out)
-
     @line_magic
     def lsmagic(self, parameter_s=''):
         """List currently available magic functions."""
-        print(self._lsmagic())
+        return MagicsDisplay(self.shell.magics_manager)
 
     def _magic_docs(self, brief=False, rest=False):
         """Return docstrings from magic functions."""

IPython/core/tests/test_interactiveshell.py

@@ -578,3 +578,72 b' class TestAstTransformError(unittest.TestCase):'
 def test__IPYTHON__():
     # This shouldn't raise a NameError, that's all
     __IPYTHON__
+
+
+class DummyRepr(object):
+    def __repr__(self):
+        return "DummyRepr"
+    
+    def _repr_html_(self):
+        return "<b>dummy</b>"
+    
+    def _repr_javascript_(self):
+        return "console.log('hi');", {'key': 'value'}
+    
+
+def test_user_variables():
+    # enable all formatters
+    ip.display_formatter.active_types = ip.display_formatter.format_types
+    
+    ip.user_ns['dummy'] = d = DummyRepr()
+    keys = set(['dummy', 'doesnotexist'])
+    r = ip.user_variables(keys)
+
+    nt.assert_equal(keys, set(r.keys()))
+    dummy = r['dummy']
+    nt.assert_equal(set(['status', 'data', 'metadata']), set(dummy.keys()))
+    nt.assert_equal(dummy['status'], 'ok')
+    data = dummy['data']
+    metadata = dummy['metadata']
+    nt.assert_equal(data.get('text/html'), d._repr_html_())
+    js, jsmd = d._repr_javascript_()
+    nt.assert_equal(data.get('application/javascript'), js)
+    nt.assert_equal(metadata.get('application/javascript'), jsmd)
+    
+    dne = r['doesnotexist']
+    nt.assert_equal(dne['status'], 'error')
+    nt.assert_equal(dne['ename'], 'KeyError')
+    
+    # back to text only
+    ip.display_formatter.active_types = ['text/plain']
+    
+def test_user_expression():
+    # enable all formatters
+    ip.display_formatter.active_types = ip.display_formatter.format_types
+    query = {
+        'a' : '1 + 2',
+        'b' : '1/0',
+    }
+    r = ip.user_expressions(query)
+    import pprint
+    pprint.pprint(r)
+    nt.assert_equal(r.keys(), query.keys())
+    a = r['a']
+    nt.assert_equal(set(['status', 'data', 'metadata']), set(a.keys()))
+    nt.assert_equal(a['status'], 'ok')
+    data = a['data']
+    metadata = a['metadata']
+    nt.assert_equal(data.get('text/plain'), '3')
+    
+    b = r['b']
+    nt.assert_equal(b['status'], 'error')
+    nt.assert_equal(b['ename'], 'ZeroDivisionError')
+    
+    # back to text only
+    ip.display_formatter.active_types = ['text/plain']
+    
+
+
+
+
+

IPython/frontend/qt/console/history_console_widget.py

@@ -241,7 +241,9 b' class HistoryConsoleWidget(ConsoleWidget):'
             content = msg['content']
             status = content['status']
             if status == 'ok':
-                self._max_session_history=(int(content['user_expressions']['hlen']))
+                self._max_session_history = int(
+                    content['user_expressions']['hlen']['data']['text/plain']
+                )
 
     def save_magic(self):
         # update the session history length

IPython/frontend/qt/console/mainwindow.py

@@ -20,15 +20,17 b' Authors:'
 #-----------------------------------------------------------------------------
 
 # stdlib imports
-import sys
+import json
 import re
+import sys
 import webbrowser
-import ast
 from threading import Thread
 
 # System library imports
 from IPython.external.qt import QtGui,QtCore
 
+from IPython.core.magic import magic_escapes
+
 def background(f):
     """call a function in a simple thread, to prevent blocking"""
     t = Thread(target=f)
@@ -615,37 +617,36 b' class MainWindow(QtGui.QMainWindow):'
         inner_dynamic_magic.__name__ = "dynamics_magic_s"
         return inner_dynamic_magic
 
-    def populate_all_magic_menu(self, listofmagic=None):
-        """Clean "All Magics..." menu and repopulate it with `listofmagic`
+    def populate_all_magic_menu(self, display_data=None):
+        """Clean "All Magics..." menu and repopulate it with `display_data`
 
         Parameters
         ----------
-        listofmagic : string,
-            repr() of a list of strings, send back by the kernel
+        display_data : dict,
+            dict of display_data for the magics dict of a MagicsManager.
+            Expects json data, as the result of %lsmagic
 
-        Notes
-        -----
-        `listofmagic`is a repr() of list because it is fed with the result of
-        a 'user_expression'
         """
         for k,v in self._magic_menu_dict.items():
             v.clear()
         self.all_magic_menu.clear()
 
+        if not display_data:
+            return
 
-        mlist=ast.literal_eval(listofmagic)
-        for magic in mlist:
-            cell = (magic['type'] == 'cell')
-            name = magic['name']
-            mclass = magic['class']
-            if cell :
-                prefix='%%'
-            else :
-                prefix='%'
-            magic_menu = self._get_magic_menu(mclass)
+        if display_data['status'] != 'ok':
+            self.log.warn("%%lsmagic user-expression failed: %s" % display_data)
+            return
 
-            pmagic = '%s%s'%(prefix,name)
+        mdict = json.loads(display_data['data'].get('application/json', {}))
 
+        for mtype in sorted(mdict):
+            subdict = mdict[mtype]
+            prefix = magic_escapes[mtype]
+            for name in sorted(subdict):
+                mclass = subdict[name]
+                magic_menu = self._get_magic_menu(mclass)
+                pmagic = prefix + name
             xaction = QtGui.QAction(pmagic,
                 self,
                 triggered=self._make_dynamic_magic(pmagic)
@@ -660,7 +661,7 b' class MainWindow(QtGui.QMainWindow):'
         menu with the list received back
 
         """
-        self.active_frontend._silent_exec_callback('get_ipython().magics_manager.lsmagic_info()',
+        self.active_frontend._silent_exec_callback('get_ipython().magic("lsmagic")',
                 self.populate_all_magic_menu)
 
     def _get_magic_menu(self,menuidentifier, menulabel=None):

IPython/kernel/tests/test_message_spec.py

@@ -361,7 +361,21 b' def test_user_variables():'
 
     msg_id, reply = execute(code='x=1', user_variables=['x'])
     user_variables = reply['user_variables']
-    nt.assert_equal(user_variables, {u'x' : u'1'})
+    nt.assert_equal(user_variables, {u'x': {
+        u'status': u'ok',
+        u'data': {u'text/plain': u'1'},
+        u'metadata': {},
+    }})
+
+
+def test_user_variables_fail():
+    flush_channels()
+
+    msg_id, reply = execute(code='x=1', user_variables=['nosuchname'])
+    user_variables = reply['user_variables']
+    foo = user_variables['nosuchname']
+    nt.assert_equal(foo['status'], 'error')
+    nt.assert_equal(foo['ename'], 'KeyError')
 
 
 def test_user_expressions():
@@ -369,7 +383,21 b' def test_user_expressions():'
 
     msg_id, reply = execute(code='x=1', user_expressions=dict(foo='x+1'))
     user_expressions = reply['user_expressions']
-    nt.assert_equal(user_expressions, {u'foo' : u'2'})
+    nt.assert_equal(user_expressions, {u'foo': {
+        u'status': u'ok',
+        u'data': {u'text/plain': u'2'},
+        u'metadata': {},
+    }})
+
+
+def test_user_expressions_fail():
+    flush_channels()
+
+    msg_id, reply = execute(code='x=0', user_expressions=dict(foo='nosuchname'))
+    user_expressions = reply['user_expressions']
+    foo = user_expressions['foo']
+    nt.assert_equal(foo['status'], 'error')
+    nt.assert_equal(foo['ename'], 'NameError')
 
 
 @dec.parametric

IPython/kernel/zmq/zmqshell.py

@@ -472,27 +472,6 b' class KernelMagics(Magics):'
         else:
             print("Autosave disabled")
 
-def safe_unicode(e):
-    """unicode(e) with various fallbacks. Used for exceptions, which may not be
-    safe to call unicode() on.
-    """
-    try:
-        return unicode(e)
-    except UnicodeError:
-        pass
-
-    try:
-        return py3compat.str_to_unicode(str(e))
-    except UnicodeError:
-        pass
-
-    try:
-        return py3compat.str_to_unicode(repr(e))
-    except UnicodeError:
-        pass
-
-    return u'Unrecoverably corrupt evalue'
-
 
 class ZMQInteractiveShell(InteractiveShell):
     """A subclass of InteractiveShell for ZMQ."""
@@ -572,7 +551,7 b' class ZMQInteractiveShell(InteractiveShell):'
         exc_content = {
             u'traceback' : stb,
             u'ename' : unicode(etype.__name__),
-            u'evalue' : safe_unicode(evalue)
+            u'evalue' : py3compat.safe_unicode(evalue),
         }
 
         dh = self.displayhook

IPython/utils/py3compat.py

@@ -50,6 +50,27 b' def _modify_str_or_docstring(str_change_func):'
         return doc
     return wrapper
 
+def safe_unicode(e):
+    """unicode(e) with various fallbacks. Used for exceptions, which may not be
+    safe to call unicode() on.
+    """
+    try:
+        return unicode(e)
+    except UnicodeError:
+        pass
+
+    try:
+        return py3compat.str_to_unicode(str(e))
+    except UnicodeError:
+        pass
+
+    try:
+        return py3compat.str_to_unicode(repr(e))
+    except UnicodeError:
+        pass
+
+    return u'Unrecoverably corrupt evalue'
+
 if sys.version_info[0] >= 3:
     PY3 = True
     

docs/source/development/messaging.txt

@@ -171,8 +171,9 b' Message type: ``execute_request``::'
     # is forced to be False.
     'store_history' : bool,
 
-    # A list of variable names from the user's namespace to be retrieved.  What
-    # returns is a JSON string of the variable's repr(), not a python object.
+    # A list of variable names from the user's namespace to be retrieved.
+    # What returns is a rich representation of each variable (dict keyed by name).
+    # See the display_data content for the structure of the representation data.
     'user_variables' : list,
 
     # Similarly, a dict mapping names to expressions to be evaluated in the