Merge pull request #9551 from SylvainCorlay/sphinxify

Sphinxify

IPython/core/interactiveshell.py

@@ -73,8 +73,8 b' from IPython.utils.py3compat import (builtin_mod, unicode_type, string_types,'
                                      with_metaclass, iteritems)
 from IPython.utils.strdispatch import StrDispatch
 from IPython.utils.syspathcontext import prepended_to_syspath
-from IPython.utils.text import (format_screen, LSString, SList,
-                                DollarFormatter)
+from IPython.utils.text import format_screen, LSString, SList, DollarFormatter
+from IPython.utils.tempdir import TemporaryDirectory
 from traitlets import (
     Integer, Bool, CaselessStrEnum, Enum, List, Dict, Unicode, Instance, Type,
     observe, default,
@@ -83,6 +83,18 b' from warnings import warn'
 from logging import error
 import IPython.core.hooks
 
+try:
+    import docrepr.sphinxify as sphx
+
+    def sphinxify(doc):
+        with TemporaryDirectory() as dirname:
+            return {
+                'text/html': sphx.sphinxify(doc, dirname),
+                'text/plain': doc
+            }
+except ImportError:
+    sphinxify = None
+
 #-----------------------------------------------------------------------------
 # Globals
 #-----------------------------------------------------------------------------
@@ -265,6 +277,16 b' class InteractiveShell(SingletonConfigurable):'
     display_formatter = Instance(DisplayFormatter, allow_none=True)
     displayhook_class = Type(DisplayHook)
     display_pub_class = Type(DisplayPublisher)
+    sphinxify_docstring = Bool(False, help=
+        """
+        Enables rich html representation of docstrings. (This requires the
+        docrepr module).
+        """).tag(config=True)
+    enable_html_pager = Bool(False, help=
+        """
+        (Provisional API) enables html representation in mime bundles sent
+        to pagers.
+        """).tag(config=True)
     data_pub_class = None
 
     exit_now = Bool(False)
@@ -281,12 +303,12 b' class InteractiveShell(SingletonConfigurable):'
     # is ready to be executed.
     input_splitter = Instance('IPython.core.inputsplitter.IPythonInputSplitter',
                               (), {'line_input_checker': True})
-    
+
     # This InputSplitter instance is used to transform completed cells before
     # running them. It allows cell magics to contain blank lines.
     input_transformer_manager = Instance('IPython.core.inputsplitter.IPythonInputSplitter',
                                          (), {'line_input_checker': False})
-    
+
     logstart = Bool(False, help=
         """
         Start logging to the default log file in overwrite mode.
@@ -341,11 +363,11 b' class InteractiveShell(SingletonConfigurable):'
                 name=name)
         )
         # protect against weird cases where self.config may not exist:
-    
+
     show_rewritten_input = Bool(True,
         help="Show rewritten input, e.g. for autocall."
     ).tag(config=True)
-    
+
     quiet = Bool(False).tag(config=True)
 
     history_length = Integer(10000,
@@ -1358,7 +1380,7 b' class InteractiveShell(SingletonConfigurable):'
                 user_ns_hidden.pop(name, None)
         else:
             user_ns_hidden.update(vdict)
-    
+
     def drop_by_id(self, variables):
         """Remove a dict of variables from the user namespace, if they are the
         same as the values in the dictionary.
@@ -1524,15 +1546,20 b' class InteractiveShell(SingletonConfigurable):'
     def _inspect(self, meth, oname, namespaces=None, **kw):
         """Generic interface to the inspector system.
 
-        This function is meant to be called by pdef, pdoc & friends."""
+        This function is meant to be called by pdef, pdoc & friends.
+        """
         info = self._object_find(oname, namespaces)
+        docformat = sphinxify if self.sphinxify_docstring else None
         if info.found:
             pmethod = getattr(self.inspector, meth)
-            formatter = format_screen if info.ismagic else None
+            # TODO: only apply format_screen to the plain/text repr of the mime
+            # bundle.
+            formatter = format_screen if info.ismagic else docformat
             if meth == 'pdoc':
                 pmethod(info.obj, oname, formatter)
             elif meth == 'pinfo':
-                pmethod(info.obj, oname, formatter, info, **kw)
+                pmethod(info.obj, oname, formatter, info, 
+                        enable_html_pager=self.enable_html_pager, **kw)
             else:
                 pmethod(info.obj, oname)
         else:
@@ -1555,7 +1582,7 b' class InteractiveShell(SingletonConfigurable):'
         with self.builtin_trap:
             info = self._object_find(oname)
             if info.found:
-                return self.inspector._format_info(info.obj, oname, info=info,
+                return self.inspector._get_info(info.obj, oname, info=info,
                             detail_level=detail_level
                 )
             else:

IPython/core/oinspect.py

@@ -17,6 +17,7 b" __all__ = ['Inspector','InspectColors']"
 # stdlib modules
 import inspect
 import linecache
+import warnings
 import os
 from textwrap import dedent
 import types
@@ -43,6 +44,13 b' from IPython.utils.py3compat import cast_unicode, string_types, PY3'
 from IPython.utils.signatures import signature
 from IPython.utils.colorable import Colorable
 
+from pygments import highlight
+from pygments.lexers import PythonLexer
+from pygments.formatters import HtmlFormatter
+
+def pylight(code):
+    return highlight(code, PythonLexer(), HtmlFormatter(noclasses=True))
+
 # builtin docstrings to ignore
 _func_call_docstring = types.FunctionType.__call__.__doc__
 _object_init_docstring = object.__init__.__doc__
@@ -112,7 +120,8 b' def getdoc(obj):'
 
     It also attempts to call a getdoc() method on the given object.  This
     allows objects which provide their docstrings via non-standard mechanisms
-    (like Pyro proxies) to still be inspected by ipython's ? system."""
+    (like Pyro proxies) to still be inspected by ipython's ? system.
+    """
     # Allow objects to offer customized documentation via a getdoc method:
     try:
         ds = obj.getdoc()
@@ -122,14 +131,13 b' def getdoc(obj):'
         # if we get extra info, we add it to the normal docstring.
         if isinstance(ds, string_types):
             return inspect.cleandoc(ds)
-    
     try:
         docstr = inspect.getdoc(obj)
         encoding = get_encoding(obj)
         return py3compat.cast_unicode(docstr, encoding=encoding)
     except Exception:
         # Harden against an inspect failure, which can occur with
-        # SWIG-wrapped extensions.
+        # extensions modules.
         raise
         return None
 
@@ -364,12 +372,12 b' def find_source_lines(obj):'
 
     return lineno
 
-
 class Inspector(Colorable):
+
     def __init__(self, color_table=InspectColors,
                  code_color_table=PyColorize.ANSICodeColors,
                  scheme='NoColor',
-                 str_detail_level=0, 
+                 str_detail_level=0,
                  parent=None, config=None):
         super(Inspector, self).__init__(parent=parent, config=config)
         self.color_table = color_table
@@ -430,7 +438,7 b' class Inspector(Colorable):'
 
     # In Python 3, all classes are new-style, so they all have __init__.
     @skip_doctest_py3
-    def pdoc(self,obj,oname='',formatter = None):
+    def pdoc(self, obj, oname='', formatter=None):
         """Print the docstring for any object.
 
         Optional:
@@ -468,7 +476,7 b' class Inspector(Colorable):'
         lines = []
         ds = getdoc(obj)
         if formatter:
-            ds = formatter(ds)
+            ds = formatter(ds).get('plain/text', ds)
         if ds:
             lines.append(head("Class docstring:"))
             lines.append(indent(ds))
@@ -505,7 +513,7 b' class Inspector(Colorable):'
 
     def pfile(self, obj, oname=''):
         """Show the whole file where an object was defined."""
-        
+
         lineno = find_source_lines(obj)
         if lineno is None:
             self.noinfo('file', oname)
@@ -541,118 +549,171 b' class Inspector(Colorable):'
             title_width = max(len(title) + 2 for title, _ in fields)
         for title, content in fields:
             if len(content.splitlines()) > 1:
-                title = header(title + ":") + "\n"
+                title = header(title + ':') + '\n'
             else:
-                title = header((title+":").ljust(title_width))
+                title = header((title + ':').ljust(title_width))
             out.append(cast_unicode(title) + cast_unicode(content))
         return "\n".join(out)
-    
-    def _format_info(self, obj, oname='', formatter=None, info=None, detail_level=0):
-        """Format an info dict as text"""
-        info = self.info(obj, oname=oname, formatter=formatter,
-                            info=info, detail_level=detail_level)
-        displayfields = []
-        def add_fields(fields):
-            for title, key in fields:
-                field = info[key]
-                if field is not None:
-                    if key == "source":
-                        displayfields.append((title, self.format(cast_unicode(field.rstrip()))))
-                    else:
-                        displayfields.append((title, field.rstrip()))
+
+    def _mime_format(self, text, formatter=None):
+        """Return a mime bundle representation of the input text.
+
+        - if `formatter` is None, the returned mime bundle has
+           a `text/plain` field, with the input text.
+           a `text/html` field with a `<pre>` tag containing the input text.
+
+        - if `formatter` is not None, it must be a callable transforming the
+          input text into a mime bundle. Default values for `text/plain` and
+          `text/html` representations are the ones described above.
+
+        Note:
+
+        Formatters returning strings are supported but this behavior is deprecated.
+
+        """
+        text = cast_unicode(text)
+        defaults = {
+            'text/plain': text,
+            'text/html': '<pre>' + text + '</pre>'
+        }
+
+        if formatter is None:
+            return defaults
+        else:
+            formatted = formatter(text)
+
+            if not isinstance(formatted, dict):
+                # Handle the deprecated behavior of a formatter returning
+                # a string instead of a mime bundle.
+                return {
+                    'text/plain': formatted,
+                    'text/html': '<pre>' + formatted + '</pre>'
+                }
+
+            else:
+                return dict(defaults, **formatted)
+
+    def _get_info(self, obj, oname='', formatter=None, info=None, detail_level=0):
+        """Retrieve an info dict and format it."""
+
+        info = self._info(obj, oname=oname, info=info, detail_level=detail_level)
+
+        mime = {
+            'text/plain': '',
+            'text/html': '',
+        }
+
+        def append_field(bundle, title, key, formatter=None):
+            field = info[key]
+            if field is not None:
+                formatted_field = self._mime_format(field, formatter)
+                bundle['text/plain'] += self.__head(title) + ':\n' + formatted_field['text/plain'] + '\n'
+                bundle['text/html'] += '<h1>' + title + '</h1>\n' + formatted_field['text/html'] + '\n'
+
+        def code_formatter(text):
+            return {
+                'text/plain': self.format(text),
+                'text/html': pylight(text)
+            }
 
         if info['isalias']:
-            add_fields([('Repr', "string_form")])
+            append_field(mime, 'Repr', 'string_form')
 
         elif info['ismagic']:
-            if detail_level > 0 and info['source'] is not None:
-                add_fields([("Source", "source")])
+            if detail_level > 0:
+                append_field(mime, 'Source', 'source', code_formatter)
             else:
-                add_fields([("Docstring", "docstring")])
-
-            add_fields([("File", "file"),
-                       ])
+                append_field(mime, 'Docstring', 'docstring', formatter)
+            append_field(mime, 'File', 'file')
 
         elif info['isclass'] or is_simple_callable(obj):
             # Functions, methods, classes
-            add_fields([("Signature", "definition"),
-                        ("Init signature", "init_definition"),
-                       ])
-            if detail_level > 0 and info['source'] is not None:
-                add_fields([("Source", "source")])
+            append_field(mime, 'Signature', 'definition', code_formatter)
+            append_field(mime, 'Init signature', 'init_definition', code_formatter)
+            if detail_level > 0:
+                append_field(mime, 'Source', 'source', code_formatter)
             else:
-                add_fields([("Docstring", "docstring"),
-                            ("Init docstring", "init_docstring"),
-                           ])
+                append_field(mime, 'Docstring', 'docstring', formatter)
+                append_field(mime, 'Init docstring', 'init_docstring', formatter)
 
-            add_fields([('File', 'file'),
-                        ('Type', 'type_name'),
-                       ])
+            append_field(mime, 'File', 'file')
+            append_field(mime, 'Type', 'type_name')
 
         else:
             # General Python objects
-            add_fields([("Type", "type_name")])
+            append_field(mime, 'Type', 'type_name')
 
             # Base class for old-style instances
             if (not py3compat.PY3) and isinstance(obj, types.InstanceType) and info['base_class']:
-                displayfields.append(("Base Class", info['base_class'].rstrip()))
+                append_field(mime, 'Base Class', 'base_class')
 
-            add_fields([("String form", "string_form")])
+            append_field(mime, 'String form', 'string_form')
 
             # Namespace
             if info['namespace'] != 'Interactive':
-                displayfields.append(("Namespace", info['namespace'].rstrip()))
+                append_field(mime, 'Namespace', 'namespace')
 
-            add_fields([("Length", "length"),
-                        ("File", "file"),
-                        ("Signature", "definition"),
-                       ])
+            append_field(mime, 'Length', 'length')
+            append_field(mime, 'File', 'file'),
+            append_field(mime, 'Signature', 'definition', code_formatter)
 
             # Source or docstring, depending on detail level and whether
             # source found.
-            if detail_level > 0 and info['source'] is not None:
-                displayfields.append(("Source",
-                                      self.format(cast_unicode(info['source']))))
-            elif info['docstring'] is not None:
-                displayfields.append(("Docstring", info["docstring"]))
-
-            add_fields([("Class docstring", "class_docstring"),
-                        ("Init docstring", "init_docstring"),
-                        ("Call signature", "call_def"),
-                        ("Call docstring", "call_docstring")])
-        
-        if displayfields:
-            return self._format_fields(displayfields)
-        else:
-            return u''
-    
-    def pinfo(self, obj, oname='', formatter=None, info=None, detail_level=0):
+            if detail_level > 0:
+                append_field(mime, 'Source', 'source', code_formatter)
+            else:
+                append_field(mime, 'Docstring', 'docstring', formatter)
+
+            append_field(mime, 'Class docstring', 'class_docstring', formatter)
+            append_field(mime, 'Init docstring', 'init_docstring', formatter)
+            append_field(mime, 'Call signature', 'call_def', code_formatter)
+            append_field(mime, 'Call docstring', 'call_docstring', formatter)
+
+        return mime
+
+    def pinfo(self, obj, oname='', formatter=None, info=None, detail_level=0, enable_html_pager=True):
         """Show detailed information about an object.
 
         Optional arguments:
 
         - oname: name of the variable pointing to the object.
 
-        - formatter: special formatter for docstrings (see pdoc)
+        - formatter: callable (optional)
+              A special formatter for docstrings.
+
+              The formatter is a callable that takes a string as an input
+              and returns either a formatted string or a mime type bundle
+              in the form of a dictionnary.
+
+              Although the support of custom formatter returning a string
+              instead of a mime type bundle is deprecated.
 
         - info: a structure with some information fields which may have been
           precomputed already.
 
         - detail_level: if set to 1, more information is given.
         """
-        text = self._format_info(obj, oname, formatter, info, detail_level)
-        if text:
-            page.page(text)
-    
+        info = self._get_info(obj, oname, formatter, info, detail_level)
+        if not enable_html_pager:
+            del info['text/html']
+        page.page(info)
+
     def info(self, obj, oname='', formatter=None, info=None, detail_level=0):
+        """DEPRECATED. Compute a dict with detailed information about an object.
+        """
+        if formatter is not None:
+            warnings.warn('The `formatter` keyword argument to `Inspector.info`'
+                     'is deprecated as of IPython 5.0 and will have no effects.',
+                      DeprecationWarning, stacklevel=2)
+        return self._info(obj, oname=oname, info=info, detail_level=detail_level)
+
+    def _info(self, obj, oname='', info=None, detail_level=0):
         """Compute a dict with detailed information about an object.
 
         Optional arguments:
 
         - oname: name of the variable pointing to the object.
 
-        - formatter: special formatter for docstrings (see pdoc)
-
         - info: a structure with some information fields which may have been
           precomputed already.
 
@@ -685,14 +746,12 b' class Inspector(Colorable):'
             ds = getdoc(obj)
             if ds is None:
                 ds = '<no docstring>'
-        if formatter is not None:
-            ds = formatter(ds)
 
         # store output in a dict, we initialize it here and fill it as we go
         out = dict(name=oname, found=True, isalias=isalias, ismagic=ismagic)
 
         string_max = 200 # max size of strings to show (snipped if longer)
-        shalf = int((string_max -5)/2)
+        shalf = int((string_max - 5) / 2)
 
         if ismagic:
             obj_type_name = 'Magic function'
@@ -793,7 +852,7 b' class Inspector(Colorable):'
             # reconstruct the function definition and print it:
             defln = self._getdef(obj, oname)
             if defln:
-                out['definition'] = self.format(defln)
+                out['definition'] = defln
 
             # First, check whether the instance docstring is identical to the
             # class one, and print it separately if they don't coincide.  In
@@ -826,12 +885,10 b' class Inspector(Colorable):'
             # Call form docstring for callable instances
             if safe_hasattr(obj, '__call__') and not is_simple_callable(obj):
                 call_def = self._getdef(obj.__call__, oname)
-                if call_def:
-                    call_def = self.format(call_def)
+                if call_def and (call_def != out.get('definition')):
                     # it may never be the case that call def and definition differ,
                     # but don't include the same signature twice
-                    if call_def != out.get('definition'):
-                        out['call_def'] = call_def
+                    out['call_def'] = call_def
                 call_ds = getdoc(obj.__call__)
                 # Skip Python's auto-generated docstrings
                 if call_ds == _func_call_docstring:

IPython/core/page.py

@@ -38,7 +38,7 b' def display_page(strng, start=0, screen_lines=25):'
     else:
         if start:
             strng = u'\n'.join(strng.splitlines()[start:])
-        data = {'text/plain': strng}
+        data = { 'text/plain': strng }
     display(data, raw=True)
 
 
@@ -56,8 +56,10 b' def page_dumb(strng, start=0, screen_lines=25):'
     """Very dumb 'pager' in Python, for when nothing else works.
 
     Only moves forward, same interface as page(), except for pager_cmd and
-    mode."""
-
+    mode.
+    """
+    if isinstance(strng, dict):
+        strng = strng.get('text/plain', '')
     out_ln  = strng.splitlines()[start:]
     screens = chop(out_ln,screen_lines-1)
     if len(screens) == 1:

IPython/core/tests/test_oinspect.py

@@ -341,13 +341,13 b' if py3compat.PY3:'
 @skipif(not py3compat.PY3)
 def test_definition_kwonlyargs():
     i = inspector.info(f_kwarg, oname='f_kwarg')  # analysis:ignore
-    nt.assert_equal(i['definition'], "f_kwarg(pos, *, kwonly)\n")
+    nt.assert_equal(i['definition'], "f_kwarg(pos, *, kwonly)")
 
 def test_getdoc():
     class A(object):
         """standard docstring"""
         pass
-    
+
     class B(object):
         """standard docstring"""
         def getdoc(self):

IPython/testing/globalipapp.py

@@ -20,7 +20,6 b' from __future__ import print_function'
 #-----------------------------------------------------------------------------
 
 # stdlib
-import os
 import sys
 
 # our own
@@ -141,6 +140,8 b' def start_ipython():'
     
     # Override paging, so we don't require user interaction during the tests.
     def nopage(strng, start=0, screen_lines=0, pager_cmd=None):
+        if isinstance(strng, dict):
+           strng = strng.get('text/plain', '')
         print(strng)
     
     page.orig_page = page.pager_page

docs/source/whatsnew/version5.rst

@@ -41,3 +41,46 b' IPython itself.'
 the `PromptManager` class have been removed, and the prompt machinery simplified. 
 See `TerminalINteractiveShell.prompts` configurable for how to setup your prompts. 
 
+
+
+Provisional Changes
+-------------------
+
+Provisional changes are in experimental functionality that may, or may not make
+it to future version of IPython, and which API may change without warnings.
+Activating these feature and using these API is at your own risk, and may have
+security implication for your system, especially if used with the Jupyter notebook,
+
+When running via the Jupyter notebook interfaces, or other compatible client,
+you can enable rich documentation experimental functionality:
+
+When the ``docrepr`` package is installed setting the boolean flag
+``InteractiveShell.sphinxify_docstring`` to ``True``, will process the various
+object through sphinx before displaying them (see the ``docrepr`` package
+documentation for more information.
+
+You need to also enable the IPython pager display rich HTML representation
+using the ``InteractiveShell.enable_html_pager`` boolean configuration option.
+As usual you can set these configuration options globally in your configuration
+files, alternatively you can turn them on dynamically using the following
+snippet:
+
+.. code-block:: python
+
+    ip = get_ipython()
+    ip.sphinxify_docstring = True
+    ip.enable_html_pager = True
+
+You can test the effect of various combinations of the above configuration in
+the Jupyter notebook, with things example like :
+
+.. code-block:: python
+
+    import numpy as np
+    np.histogram?
+
+This is part of an effort to make Documentation in Python richer and provide in
+the long term if possible dynamic examples that can contain math, images,
+widgets... As stated above this is nightly experimental feature with a lot of
+(fun) problem to solve. We would be happy to get your feedback and expertise on
+it.