Merge pull request #11848 from Carreau/mime-repr-hook

Provide hooks for arbitrary mimetypes handling.

docs/source/config/shell_mimerenderer.rst

@@ -0,0 +1,60 b''
+
+.. _shell_mimerenderer:
+
+
+Mime Renderer Extensions
+========================
+
+Like it's cousins, Jupyter Notebooks and JupyterLab, Terminal IPython can be
+thought to render a number of mimetypes in the shell. This can be used to either
+display inline images if your terminal emulator supports it; or open some
+display results with external file viewers.
+
+Registering new mimetype handlers can so far only be done my extensions and
+requires 4 steps:
+
+   - Define a callable that takes 2 parameters:``data`` and ``metadata``; return
+     value of the callable is so far ignored. This callable is responsible for
+     "displaying" the given mimetype. Which can be sending the right escape
+     sequences and bytes to the current terminal; or open an external program. -
+   - Appending the right mimetype to ``ipython.display_formatter.active_types``
+     for IPython to know it should not ignore those mimetypes.
+   - Enabling the given mimetype: ``ipython.display_formatter.formatters[mime].enabled = True``
+   - Registering above callable with mimetype handler:
+     ``ipython.mime_renderers[mime] = handler``
+
+
+Here is a complete IPython extension to display images inline and convert math
+to png, before displaying it inline for iterm2 on macOS ::
+
+
+    from base64 import encodebytes
+    from IPython.lib.latextools import latex_to_png
+
+
+    def mathcat(data, meta):
+        png = latex_to_png(f'$${data}$$'.replace('\displaystyle', '').replace('$$$', '$$'))
+        imcat(png, meta)
+
+    IMAGE_CODE = '\033]1337;File=name=name;inline=true;:{}\a'
+
+    def imcat(image_data, metadata):
+        try:
+            print(IMAGE_CODE.format(encodebytes(image_data).decode()))
+        # bug workaround
+        except:
+            print(IMAGE_CODE.format(image_data))
+
+    def register_mimerenderer(ipython, mime, handler):
+        ipython.display_formatter.active_types.append(mime)
+        ipython.display_formatter.formatters[mime].enabled = True
+        ipython.mime_renderers[mime] = handler
+
+    def load_ipython_extension(ipython):
+        register_mimerenderer(ipython, 'image/png', imcat)
+        register_mimerenderer(ipython, 'image/jpeg', imcat)
+        register_mimerenderer(ipython, 'text/latex', mathcat)
+
+This example only work for iterm2 on macOS and skip error handling for brevity.
+One could also invoke an external viewer with ``subprocess.run()`` and a
+temporary file, which is left as an exercise.

docs/source/whatsnew/pr/mimerenderer.rst

@@ -0,0 +1,22 b''
+Arbitrary Mimetypes Handing in Terminal
+=======================================
+
+When using IPython terminal it is now possible to register function to handle
+arbitrary mimetypes. While rendering non-text based representation was possible in
+many jupyter frontend; it was not possible in terminal IPython, as usually
+terminal are limited to displaying text. As many terminal these days provide
+escape sequences to display non-text; bringing this loved feature to IPython CLI
+made a lot of sens. This functionality will not only allow inline images; but
+allow opening of external program; for example ``mplayer`` to "display" sound
+files.
+
+So far only the hooks necessary for this are in place, but no default mime
+renderers added; so inline images will only be available via extensions. We will
+progressively enable these features by default in the next few releases, and
+contribution is welcomed.
+
+We welcome any feedback on the API. See :ref:`shell_mimerenderer` for more
+informations.
+
+This is originally based on work form in :ghpull:`10610` from stephanh42
+started over two years ago, and still a lot need to be done.

IPython/core/displayhook.py

@@ -153,7 +153,7 b' class DisplayHook(Configurable):'
     # This can be set to True by the write_output_prompt method in a subclass
     prompt_end_newline = False
 
-    def write_format_data(self, format_dict, md_dict=None):
+    def write_format_data(self, format_dict, md_dict=None) -> None:
         """Write the format data dict to the frontend.
 
         This default version of this method simply writes the plain text

IPython/core/displaypub.py

@@ -19,7 +19,7 b' spec.'
 import sys
 
 from traitlets.config.configurable import Configurable
-from traitlets import List
+from traitlets import List, Dict
 
 # This used to be defined here - it is imported for backwards compatibility
 from .display import publish_display_data
@@ -28,6 +28,7 b' from .display import publish_display_data'
 # Main payload class
 #-----------------------------------------------------------------------------
 
+
 class DisplayPublisher(Configurable):
     """A traited class that publishes display data to frontends.
 
@@ -35,6 +36,10 b' class DisplayPublisher(Configurable):'
     be accessed there.
     """
 
+    def __init__(self, shell=None, *args, **kwargs):
+        self.shell = shell
+        super().__init__(*args, **kwargs)
+
     def _validate_data(self, data, metadata=None):
         """Validate the display data.
 
@@ -53,7 +58,7 b' class DisplayPublisher(Configurable):'
                 raise TypeError('metadata must be a dict, got: %r' % data)
 
     # use * to indicate transient, update are keyword-only
-    def publish(self, data, metadata=None, source=None, *, transient=None, update=False, **kwargs):
+    def publish(self, data, metadata=None, source=None, *, transient=None, update=False, **kwargs) -> None:
         """Publish data and metadata to all frontends.
 
         See the ``display_data`` message in the messaging documentation for
@@ -98,7 +103,15 b' class DisplayPublisher(Configurable):'
             rather than creating a new output.
         """
 
-        # The default is to simply write the plain text data using sys.stdout.
+        handlers = {}
+        if self.shell is not None:
+            handlers = self.shell.mime_renderers
+
+        for mime, handler in handlers.items():
+            if mime in data:
+                handler(data[mime], metadata.get(mime, None))
+                return
+
         if 'text/plain' in data:
             print(data['text/plain'])
 

IPython/core/interactiveshell.py

@@ -857,7 +857,7 b' class InteractiveShell(SingletonConfigurable):'
         self.configurables.append(self.display_formatter)
 
     def init_display_pub(self):
-        self.display_pub = self.display_pub_class(parent=self)
+        self.display_pub = self.display_pub_class(parent=self, shell=self)
         self.configurables.append(self.display_pub)
 
     def init_data_pub(self):

IPython/terminal/interactiveshell.py

@@ -96,6 +96,8 b' def black_reformat_handler(text_before_cursor):'
 
 
 class TerminalInteractiveShell(InteractiveShell):
+    mime_renderers = Dict().tag(config=True)
+
     space_for_menu = Integer(6, help='Number of line at the bottom of the screen '
                                      'to reserve for the completion menu'
                             ).tag(config=True)
@@ -551,7 +553,7 b' class TerminalInteractiveShell(InteractiveShell):'
 
     active_eventloop = None
     def enable_gui(self, gui=None):
-        if gui:
+        if gui and (gui != 'inline') :
             self.active_eventloop, self._inputhook =\
                 get_inputhook_name_and_func(gui)
         else:

IPython/terminal/prompts.py

@@ -89,3 +89,14 b' class RichPromptDisplayHook(DisplayHook):'
                 )
             else:
                 sys.stdout.write(prompt_txt)
+
+    def write_format_data(self, format_dict, md_dict=None) -> None:
+        if self.shell.mime_renderers:
+
+            for mime, handler in self.shell.mime_renderers.items():
+                if mime in format_dict:
+                    handler(format_dict[mime], None)
+                    return
+                
+        super().write_format_data(format_dict, md_dict)
+

IPython/terminal/pt_inputhooks/__init__.py

@@ -12,7 +12,7 b' backends = ['
     'tk',
     'wx',
     'pyglet', 'glut',
-    'osx',
+    'osx'
 ]
 
 registered = {}

docs/source/config/index.rst

@@ -29,6 +29,7 b' Extending and integrating with IPython'
    extensions/index
    integrating
    custommagics
+   shell_mimerenderer
    inputtransforms
    callbacks
    eventloops