Merge pull request #1 from ipython/master

Merged new changes

IPython/terminal/pt_inputhooks/asyncio.py

@@ -0,0 +1,64 b''
+"""
+Inputhook for running the original asyncio event loop while we're waiting for
+input.
+
+By default, in IPython, we run the prompt with a different asyncio event loop,
+because otherwise we risk that people are freezing the prompt by scheduling bad
+coroutines. E.g., a coroutine that does a while/true and never yield back
+control to the loop. We can't cancel that.
+
+However, sometimes we want the asyncio loop to keep running while waiting for
+a prompt.
+
+The following example will print the numbers from 1 to 10 above the prompt,
+while we are waiting for input. (This works also because we use
+prompt_toolkit`s `patch_stdout`)::
+
+    In [1]: import asyncio
+
+    In [2]: %gui asyncio
+
+    In [3]: async def f():
+       ...:     for i in range(10):
+       ...:         await asyncio.sleep(1)
+       ...:         print(i)
+
+
+    In [4]: asyncio.ensure_future(f())
+
+"""
+import asyncio
+from prompt_toolkit import __version__ as ptk_version
+
+PTK3 = ptk_version.startswith('3.')
+
+
+# Keep reference to the original asyncio loop, because getting the event loop
+# within the input hook would return the other loop.
+loop = asyncio.get_event_loop()
+
+
+def inputhook(context):
+    """
+    Inputhook for asyncio event loop integration.
+    """
+    # For prompt_toolkit 3.0, this input hook literally doesn't do anything.
+    # The event loop integration here is implemented in `interactiveshell.py`
+    # by running the prompt itself in the current asyncio loop. The main reason
+    # for this is that nesting asyncio event loops is unreliable.
+    if PTK3:
+        return
+
+    # For prompt_toolkit 2.0, we can run the current asyncio event loop,
+    # because prompt_toolkit 2.0 uses a different event loop internally.
+
+    def stop():
+        loop.stop()
+
+    fileno = context.fileno()
+    loop.add_reader(fileno, stop)
+    try:
+        loop.run_forever()
+    finally:
+        loop.remove_reader(fileno)
+

docs/source/config/shell_mimerenderer.rst

@@ -0,0 +1,60 b''
+
+.. _shell_mimerenderer:
+
+
+Mime Renderer Extensions
+========================
+
+Like it's cousins, Jupyter Notebooks and JupyterLab, Terminal IPython can be
+thought to render a number of mimetypes in the shell. This can be used to either
+display inline images if your terminal emulator supports it; or open some
+display results with external file viewers.
+
+Registering new mimetype handlers can so far only be done my extensions and
+requires 4 steps:
+
+   - Define a callable that takes 2 parameters:``data`` and ``metadata``; return
+     value of the callable is so far ignored. This callable is responsible for
+     "displaying" the given mimetype. Which can be sending the right escape
+     sequences and bytes to the current terminal; or open an external program. -
+   - Appending the right mimetype to ``ipython.display_formatter.active_types``
+     for IPython to know it should not ignore those mimetypes.
+   - Enabling the given mimetype: ``ipython.display_formatter.formatters[mime].enabled = True``
+   - Registering above callable with mimetype handler:
+     ``ipython.mime_renderers[mime] = handler``
+
+
+Here is a complete IPython extension to display images inline and convert math
+to png, before displaying it inline for iterm2 on macOS ::
+
+
+    from base64 import encodebytes
+    from IPython.lib.latextools import latex_to_png
+
+
+    def mathcat(data, meta):
+        png = latex_to_png(f'$${data}$$'.replace('\displaystyle', '').replace('$$$', '$$'))
+        imcat(png, meta)
+
+    IMAGE_CODE = '\033]1337;File=name=name;inline=true;:{}\a'
+
+    def imcat(image_data, metadata):
+        try:
+            print(IMAGE_CODE.format(encodebytes(image_data).decode()))
+        # bug workaround
+        except:
+            print(IMAGE_CODE.format(image_data))
+
+    def register_mimerenderer(ipython, mime, handler):
+        ipython.display_formatter.active_types.append(mime)
+        ipython.display_formatter.formatters[mime].enabled = True
+        ipython.mime_renderers[mime] = handler
+
+    def load_ipython_extension(ipython):
+        register_mimerenderer(ipython, 'image/png', imcat)
+        register_mimerenderer(ipython, 'image/jpeg', imcat)
+        register_mimerenderer(ipython, 'text/latex', mathcat)
+
+This example only work for iterm2 on macOS and skip error handling for brevity.
+One could also invoke an external viewer with ``subprocess.run()`` and a
+temporary file, which is left as an exercise.

IPython/core/crashhandler.py

@@ -29,6 +29,8 b' from IPython.core.release import author_email'
 from IPython.utils.sysinfo import sys_info
 from IPython.utils.py3compat import input
 
+from IPython.core.release import __version__ as version
+
 #-----------------------------------------------------------------------------
 # Code
 #-----------------------------------------------------------------------------
@@ -68,7 +70,7 b' To ensure accurate tracking of this issue, please file a report about it at:'
 """
 
 _lite_message_template = """
-If you suspect this is an IPython bug, please report it at:
+If you suspect this is an IPython {version} bug, please report it at:
     https://github.com/ipython/ipython/issues
 or send an email to the mailing list at {email}
 
@@ -222,5 +224,5 b' def crash_handler_lite(etype, evalue, tb):'
     else:
         # we are not in a shell, show generic config
         config = "c."
-    print(_lite_message_template.format(email=author_email, config=config), file=sys.stderr)
+    print(_lite_message_template.format(email=author_email, config=config, version=version), file=sys.stderr)
 

IPython/core/displayhook.py

@@ -153,7 +153,7 b' class DisplayHook(Configurable):'
     # This can be set to True by the write_output_prompt method in a subclass
     prompt_end_newline = False
 
-    def write_format_data(self, format_dict, md_dict=None):
+    def write_format_data(self, format_dict, md_dict=None) -> None:
         """Write the format data dict to the frontend.
 
         This default version of this method simply writes the plain text

IPython/core/displaypub.py

@@ -19,7 +19,7 b' spec.'
 import sys
 
 from traitlets.config.configurable import Configurable
-from traitlets import List
+from traitlets import List, Dict
 
 # This used to be defined here - it is imported for backwards compatibility
 from .display import publish_display_data
@@ -28,6 +28,7 b' from .display import publish_display_data'
 # Main payload class
 #-----------------------------------------------------------------------------
 
+
 class DisplayPublisher(Configurable):
     """A traited class that publishes display data to frontends.
 
@@ -35,6 +36,10 b' class DisplayPublisher(Configurable):'
     be accessed there.
     """
 
+    def __init__(self, shell=None, *args, **kwargs):
+        self.shell = shell
+        super().__init__(*args, **kwargs)
+
     def _validate_data(self, data, metadata=None):
         """Validate the display data.
 
@@ -53,7 +58,7 b' class DisplayPublisher(Configurable):'
                 raise TypeError('metadata must be a dict, got: %r' % data)
 
     # use * to indicate transient, update are keyword-only
-    def publish(self, data, metadata=None, source=None, *, transient=None, update=False, **kwargs):
+    def publish(self, data, metadata=None, source=None, *, transient=None, update=False, **kwargs) -> None:
         """Publish data and metadata to all frontends.
 
         See the ``display_data`` message in the messaging documentation for
@@ -98,7 +103,15 b' class DisplayPublisher(Configurable):'
             rather than creating a new output.
         """
 
-        # The default is to simply write the plain text data using sys.stdout.
+        handlers = {}
+        if self.shell is not None:
+            handlers = getattr(self.shell, 'mime_renderers', {})
+
+        for mime, handler in handlers.items():
+            if mime in data:
+                handler(data[mime], metadata.get(mime, None))
+                return
+
         if 'text/plain' in data:
             print(data['text/plain'])
 

IPython/core/interactiveshell.py

@@ -857,7 +857,7 b' class InteractiveShell(SingletonConfigurable):'
         self.configurables.append(self.display_formatter)
 
     def init_display_pub(self):
-        self.display_pub = self.display_pub_class(parent=self)
+        self.display_pub = self.display_pub_class(parent=self, shell=self)
         self.configurables.append(self.display_pub)
 
     def init_data_pub(self):

IPython/core/magics/execution.py

@@ -853,6 +853,8 b' python-profiler package from non-free.""")'
             sys.argv = save_argv
             if restore_main:
                 sys.modules['__main__'] = restore_main
+                if '__mp_main__' in sys.modules:
+                    sys.modules['__mp_main__'] = restore_main
             else:
                 # Remove from sys.modules the reference to main_mod we'd
                 # added.  Otherwise it will trap references to objects

IPython/core/oinspect.py

@@ -76,7 +76,7 b" info_fields = ['type_name', 'base_class', 'string_form', 'namespace',"
                'call_def', 'call_docstring',
                # These won't be printed but will be used to determine how to
                # format the object
-               'ismagic', 'isalias', 'isclass', 'argspec', 'found', 'name'
+               'ismagic', 'isalias', 'isclass', 'found', 'name'
                ]
 
 
@@ -200,26 +200,39 b' def is_simple_callable(obj):'
     return (inspect.isfunction(obj) or inspect.ismethod(obj) or \
             isinstance(obj, _builtin_func_type) or isinstance(obj, _builtin_meth_type))
 
-
+@undoc
 def getargspec(obj):
     """Wrapper around :func:`inspect.getfullargspec` on Python 3, and
     :func:inspect.getargspec` on Python 2.
 
     In addition to functions and methods, this can also handle objects with a
     ``__call__`` attribute.
+
+    DEPRECATED: Deprecated since 7.10. Do not use, will be removed.
     """
+
+    warnings.warn('`getargspec` function is deprecated as of IPython 7.10'
+                  'and will be removed in future versions.', DeprecationWarning, stacklevel=2)
+
     if safe_hasattr(obj, '__call__') and not is_simple_callable(obj):
         obj = obj.__call__
 
     return inspect.getfullargspec(obj)
 
-
+@undoc
 def format_argspec(argspec):
     """Format argspect, convenience wrapper around inspect's.
 
     This takes a dict instead of ordered arguments and calls
     inspect.format_argspec with the arguments in the necessary order.
+
+    DEPRECATED: Do not use; will be removed in future versions.
     """
+    
+    warnings.warn('`format_argspec` function is deprecated as of IPython 7.10'
+                  'and will be removed in future versions.', DeprecationWarning, stacklevel=2)
+
+
     return inspect.formatargspec(argspec['args'], argspec['varargs'],
                                  argspec['varkw'], argspec['defaults'])
 
@@ -916,33 +929,6 b' class Inspector(Colorable):'
                 if call_ds:
                     out['call_docstring'] = call_ds
 
-        # Compute the object's argspec as a callable.  The key is to decide
-        # whether to pull it from the object itself, from its __init__ or
-        # from its __call__ method.
-
-        if inspect.isclass(obj):
-            # Old-style classes need not have an __init__
-            callable_obj = getattr(obj, "__init__", None)
-        elif callable(obj):
-            callable_obj = obj
-        else:
-            callable_obj = None
-
-        if callable_obj is not None:
-            try:
-                argspec = getargspec(callable_obj)
-            except Exception:
-                # For extensions/builtins we can't retrieve the argspec
-                pass
-            else:
-                # named tuples' _asdict() method returns an OrderedDict, but we
-                # we want a normal
-                out['argspec'] = argspec_dict = dict(argspec._asdict())
-                # We called this varkw before argspec became a named tuple.
-                # With getfullargspec it's also called varkw.
-                if 'varkw' not in argspec_dict:
-                    argspec_dict['varkw'] = argspec_dict.pop('keywords')
-
         return object_info(**out)
 
     @staticmethod

IPython/core/page.py

@@ -15,9 +15,11 b' rid of that dependency, we could move it there.'
 
 
 import os
+import io
 import re
 import sys
 import tempfile
+import subprocess
 
 from io import UnsupportedOperation
 
@@ -208,9 +210,13 b' def pager_page(strng, start=0, screen_lines=0, pager_cmd=None):'
         else:
             try:
                 retval = None
-                # if I use popen4, things hang. No idea why.
-                #pager,shell_out = os.popen4(pager_cmd)
-                pager = os.popen(pager_cmd, 'w')
+                # Emulate os.popen, but redirect stderr
+                proc = subprocess.Popen(pager_cmd,
+                                shell=True,
+                                stdin=subprocess.PIPE,
+                                stderr=subprocess.DEVNULL
+                                )
+                pager = os._wrap_close(io.TextIOWrapper(proc.stdin), proc)
                 try:
                     pager_encoding = pager.encoding or sys.stdout.encoding
                     pager.write(strng)

IPython/core/release.py

@@ -20,7 +20,7 b" name = 'ipython'"
 # release.  'dev' as a _version_extra string means this is a development
 # version
 _version_major = 7
-_version_minor = 10
+_version_minor = 11
 _version_patch = 0
 _version_extra = '.dev'
 # _version_extra = 'b1'

IPython/core/tests/test_autocall.py

@@ -7,7 +7,6 b" with better-isolated tests that don't rely on the global instance in iptest."
 """
 from IPython.core.splitinput import LineInfo
 from IPython.core.prefilter import AutocallChecker
-from IPython.utils import py3compat
 
 def doctest_autocall():
     """

IPython/core/tests/test_handlers.py

@@ -49,11 +49,11 b' def test_handlers():'
     # For many of the below, we're also checking that leading whitespace
     # turns off the esc char, which it should unless there is a continuation
     # line.
-    run([(i,py3compat.u_format(o)) for i,o in \
+    run(
         [('"no change"', '"no change"'),             # normal
          (u"lsmagic",     "get_ipython().run_line_magic('lsmagic', '')"),   # magic
          #("a = b # PYTHON-MODE", '_i'),          # emacs -- avoids _in cache
-         ]])
+         ])
 
     # Objects which are instances of IPyAutocall are *always* autocalled
     autocallable = Autocallable()

IPython/core/tests/test_run.py

@@ -539,6 +539,35 b' def test_run_tb():'
         del ip.user_ns['bar']
         del ip.user_ns['foo']
 
+
+def test_multiprocessing_run():
+    """Set we can run mutiprocesgin without messing up up main namespace
+
+    Note that import `nose.tools as nt` mdify the value s
+    sys.module['__mp_main__'] so wee need to temporarily set it to None to test
+    the issue.
+    """
+    with TemporaryDirectory() as td:
+        mpm = sys.modules.get('__mp_main__')
+        assert mpm is not None
+        sys.modules['__mp_main__'] = None
+        try:
+            path = pjoin(td, 'test.py')
+            with open(path, 'w') as f:
+                f.write("import multiprocessing\nprint('hoy')")
+            with capture_output() as io:
+                _ip.run_line_magic('run', path)
+                _ip.run_cell("i_m_undefined")
+            out = io.stdout
+            nt.assert_in("hoy", out)
+            nt.assert_not_in("AttributeError", out)
+            nt.assert_in("NameError", out)
+            nt.assert_equal(out.count("---->"), 1)
+        except:
+            raise
+        finally:
+            sys.modules['__mp_main__'] = mpm
+
 @dec.knownfailureif(sys.platform == 'win32', "writes to io.stdout aren't captured on Windows")
 def test_script_tb():
     """Test traceback offset in `ipython script.py`"""

IPython/core/tests/test_ultratb.py

@@ -296,6 +296,25 b' except Exception:'
              tt.AssertPrints("ValueError", suppress=False):
             ip.run_cell(self.SUPPRESS_CHAINING_CODE)
 
+    def test_plain_direct_cause_error(self):
+        with tt.AssertPrints(["KeyError", "NameError", "direct cause"]):
+            ip.run_cell("%xmode Plain")
+            ip.run_cell(self.DIRECT_CAUSE_ERROR_CODE)
+            ip.run_cell("%xmode Verbose")
+
+    def test_plain_exception_during_handling_error(self):
+        with tt.AssertPrints(["KeyError", "NameError", "During handling"]):
+            ip.run_cell("%xmode Plain")
+            ip.run_cell(self.EXCEPTION_DURING_HANDLING_CODE)
+            ip.run_cell("%xmode Verbose")
+
+    def test_plain_suppress_exception_chaining(self):
+        with tt.AssertNotPrints("ZeroDivisionError"), \
+             tt.AssertPrints("ValueError", suppress=False):
+            ip.run_cell("%xmode Plain")
+            ip.run_cell(self.SUPPRESS_CHAINING_CODE)
+            ip.run_cell("%xmode Verbose")
+
 
 class RecursionTest(unittest.TestCase):
     DEFINITIONS = """

IPython/core/ultratb.py

@@ -530,6 +530,30 b' class TBTools(colorable.Colorable):'
 
     ostream = property(_get_ostream, _set_ostream)
 
+    def get_parts_of_chained_exception(self, evalue):
+        def get_chained_exception(exception_value):
+            cause = getattr(exception_value, '__cause__', None)
+            if cause:
+                return cause
+            if getattr(exception_value, '__suppress_context__', False):
+                return None
+            return getattr(exception_value, '__context__', None)
+
+        chained_evalue = get_chained_exception(evalue)
+
+        if chained_evalue:
+            return chained_evalue.__class__, chained_evalue, chained_evalue.__traceback__
+
+    def prepare_chained_exception_message(self, cause):
+        direct_cause = "\nThe above exception was the direct cause of the following exception:\n"
+        exception_during_handling = "\nDuring handling of the above exception, another exception occurred:\n"
+
+        if cause:
+            message = [[direct_cause]]
+        else:
+            message = [[exception_during_handling]]
+        return message
+
     def set_colors(self, *args, **kw):
         """Shorthand access to the color table scheme selector method."""
 
@@ -603,7 +627,7 b' class ListTB(TBTools):'
         self.ostream.write(self.text(etype, value, elist))
         self.ostream.write('\n')
 
-    def structured_traceback(self, etype, value, elist, tb_offset=None,
+    def structured_traceback(self, etype, evalue, etb=None, tb_offset=None,
                              context=5):
         """Return a color formatted string with the traceback info.
 
@@ -612,15 +636,16 b' class ListTB(TBTools):'
         etype : exception type
           Type of the exception raised.
 
-        value : object
+        evalue : object
           Data stored in the exception
 
-        elist : list
-          List of frames, see class docstring for details.
+        etb : object
+          If list: List of frames, see class docstring for details.
+          If Traceback: Traceback of the exception.
 
         tb_offset : int, optional
           Number of frames in the traceback to skip.  If not given, the
-          instance value is used (set in constructor).
+          instance evalue is used (set in constructor).
 
         context : int, optional
           Number of lines of context information to print.
@@ -629,6 +654,19 b' class ListTB(TBTools):'
         -------
         String with formatted exception.
         """
+        # This is a workaround to get chained_exc_ids in recursive calls
+        # etb should not be a tuple if structured_traceback is not recursive
+        if isinstance(etb, tuple):
+            etb, chained_exc_ids = etb
+        else:
+            chained_exc_ids = set()
+
+        if isinstance(etb, list):
+            elist = etb
+        elif etb is not None:
+            elist = self._extract_tb(etb)
+        else:
+            elist = []
         tb_offset = self.tb_offset if tb_offset is None else tb_offset
         Colors = self.Colors
         out_list = []
@@ -641,9 +679,25 b' class ListTB(TBTools):'
                             (Colors.normalEm, Colors.Normal) + '\n')
             out_list.extend(self._format_list(elist))
         # The exception info should be a single entry in the list.
-        lines = ''.join(self._format_exception_only(etype, value))
+        lines = ''.join(self._format_exception_only(etype, evalue))
         out_list.append(lines)
 
+        exception = self.get_parts_of_chained_exception(evalue)
+
+        if exception and not id(exception[1]) in chained_exc_ids:
+            chained_exception_message = self.prepare_chained_exception_message(
+                evalue.__cause__)[0]
+            etype, evalue, etb = exception
+            # Trace exception to avoid infinite 'cause' loop
+            chained_exc_ids.add(id(exception[1]))
+            chained_exceptions_tb_offset = 0
+            out_list = (
+                self.structured_traceback(
+                    etype, evalue, (etb, chained_exc_ids),
+                    chained_exceptions_tb_offset, context)
+                + chained_exception_message
+                + out_list)
+
         return out_list
 
     def _format_list(self, extracted_list):
@@ -763,7 +817,7 b' class ListTB(TBTools):'
         etype : exception type
         value : exception value
         """
-        return ListTB.structured_traceback(self, etype, value, [])
+        return ListTB.structured_traceback(self, etype, value)
 
     def show_exception_only(self, etype, evalue):
         """Only print the exception type and message, without a traceback.
@@ -1013,16 +1067,6 b' class VerboseTB(TBTools):'
                 _format_traceback_lines(lnum, index, lines, Colors, lvals,
                                          _line_format)))
 
-    def prepare_chained_exception_message(self, cause):
-        direct_cause = "\nThe above exception was the direct cause of the following exception:\n"
-        exception_during_handling = "\nDuring handling of the above exception, another exception occurred:\n"
-
-        if cause:
-            message = [[direct_cause]]
-        else:
-            message = [[exception_during_handling]]
-        return message
-
     def prepare_header(self, etype, long_version=False):
         colors = self.Colors  # just a shorthand + quicker name lookup
         colorsnormal = colors.Normal  # used a lot
@@ -1117,20 +1161,6 b' class VerboseTB(TBTools):'
             info('\nUnfortunately, your original traceback can not be constructed.\n')
             return None
 
-    def get_parts_of_chained_exception(self, evalue):
-        def get_chained_exception(exception_value):
-            cause = getattr(exception_value, '__cause__', None)
-            if cause:
-                return cause
-            if getattr(exception_value, '__suppress_context__', False):
-                return None
-            return getattr(exception_value, '__context__', None)
-
-        chained_evalue = get_chained_exception(evalue)
-
-        if chained_evalue:
-            return chained_evalue.__class__, chained_evalue, chained_evalue.__traceback__
-
     def structured_traceback(self, etype, evalue, etb, tb_offset=None,
                              number_of_lines_of_context=5):
         """Return a nice text document describing the traceback."""
@@ -1294,9 +1324,8 b' class FormattedTB(VerboseTB, ListTB):'
             # out-of-date source code.
             self.check_cache()
             # Now we can extract and format the exception
-            elist = self._extract_tb(tb)
             return ListTB.structured_traceback(
-                self, etype, value, elist, tb_offset, number_of_lines_of_context
+                self, etype, value, tb, tb_offset, number_of_lines_of_context
             )
 
     def stb2text(self, stb):

IPython/lib/clipboard.py

@@ -32,15 +32,15 b' def win32_clipboard_get():'
         win32clipboard.CloseClipboard()
     return text
 
-def osx_clipboard_get():
+def osx_clipboard_get() -> str:
     """ Get the clipboard's text on OS X.
     """
     p = subprocess.Popen(['pbpaste', '-Prefer', 'ascii'],
         stdout=subprocess.PIPE)
-    text, stderr = p.communicate()
+    bytes_, stderr = p.communicate()
     # Text comes in with old Mac \r line endings. Change them to \n.
-    text = text.replace(b'\r', b'\n')
-    text = py3compat.cast_unicode(text, py3compat.DEFAULT_ENCODING)
+    bytes_ = bytes_.replace(b'\r', b'\n')
+    text = py3compat.decode(bytes_)
     return text
 
 def tkinter_clipboard_get():

IPython/lib/pretty.py

@@ -97,9 +97,6 b" __all__ = ['pretty', 'pprint', 'PrettyPrinter', 'RepresentationPrinter',"
 
 
 MAX_SEQ_LENGTH = 1000
-# The language spec says that dicts preserve order from 3.7, but CPython
-# does so from 3.6, so it seems likely that people will expect that.
-DICT_IS_ORDERED = True
 _re_pattern_type = type(re.compile(''))
 
 def _safe_getattr(obj, attr, default=None):
@@ -606,11 +603,6 b' def _dict_pprinter_factory(start, end):'
         step = len(start)
         p.begin_group(step, start)
         keys = obj.keys()
-        # if dict isn't large enough to be truncated, sort keys before displaying
-        # From Python 3.7, dicts preserve order by definition, so we don't sort.
-        if not DICT_IS_ORDERED \
-                and not (p.max_seq_length and len(obj) >= p.max_seq_length):
-            keys = _sorted_for_pprint(keys)
         for idx, key in p._enumerate(keys):
             if idx:
                 p.text(',')

IPython/paths.py

@@ -12,7 +12,7 b' from IPython.utils.path import ('
     ensure_dir_exists, fs_encoding)
 from IPython.utils import py3compat
 
-def get_ipython_dir():
+def get_ipython_dir() -> str:
     """Get the IPython directory for this platform and user.
 
     This uses the logic in `get_home_dir` to find the home directory
@@ -28,10 +28,9 b' def get_ipython_dir():'
     home_dir = get_home_dir()
     xdg_dir = get_xdg_dir()
 
-    # import pdb; pdb.set_trace()  # dbg
     if 'IPYTHON_DIR' in env:
-        warn('The environment variable IPYTHON_DIR is deprecated. '
-                'Please use IPYTHONDIR instead.')
+        warn('The environment variable IPYTHON_DIR is deprecated since IPython 3.0. '
+                'Please use IPYTHONDIR instead.', DeprecationWarning)
     ipdir = env.get('IPYTHONDIR', env.get('IPYTHON_DIR', None))
     if ipdir is None:
         # not set explicitly, use ~/.ipython
@@ -67,11 +66,11 b' def get_ipython_dir():'
             warn("IPython parent '{0}' is not a writable location,"
                     " using a temp directory.".format(parent))
             ipdir = tempfile.mkdtemp()
+    assert isinstance(ipdir, str), "all path manipulation should be str(unicode), but are not."
+    return ipdir
 
-    return py3compat.cast_unicode(ipdir, fs_encoding)
 
-
-def get_ipython_cache_dir():
+def get_ipython_cache_dir() -> str:
     """Get the cache directory it is created if it does not exist."""
     xdgdir = get_xdg_cache_dir()
     if xdgdir is None:
@@ -82,13 +81,14 b' def get_ipython_cache_dir():'
     elif not _writable_dir(xdgdir):
         return get_ipython_dir()
 
-    return py3compat.cast_unicode(ipdir, fs_encoding)
+    return ipdir
 
 
-def get_ipython_package_dir():
+def get_ipython_package_dir() -> str:
     """Get the base directory where IPython itself is installed."""
     ipdir = os.path.dirname(IPython.__file__)
-    return py3compat.cast_unicode(ipdir, fs_encoding)
+    assert isinstance(ipdir, str)
+    return ipdir
 
 
 def get_ipython_module_path(module_str):

IPython/terminal/debugger.py

@@ -17,6 +17,9 b' from prompt_toolkit.shortcuts.prompt import PromptSession'
 from prompt_toolkit.enums import EditingMode
 from prompt_toolkit.formatted_text import PygmentsTokens
 
+from prompt_toolkit import __version__ as ptk_version
+PTK3 = ptk_version.startswith('3.')
+
 
 class TerminalPdb(Pdb):
     """Standalone IPython debugger."""
@@ -49,20 +52,23 b' class TerminalPdb(Pdb):'
                                   & ~cursor_in_leading_ws
                               ))(display_completions_like_readline)
 
-        self.pt_app = PromptSession(
-                            message=(lambda: PygmentsTokens(get_prompt_tokens())),
-                            editing_mode=getattr(EditingMode, self.shell.editing_mode.upper()),
-                            key_bindings=kb,
-                            history=self.shell.debugger_history,
-                            completer=self._ptcomp,
-                            enable_history_search=True,
-                            mouse_support=self.shell.mouse_support,
-                            complete_style=self.shell.pt_complete_style,
-                            style=self.shell.style,
-                            inputhook=self.shell.inputhook,
-                            color_depth=self.shell.color_depth,
+        options = dict(
+            message=(lambda: PygmentsTokens(get_prompt_tokens())),
+            editing_mode=getattr(EditingMode, self.shell.editing_mode.upper()),
+            key_bindings=kb,
+            history=self.shell.debugger_history,
+            completer=self._ptcomp,
+            enable_history_search=True,
+            mouse_support=self.shell.mouse_support,
+            complete_style=self.shell.pt_complete_style,
+            style=self.shell.style,
+            color_depth=self.shell.color_depth,
         )
 
+        if not PTK3:
+            options['inputhook'] = self.shell.inputhook
+        self.pt_app = PromptSession(**options)
+
     def cmdloop(self, intro=None):
         """Repeatedly issue a prompt, accept input, parse an initial prefix
         off the received input, and dispatch to action methods, passing them

IPython/terminal/interactiveshell.py

@@ -1,5 +1,6 b''
 """IPython terminal interface using prompt_toolkit"""
 
+import asyncio
 import os
 import sys
 import warnings
@@ -25,6 +26,7 b' from prompt_toolkit.patch_stdout import patch_stdout'
 from prompt_toolkit.shortcuts import PromptSession, CompleteStyle, print_formatted_text
 from prompt_toolkit.styles import DynamicStyle, merge_styles
 from prompt_toolkit.styles.pygments import style_from_pygments_cls, style_from_pygments_dict
+from prompt_toolkit import __version__ as ptk_version
 
 from pygments.styles import get_style_by_name
 from pygments.style import Style
@@ -38,6 +40,7 b' from .ptutils import IPythonPTCompleter, IPythonPTLexer'
 from .shortcuts import create_ipython_shortcuts
 
 DISPLAY_BANNER_DEPRECATED = object()
+PTK3 = ptk_version.startswith('3.')
 
 
 class _NoStyle(Style): pass
@@ -87,7 +90,17 b' else:'
 
 _use_simple_prompt = ('IPY_TEST_SIMPLE_PROMPT' in os.environ) or (not _is_tty)
 
+def black_reformat_handler(text_before_cursor):
+    import black
+    formatted_text = black.format_str(text_before_cursor, mode=black.FileMode())
+    if not text_before_cursor.endswith('\n') and formatted_text.endswith('\n'):
+       formatted_text = formatted_text[:-1]
+    return formatted_text
+
+
 class TerminalInteractiveShell(InteractiveShell):
+    mime_renderers = Dict().tag(config=True)
+
     space_for_menu = Integer(6, help='Number of line at the bottom of the screen '
                                      'to reserve for the completion menu'
                             ).tag(config=True)
@@ -120,6 +133,11 b' class TerminalInteractiveShell(InteractiveShell):'
         help="Shortcut style to use at the prompt. 'vi' or 'emacs'.",
     ).tag(config=True)
 
+    autoformatter = Unicode(None,
+        help="Autoformatter to reformat Terminal code. Can be `'black'` or `None`",
+        allow_none=True
+    ).tag(config=True)
+
     mouse_support = Bool(False,
         help="Enable mouse support in the prompt\n(Note: prevents selecting text with the mouse)"
     ).tag(config=True)
@@ -150,6 +168,16 b' class TerminalInteractiveShell(InteractiveShell):'
         if self.pt_app:
             self.pt_app.editing_mode = u_mode
 
+    @observe('autoformatter')
+    def _autoformatter_changed(self, change):
+        formatter = change.new
+        if formatter is None:
+            self.reformat_handler = lambda x:x
+        elif formatter == 'black':
+            self.reformat_handler = black_reformat_handler
+        else:
+            raise ValueError
+
     @observe('highlighting_style')
     @observe('colors')
     def _highlighting_style_changed(self, change):
@@ -282,6 +310,7 b' class TerminalInteractiveShell(InteractiveShell):'
 
         editing_mode = getattr(EditingMode, self.editing_mode.upper())
 
+        self.pt_loop = asyncio.new_event_loop()
         self.pt_app = PromptSession(
                             editing_mode=editing_mode,
                             key_bindings=key_bindings,
@@ -390,7 +419,7 b' class TerminalInteractiveShell(InteractiveShell):'
             # work around this.
             get_message = get_message()
 
-        return {
+        options = {
                 'complete_in_thread': False,
                 'lexer':IPythonPTLexer(),
                 'reserve_space_for_menu':self.space_for_menu,
@@ -407,8 +436,11 b' class TerminalInteractiveShell(InteractiveShell):'
                         processor=HighlightMatchingBracketProcessor(chars='[](){}'),
                         filter=HasFocus(DEFAULT_BUFFER) & ~IsDone() &
                             Condition(lambda: self.highlight_matching_brackets))],
-                'inputhook': self.inputhook,
                 }
+        if not PTK3:
+            options['inputhook'] = self.inputhook
+
+        return options
 
     def prompt_for_code(self):
         if self.rl_next_input:
@@ -417,11 +449,28 b' class TerminalInteractiveShell(InteractiveShell):'
         else:
             default = ''
 
-        with patch_stdout(raw=True):
-            text = self.pt_app.prompt(
-                default=default,
-#                pre_run=self.pre_prompt,# reset_current_buffer=True,
-                **self._extra_prompt_options())
+        # In order to make sure that asyncio code written in the
+        # interactive shell doesn't interfere with the prompt, we run the
+        # prompt in a different event loop.
+        # If we don't do this, people could spawn coroutine with a
+        # while/true inside which will freeze the prompt.
+
+        try:
+            old_loop = asyncio.get_event_loop()
+        except RuntimeError:
+            # This happens when the user used `asyncio.run()`.
+            old_loop = None
+
+        asyncio.set_event_loop(self.pt_loop)
+        try:
+            with patch_stdout(raw=True):
+                text = self.pt_app.prompt(
+                    default=default,
+                    **self._extra_prompt_options())
+        finally:
+            # Restore the original event loop.
+            asyncio.set_event_loop(old_loop)
+
         return text
 
     def enable_win_unicode_console(self):
@@ -528,12 +577,33 b' class TerminalInteractiveShell(InteractiveShell):'
 
     active_eventloop = None
     def enable_gui(self, gui=None):
-        if gui:
+        if gui and (gui != 'inline') :
             self.active_eventloop, self._inputhook =\
                 get_inputhook_name_and_func(gui)
         else:
             self.active_eventloop = self._inputhook = None
 
+        # For prompt_toolkit 3.0. We have to create an asyncio event loop with
+        # this inputhook.
+        if PTK3:
+            import asyncio
+            from prompt_toolkit.eventloop import new_eventloop_with_inputhook
+
+            if gui == 'asyncio':
+                # When we integrate the asyncio event loop, run the UI in the
+                # same event loop as the rest of the code. don't use an actual
+                # input hook. (Asyncio is not made for nesting event loops.)
+                self.pt_loop = asyncio.get_event_loop()
+
+            elif self._inputhook:
+                # If an inputhook was set, create a new asyncio event loop with
+                # this inputhook for the prompt.
+                self.pt_loop = new_eventloop_with_inputhook(self._inputhook)
+            else:
+                # When there's no inputhook, run the prompt in a separate
+                # asyncio event loop.
+                self.pt_loop = asyncio.new_event_loop()
+
     # Run !system commands directly, not through pipes, so terminal programs
     # work correctly.
     system = InteractiveShell.system_raw

IPython/terminal/prompts.py

@@ -89,3 +89,14 b' class RichPromptDisplayHook(DisplayHook):'
                 )
             else:
                 sys.stdout.write(prompt_txt)
+
+    def write_format_data(self, format_dict, md_dict=None) -> None:
+        if self.shell.mime_renderers:
+
+            for mime, handler in self.shell.mime_renderers.items():
+                if mime in format_dict:
+                    handler(format_dict[mime], None)
+                    return
+                
+        super().write_format_data(format_dict, md_dict)
+

IPython/terminal/pt_inputhooks/__init__.py

@@ -13,6 +13,7 b' backends = ['
     'wx',
     'pyglet', 'glut',
     'osx',
+    'asyncio'
 ]
 
 registered = {}

IPython/terminal/shortcuts.py

@@ -44,6 +44,15 b' def create_ipython_shortcuts(shell):'
                             & insert_mode
                         ))(return_handler)
 
+    def reformat_and_execute(event):
+        reformat_text_before_cursor(event.current_buffer, event.current_buffer.document, shell)
+        event.current_buffer.validate_and_handle()
+
+    kb.add('escape', 'enter', filter=(has_focus(DEFAULT_BUFFER)
+                            & ~has_selection
+                            & insert_mode
+                                      ))(reformat_and_execute)
+
     kb.add('c-\\')(force_exit)
 
     kb.add('c-p', filter=(vi_insert_mode & has_focus(DEFAULT_BUFFER))
@@ -86,7 +95,17 b' def create_ipython_shortcuts(shell):'
     return kb
 
 
+def reformat_text_before_cursor(buffer, document, shell):
+    text = buffer.delete_before_cursor(len(document.text[:document.cursor_position]))
+    try:
+        formatted_text = shell.reformat_handler(text)
+        buffer.insert_text(formatted_text)
+    except Exception as e:
+        buffer.insert_text(text)
+
+
 def newline_or_execute_outer(shell):
+
     def newline_or_execute(event):
         """When the user presses return, insert a newline or execute the code."""
         b = event.current_buffer
@@ -107,7 +126,12 b' def newline_or_execute_outer(shell):'
         else:
             check_text = d.text[:d.cursor_position]
         status, indent = shell.check_complete(check_text)
-
+       
+        # if all we have after the cursor is whitespace: reformat current text
+        # before cursor
+        after_cursor = d.text[d.cursor_position:]
+        if not after_cursor.strip():
+            reformat_text_before_cursor(b, d, shell)
         if not (d.on_last_line or
                 d.cursor_position_row >= d.line_count - d.empty_line_count_at_the_end()
                 ):
@@ -118,6 +142,7 b' def newline_or_execute_outer(shell):'
             return
 
         if (status != 'incomplete') and b.accept_handler:
+            reformat_text_before_cursor(b, d, shell)
             b.validate_and_handle()
         else:
             if shell.autoindent:

IPython/utils/path.py

@@ -169,7 +169,7 b' class HomeDirError(Exception):'
     pass
 
 
-def get_home_dir(require_writable=False):
+def get_home_dir(require_writable=False) -> str:
     """Return the 'home' directory, as a unicode string.
 
     Uses os.path.expanduser('~'), and checks for writability.
@@ -197,21 +197,18 b' def get_home_dir(require_writable=False):'
     if not _writable_dir(homedir) and os.name == 'nt':
         # expanduser failed, use the registry to get the 'My Documents' folder.
         try:
-            try:
-                import winreg as wreg  # Py 3
-            except ImportError:
-                import _winreg as wreg  # Py 2
-            key = wreg.OpenKey(
+            import winreg as wreg
+            with wreg.OpenKey(
                 wreg.HKEY_CURRENT_USER,
                 r"Software\Microsoft\Windows\CurrentVersion\Explorer\Shell Folders"
-            )
-            homedir = wreg.QueryValueEx(key,'Personal')[0]
-            key.Close()
+            ) as key:
+                homedir = wreg.QueryValueEx(key,'Personal')[0]
         except:
             pass
 
     if (not require_writable) or _writable_dir(homedir):
-        return py3compat.cast_unicode(homedir, fs_encoding)
+        assert isinstance(homedir, str), "Homedir shoudl be unicode not bytes"
+        return homedir
     else:
         raise HomeDirError('%s is not a writable dir, '
                 'set $HOME environment variable to override' % homedir)
@@ -254,31 +251,31 b' def get_xdg_cache_dir():'
 
 @undoc
 def get_ipython_dir():
-    warn("get_ipython_dir has moved to the IPython.paths module since IPython 4.0.", stacklevel=2)
+    warn("get_ipython_dir has moved to the IPython.paths module since IPython 4.0.", DeprecationWarning, stacklevel=2)
     from IPython.paths import get_ipython_dir
     return get_ipython_dir()
 
 @undoc
 def get_ipython_cache_dir():
-    warn("get_ipython_cache_dir has moved to the IPython.paths module since IPython 4.0.", stacklevel=2)
+    warn("get_ipython_cache_dir has moved to the IPython.paths module since IPython 4.0.", DeprecationWarning, stacklevel=2)
     from IPython.paths import get_ipython_cache_dir
     return get_ipython_cache_dir()
 
 @undoc
 def get_ipython_package_dir():
-    warn("get_ipython_package_dir has moved to the IPython.paths module since IPython 4.0.", stacklevel=2)
+    warn("get_ipython_package_dir has moved to the IPython.paths module since IPython 4.0.", DeprecationWarning, stacklevel=2)
     from IPython.paths import get_ipython_package_dir
     return get_ipython_package_dir()
 
 @undoc
 def get_ipython_module_path(module_str):
-    warn("get_ipython_module_path has moved to the IPython.paths module since IPython 4.0.", stacklevel=2)
+    warn("get_ipython_module_path has moved to the IPython.paths module since IPython 4.0.", DeprecationWarning, stacklevel=2)
     from IPython.paths import get_ipython_module_path
     return get_ipython_module_path(module_str)
 
 @undoc
 def locate_profile(profile='default'):
-    warn("locate_profile has moved to the IPython.paths module since IPython 4.0.", stacklevel=2)
+    warn("locate_profile has moved to the IPython.paths module since IPython 4.0.", DeprecationWarning, stacklevel=2)
     from IPython.paths import locate_profile
     return locate_profile(profile=profile)
 

IPython/utils/tests/test_path.py

@@ -171,8 +171,12 b' def test_get_home_dir_8():'
         env.pop(key, None)
 
     class key:
+        def __enter__(self):
+            pass
         def Close(self):
             pass
+        def __exit__(*args, **kwargs):
+            pass
 
     with patch.object(wreg, 'OpenKey', return_value=key()), \
          patch.object(wreg, 'QueryValueEx', return_value=[abspath(HOME_TEST_DIR)]):

README.rst

@@ -117,3 +117,15 b' version::'
         ...
         install_requires=install_req
     )
+
+Alternatives to IPython
+=======================
+
+IPython may not be to your taste; if that's the case there might be similar
+project that you might want to use:
+
+- the classic Python REPL.
+- `bpython <https://bpython-interpreter.org/>`_
+- `mypython <https://www.asmeurer.com/mypython/>`_
+- `ptpython and ptipython <https://pypi.org/project/ptpython/>`
+- `xonsh <https://xon.sh/>`

docs/source/config/index.rst

@@ -29,6 +29,7 b' Extending and integrating with IPython'
    extensions/index
    integrating
    custommagics
+   shell_mimerenderer
    inputtransforms
    callbacks
    eventloops

docs/source/development/execution.rst

@@ -1,13 +1,14 b''
 .. _execution_semantics:
 
-Execution semantics in the IPython kernel
-=========================================
+Execution of cells in the IPython kernel
+========================================
 
-The execution of user code consists of the following phases:
+When IPython kernel receives `execute_request <https://jupyter-client.readthedocs.io/en/latest/messaging.html#execute>`_
+with user code, it processes the message in the following phases:
 
 1. Fire the ``pre_execute`` event.
 2. Fire the ``pre_run_cell`` event unless silent is ``True``.
-3. Execute the ``code`` field, see below for details.
+3. Execute ``run_cell`` method to preprocess ``code``, compile and run it, see below for details.
 4. If execution succeeds, expressions in ``user_expressions`` are computed.
    This ensures that any error in the expressions don't affect the main code execution.
 5. Fire the ``post_execute`` event.
@@ -18,9 +19,15 b' The execution of user code consists of the following phases:'
     :doc:`/config/callbacks`
 
 
-To understand how the ``code`` field is executed, one must know that Python
-code can be compiled in one of three modes (controlled by the ``mode`` argument
-to the :func:`compile` builtin):
+Running user ``code``
+=====================
+
+First, the ``code`` cell is transformed to expand ``%magic`` and ``!system``
+commands by ``IPython.core.inputtransformer2``. Then expanded cell is compiled
+using standard Python :func:`compile` function and executed.
+
+Python :func:`compile` function provides ``mode`` argument to select one
+of three ways of compiling code:
 
 *single*
   Valid for a single interactive statement (though the source can contain
@@ -50,16 +57,16 b" execution in 'single' mode, and then:"
 
 - If there is more than one block:
 
-  * if the last one is a single line long, run all but the last in 'exec' mode
+  * if the last block is a single line long, run all but the last in 'exec' mode
     and the very last one in 'single' mode.  This makes it easy to type simple
     expressions at the end to see computed values.
 
-  * if the last one is no more than two lines long, run all but the last in
+  * if the last block is no more than two lines long, run all but the last in
     'exec' mode and the very last one in 'single' mode.  This makes it easy to
     type simple expressions at the end to see computed values.  - otherwise
     (last one is also multiline), run all in 'exec' mode
 
-  * otherwise (last one is also multiline), run all in 'exec' mode as a single
+  * otherwise (last block is also multiline), run all in 'exec' mode as a single
     unit.
 
 

docs/source/whatsnew/development.rst

@@ -11,7 +11,7 b' This document describes in-flight development work.'
     `docs/source/whatsnew/pr` folder
 
 
-Released .... ...., 2017
+Released .... ...., 2019
 
 
 Need to be updated:
@@ -22,8 +22,6 b' Need to be updated:'
 
    pr/*
 
-
-
 .. DO NOT EDIT THIS LINE BEFORE RELEASE. FEATURE INSERTION POINT.
 
 

docs/source/whatsnew/github-stats-7.rst

@@ -1,6 +1,48 b''
 Issues closed in the 7.x development cycle
 ==========================================
 
+Issues closed in 7.10.1
+-----------------------
+
+GitHub stats for 2019/11/27 - 2019/12/01 (tag: 7.10.0)
+
+These lists are automatically generated, and may be incomplete or contain duplicates.
+
+We closed 5 issues and merged 7 pull requests.
+The full list can be seen `on GitHub <https://github.com/ipython/ipython/issues?q=milestone%3A7.10.1>`__
+
+The following 2 authors contributed 14 commits.
+
+* Jonathan Slenders
+* Matthias Bussonnier
+
+Issues closed in 7.10
+---------------------
+
+GitHub stats for 2019/10/25 - 2019/11/27 (tag: 7.9.0)
+
+These lists are automatically generated, and may be incomplete or contain duplicates.
+
+We closed 4 issues and merged 22 pull requests.
+The full list can be seen `on GitHub <https://github.com/ipython/ipython/issues?q=milestone%3A7.10>`__
+
+The following 15 authors contributed 101 commits.
+
+* anatoly techtonik
+* Ben Lewis
+* Benjamin Ragan-Kelley
+* Gerrit Buss
+* grey275
+* Gökcen Eraslan
+* Jonathan Slenders
+* Joris Van den Bossche
+* kousik
+* Matthias Bussonnier
+* Nicholas Bollweg
+* Paul McCarthy
+* Srinivas Reddy Thatiparthy
+* Timo Kaufmann
+* Tony Fast
 
 Issues closed in 7.9
 --------------------

docs/source/whatsnew/version7.rst

@@ -2,8 +2,156 b''
  7.x Series
 ============
 
+.. _version 7101:
+
+IPython 7.10.1
+==============
+
+IPython 7.10.1 fix a couple of incompatibilities with Prompt toolkit 3 (please
+update Prompt toolkit to 3.0.2 at least), and fixes some interaction with
+headless IPython.
+
+.. _version 7100:
+
+IPython 7.10.0
+==============
+
+IPython 7.10 is the first double digit minor release in the  last decade, and
+first since the release of IPython 1.0, previous double digit minor release was
+in August 2009.
+
+We've been trying to give you regular release on the last Friday of every month
+for a guaranty of rapid access to bug fixes and new features.
+
+Unlike the previous first few releases that have seen only a couple of code
+changes, 7.10 bring a number of changes, new features and bugfixes.
+
+Stop Support for Python 3.5 – Adopt NEP 29
+------------------------------------------
+
+IPython has decided to follow the informational `NEP 29
+<https://numpy.org/neps/nep-0029-deprecation_policy.html>`_ which layout a clear
+policy as to which version of (C)Python and NumPy are supported.
+
+We thus dropped support for Python 3.5, and cleaned up a number of code path
+that were Python-version dependant. If you are on 3.5 or earlier pip should
+automatically give you the latest compatible version of IPython so you do not
+need to pin to a given version.
+
+Support for Prompt Toolkit 3.0
+------------------------------
+
+Prompt Toolkit 3.0 was release a week before IPython 7.10 and introduces a few
+breaking changes. We believe IPython 7.10 should be compatible with both Prompt
+Toolkit 2.x and 3.x, though it has not been extensively tested with 3.x so
+please report any issues.
+
+
+Prompt Rendering Performance improvements
+-----------------------------------------
+
+Pull Request :ghpull:`11933` introduced an optimisation in the prompt rendering
+logic that should decrease the resource usage of IPython when using the
+_default_ configuration but could potentially introduce a regression of
+functionalities if you are using a custom prompt.
+
+We know assume if you haven't changed the default keybindings that the prompt
+**will not change** during the duration of your input – which is for example
+not true when using vi insert mode that switches between `[ins]` and `[nor]`
+for the current mode.
+
+If you are experiencing any issue let us know.
+
+Code autoformatting
+-------------------
+
+The IPython terminal can now auto format your code just before entering a new
+line or executing a command. To do so use the
+``--TerminalInteractiveShell.autoformatter`` option and set it to ``'black'``;
+if black is installed IPython will use black to format your code when possible.
+
+IPython cannot always properly format your code; in particular it will
+auto formatting with *black* will only work if:
+
+   - Your code does not contains magics or special python syntax.
+
+   - There is no code after your cursor.
+
+The Black API is also still in motion; so this may not work with all versions of
+black.
+
+It should be possible to register custom formatter, though the API is till in
+flux.
+
+Arbitrary Mimetypes Handing in Terminal (Aka inline images in terminal)
+-----------------------------------------------------------------------
+
+When using IPython terminal it is now possible to register function to handle
+arbitrary mimetypes. While rendering non-text based representation was possible in
+many jupyter frontend; it was not possible in terminal IPython, as usually
+terminal are limited to displaying text. As many terminal these days provide
+escape sequences to display non-text; bringing this loved feature to IPython CLI
+made a lot of sens. This functionality will not only allow inline images; but
+allow opening of external program; for example ``mplayer`` to "display" sound
+files.
+
+So far only the hooks necessary for this are in place, but no default mime
+renderers added; so inline images will only be available via extensions. We will
+progressively enable these features by default in the next few releases, and
+contribution is welcomed.
+
+We welcome any feedback on the API. See :ref:`shell_mimerenderer` for more
+informations.
+
+This is originally based on work form in :ghpull:`10610` from @stephanh42
+started over two years ago, and still a lot need to be done.
+
+MISC
+----
+
+ - Completions can define their own ordering :ghpull:`11855`
+ - Enable Plotting in the same cell than the one that import matplotlib
+   :ghpull:`11916`
+ - Allow to store and restore multiple variables at once :ghpull:`11930`
+
+You can see `all pull-requests <https://github.com/ipython/ipython/pulls?q=is%3Apr+milestone%3A7.10+is%3Aclosed>`_ for this release.
+
+API Changes
+-----------
+
+Change of API and exposed objects automatically detected using `frappuccino <https://pypi.org/project/frappuccino/>`_ (still in beta):
+
+The following items are new in IPython 7.10::
+
+    + IPython.terminal.shortcuts.reformat_text_before_cursor(buffer, document, shell)
+    + IPython.terminal.interactiveshell.PTK3
+    + IPython.terminal.interactiveshell.black_reformat_handler(text_before_cursor)
+    + IPython.terminal.prompts.RichPromptDisplayHook.write_format_data(self, format_dict, md_dict='None')
+
+The following items have been removed in 7.10::
+
+    - IPython.lib.pretty.DICT_IS_ORDERED
+
+The following signatures differ between versions::
+
+    - IPython.extensions.storemagic.restore_aliases(ip)
+    + IPython.extensions.storemagic.restore_aliases(ip, alias='None')
+
+Special Thanks
+--------------
+
+ - @stephanh42 who started the work on inline images in terminal 2 years ago
+ - @augustogoulart who spent a lot of time triaging issues and responding to
+   users.
+ - @con-f-use who is my (@Carreau) first sponsor on GitHub, as a reminder if you
+   like IPython, Jupyter and many other library of the SciPy stack you can
+   donate to numfocus.org non profit
+
 .. _version 790:
 
+IPython 7.9.0
+=============
+
 IPython 7.9 is a small release with a couple of improvement and bug fixes.
 
  - Xterm terminal title should be restored on exit :ghpull:`11910`
@@ -13,12 +161,12 b' IPython 7.9 is a small release with a couple of improvement and bug fixes.'
    find all objects needing reload. This should avoid large objects traversal
    like pandas dataframes. :ghpull:`11876`
  - Get ready for Python 4. :ghpull:`11874`
- - `%env` Magic nonw has euristic to hide potentially sensitive values :ghpull:`11896`
+ - `%env` Magic now has heuristic to hide potentially sensitive values :ghpull:`11896`
 
 This is a small release despite a number of Pull Request Pending that need to
 be reviewed/worked on. Many of the core developers have been busy outside of
 IPython/Jupyter and we thanks all contributor for their patience; we'll work on
-these as soon as we have time. 
+these as soon as we have time.
 
 
 .. _version780:

setup.py

@@ -190,7 +190,7 b' install_requires = ['
     'decorator',
     'pickleshare',
     'traitlets>=4.2',
-    'prompt_toolkit>=2.0.0,<2.1.0',
+    'prompt_toolkit>=2.0.0,<3.1.0,!=3.0.0,!=3.0.1',
     'pygments',
     'backcall',
 ]

tools/release_helper.sh

@@ -1,5 +1,5 b''
 # Simple tool to help for release
-# when releasing with bash, simplei source it to get asked questions. 
+# when releasing with bash, simple source it to get asked questions. 
 
 # misc check before starting
 
@@ -9,14 +9,6 b" python -c 'import sphinx'"
 python -c 'import sphinx_rtd_theme'
 python -c 'import nose'
 
-echo -n 'PREV_RELEASE (X.y.z):'
-read PREV_RELEASE
-echo -n 'MILESTONE (X.y):'
-read MILESTONE
-echo -n 'VERSION (X.y.z):'
-read VERSION
-echo -n 'branch (master|X.y):'
-read branch
 
 BLACK=$(tput setaf 1)
 RED=$(tput setaf 1)
@@ -28,30 +20,66 b' CYAN=$(tput setaf 6)'
 WHITE=$(tput setaf 7)
 NOR=$(tput sgr0)
 
-echo 
-echo $BLUE"Updating what's new with informations from docs/source/whatsnew/pr"$NOR
-python tools/update_whatsnew.py
 
-echo
-echo $BLUE"please move the contents of "docs/source/whatsnew/development.rst" to version-X.rst"$NOR
-echo $GREEN"Press enter to continue"$NOR
-read
+echo -n "PREV_RELEASE (X.y.z) [$PREV_RELEASE]: "
+read input
+PREV_RELEASE=${input:-$PREV_RELEASE}
+echo -n "MILESTONE (X.y) [$MILESTONE]: "
+read input
+MILESTONE=${input:-$MILESTONE}
+echo -n "VERSION (X.y.z) [$VERSION]:"
+read input
+VERSION=${input:-$VERSION}
+echo -n "BRANCH (master|X.y) [$BRANCH]:"
+read input
+BRANCH=${input:-$BRANCH}
+
+ask_section(){
+    echo
+    echo $BLUE"$1"$NOR 
+    echo -n $GREEN"Press Enter to continue, S to skip: "$GREEN
+    read -n1 value
+    echo 
+    if [ -z $value ] || [ $value = 'y' ]  ; then
+        return 0
+    fi
+    return 1
+}
+
 
-echo 
-echo $BLUE"here are all the authors that contributed to this release:"$NOR
-git log --format="%aN <%aE>" $PREV_RELEASE... | sort -u -f
 
 echo 
-echo $BLUE"If you see any duplicates cancel (Ctrl-C), then edit .mailmap."$GREEN"Press enter to continue:"$NOR
-read
+if ask_section "Updating what's new with informations from docs/source/whatsnew/pr"
+then
+    python tools/update_whatsnew.py
 
-echo $BLUE"generating stats"$NOR
-python tools/github_stats.py --milestone $MILESTONE > stats.rst
+    echo
+    echo $BLUE"please move the contents of "docs/source/whatsnew/development.rst" to version-X.rst"$NOR
+    echo $GREEN"Press enter to continue"$NOR
+    read
+fi
 
-echo $BLUE"stats.rst files generated."$NOR
-echo $GREEN"Please merge it with the right file (github-stats-X.rst) and commit."$NOR
-echo $GREEN"press enter to continue."$NOR
-read
+if ask_section "Gen Stats, and authors"
+then
+
+    echo 
+    echo $BLUE"here are all the authors that contributed to this release:"$NOR
+    git log --format="%aN <%aE>" $PREV_RELEASE... | sort -u -f
+
+    echo 
+    echo $BLUE"If you see any duplicates cancel (Ctrl-C), then edit .mailmap."
+    echo $GREEN"Press enter to continue:"$NOR
+    read
+
+    echo $BLUE"generating stats"$NOR
+    python tools/github_stats.py --milestone $MILESTONE > stats.rst
+
+    echo $BLUE"stats.rst files generated."$NOR
+    echo $GREEN"Please merge it with the right file (github-stats-X.rst) and commit."$NOR
+    echo $GREEN"press enter to continue."$NOR
+    read
+
+fi
 
 echo "Cleaning repository"
 git clean -xfdi
@@ -61,30 +89,81 b' echo $GREEN"please update version number in ${RED}IPython/core/release.py${NOR} '
 echo $GREEN"Press enter to continue"$NOR
 read
 
-echo 
-echo "Attempting to build the docs.."
-make html -C docs
+if ask_section "Build the documentation ?"
+then
+    make html -C docs
+    echo 
+    echo $GREEN"Check the docs, press enter to continue"$NOR
+    read
 
-echo 
-echo $GREEN"Check the docs, press enter to continue"$NOR
-read
+fi
 
 echo
 echo $BLUE"Attempting to build package..."$NOR
 
 tools/build_release
-
-echo
-echo "Let\'s commit : git commit -am \"release $VERSION\" -S"
-echo $GREEN"Press enter to continue"$NOR
-read
-git commit -am "release $VERSION" -S
-
-echo
-echo "git push origin \$BRANCH ?"
-echo "Press enter to continue"
-read
-git push origin $BRANCH
-# git tag -am "release $VERSION" "$VERSION" -s
-# git push origin $VERSION
-
+rm dist/*
+
+if ask_section "Should we commit, tag, push... etc ? "
+then
+   echo
+   echo "Let's commit : git commit -am \"release $VERSION\" -S"
+   echo $GREEN"Press enter to commit"$NOR
+   read
+   git commit -am "release $VERSION" -S
+   
+   echo
+   echo $BLUE"git push origin \$BRANCH ($BRANCH)?"$NOR
+   echo $GREEN"Make sure you can push"$NOR
+   echo $GREEN"Press enter to continue"$NOR
+   read
+   git push origin $BRANCH
+   
+   echo
+   echo "Let's tag : git tag -am \"release $VERSION\" \"$VERSION\" -s"
+   echo $GREEN"Press enter to tag commit"$NOR
+   read
+   git tag -am "release $VERSION" "$VERSION" -s
+   
+   echo
+   echo $BLUE"And push the tag: git push origin \$VERSION ?"$NOR
+   echo $GREEN"Press enter to continue"$NOR
+   read
+   git push origin $VERSION
+   
+   
+   echo $GREEN"please update version number and back to .dev in ${RED}IPython/core/release.py"
+   echo ${BLUE}"Do not commit yet – we'll do it later."$NOR
+   
+   echo $GREEN"Press enter to continue"$NOR
+   read
+   
+   echo
+   echo "Let's commit : git commit -am \"back to dev\" -S"
+   echo $GREEN"Press enter to commit"$NOR
+   read
+   git commit -am "back to dev" -S
+   
+   echo
+   echo $BLUE"let's : git checkout $VERSION"$NOR
+   echo $GREEN"Press enter to continue"$NOR
+   read
+   git checkout $VERSION
+fi
+
+if ask_section "Should we build and release ?"
+then
+
+    echo
+    echo $BLUE"Attempting to build package..."$NOR
+
+    tools/build_release
+
+    echo '$ shasum -a 256 dist/*'
+    shasum -a 256 dist/*
+
+    if ask_section "upload packages ?"
+    then 
+       tools/build_release upload
+    fi
+fi