Merge pull request #1 from ipython/master

Merged new changes

IPython/terminal/pt_inputhooks/asyncio.py

@@ -0,0 +1,64 b''
+"""
+Inputhook for running the original asyncio event loop while we're waiting for
+input.
+
+By default, in IPython, we run the prompt with a different asyncio event loop,
+because otherwise we risk that people are freezing the prompt by scheduling bad
+coroutines. E.g., a coroutine that does a while/true and never yield back
+control to the loop. We can't cancel that.
+
+However, sometimes we want the asyncio loop to keep running while waiting for
+a prompt.
+
+The following example will print the numbers from 1 to 10 above the prompt,
+while we are waiting for input. (This works also because we use
+prompt_toolkit`s `patch_stdout`)::
+
+    In [1]: import asyncio
+
+    In [2]: %gui asyncio
+
+    In [3]: async def f():
+       ...:     for i in range(10):
+       ...:         await asyncio.sleep(1)
+       ...:         print(i)
+
+
+    In [4]: asyncio.ensure_future(f())
+
+"""
+import asyncio
+from prompt_toolkit import __version__ as ptk_version
+
+PTK3 = ptk_version.startswith('3.')
+
+
+# Keep reference to the original asyncio loop, because getting the event loop
+# within the input hook would return the other loop.
+loop = asyncio.get_event_loop()
+
+
+def inputhook(context):
+    """
+    Inputhook for asyncio event loop integration.
+    """
+    # For prompt_toolkit 3.0, this input hook literally doesn't do anything.
+    # The event loop integration here is implemented in `interactiveshell.py`
+    # by running the prompt itself in the current asyncio loop. The main reason
+    # for this is that nesting asyncio event loops is unreliable.
+    if PTK3:
+        return
+
+    # For prompt_toolkit 2.0, we can run the current asyncio event loop,
+    # because prompt_toolkit 2.0 uses a different event loop internally.
+
+    def stop():
+        loop.stop()
+
+    fileno = context.fileno()
+    loop.add_reader(fileno, stop)
+    try:
+        loop.run_forever()
+    finally:
+        loop.remove_reader(fileno)
+

IPython/core/crashhandler.py

@@ -1,226 +1,228 b''
 # encoding: utf-8
 """sys.excepthook for IPython itself, leaves a detailed report on disk.
 
 Authors:
 
 * Fernando Perez
 * Brian E. Granger
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2001-2007 Fernando Perez. <[e-mail]>
 #  Copyright (C) 2008-2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 import os
 import sys
 import traceback
 from pprint import pformat
 
 from IPython.core import ultratb
 from IPython.core.release import author_email
 from IPython.utils.sysinfo import sys_info
 from IPython.utils.py3compat import input
 
+from IPython.core.release import __version__ as version
+
 #-----------------------------------------------------------------------------
 # Code
 #-----------------------------------------------------------------------------
 
 # Template for the user message.
 _default_message_template = """\
 Oops, {app_name} crashed. We do our best to make it stable, but...
 
 A crash report was automatically generated with the following information:
   - A verbatim copy of the crash traceback.
   - A copy of your input history during this session.
   - Data on your current {app_name} configuration.
 
 It was left in the file named:
 \t'{crash_report_fname}'
 If you can email this file to the developers, the information in it will help
 them in understanding and correcting the problem.
 
 You can mail it to: {contact_name} at {contact_email}
 with the subject '{app_name} Crash Report'.
 
 If you want to do it now, the following command will work (under Unix):
 mail -s '{app_name} Crash Report' {contact_email} < {crash_report_fname}
 
 In your email, please also include information about:
 - The operating system under which the crash happened: Linux, macOS, Windows,
   other, and which exact version (for example: Ubuntu 16.04.3, macOS 10.13.2,
   Windows 10 Pro), and whether it is 32-bit or 64-bit;
 - How {app_name} was installed: using pip or conda, from GitHub, as part of
   a Docker container, or other, providing more detail if possible;
 - How to reproduce the crash: what exact sequence of instructions can one
   input to get the same crash? Ideally, find a minimal yet complete sequence
   of instructions that yields the crash.
 
 To ensure accurate tracking of this issue, please file a report about it at:
 {bug_tracker}
 """
 
 _lite_message_template = """
-If you suspect this is an IPython bug, please report it at:
+If you suspect this is an IPython {version} bug, please report it at:
     https://github.com/ipython/ipython/issues
 or send an email to the mailing list at {email}
 
 You can print a more detailed traceback right now with "%tb", or use "%debug"
 to interactively debug it.
 
 Extra-detailed tracebacks for bug-reporting purposes can be enabled via:
     {config}Application.verbose_crash=True
 """
 
 
 class CrashHandler(object):
     """Customizable crash handlers for IPython applications.
 
     Instances of this class provide a :meth:`__call__` method which can be
     used as a ``sys.excepthook``.  The :meth:`__call__` signature is::
 
         def __call__(self, etype, evalue, etb)
     """
 
     message_template = _default_message_template
     section_sep = '\n\n'+'*'*75+'\n\n'
 
     def __init__(self, app, contact_name=None, contact_email=None,
                  bug_tracker=None, show_crash_traceback=True, call_pdb=False):
         """Create a new crash handler
 
         Parameters
         ----------
         app :  Application
             A running :class:`Application` instance, which will be queried at
             crash time for internal information.
 
         contact_name : str
             A string with the name of the person to contact.
 
         contact_email : str
             A string with the email address of the contact.
 
         bug_tracker : str
             A string with the URL for your project's bug tracker.
 
         show_crash_traceback : bool
             If false, don't print the crash traceback on stderr, only generate
             the on-disk report
 
         Non-argument instance attributes:
 
         These instances contain some non-argument attributes which allow for
         further customization of the crash handler's behavior. Please see the
         source for further details.
         """
         self.crash_report_fname = "Crash_report_%s.txt" % app.name
         self.app = app
         self.call_pdb = call_pdb
         #self.call_pdb = True # dbg
         self.show_crash_traceback = show_crash_traceback
         self.info = dict(app_name = app.name,
                     contact_name = contact_name,
                     contact_email = contact_email,
                     bug_tracker = bug_tracker,
                     crash_report_fname = self.crash_report_fname)
 
 
     def __call__(self, etype, evalue, etb):
         """Handle an exception, call for compatible with sys.excepthook"""
         
         # do not allow the crash handler to be called twice without reinstalling it
         # this prevents unlikely errors in the crash handling from entering an
         # infinite loop.
         sys.excepthook = sys.__excepthook__
         
         # Report tracebacks shouldn't use color in general (safer for users)
         color_scheme = 'NoColor'
 
         # Use this ONLY for developer debugging (keep commented out for release)
         #color_scheme = 'Linux'   # dbg
         try:
             rptdir = self.app.ipython_dir
         except:
             rptdir = os.getcwd()
         if rptdir is None or not os.path.isdir(rptdir):
             rptdir = os.getcwd()
         report_name = os.path.join(rptdir,self.crash_report_fname)
         # write the report filename into the instance dict so it can get
         # properly expanded out in the user message template
         self.crash_report_fname = report_name
         self.info['crash_report_fname'] = report_name
         TBhandler = ultratb.VerboseTB(
             color_scheme=color_scheme,
             long_header=1,
             call_pdb=self.call_pdb,
         )
         if self.call_pdb:
             TBhandler(etype,evalue,etb)
             return
         else:
             traceback = TBhandler.text(etype,evalue,etb,context=31)
 
         # print traceback to screen
         if self.show_crash_traceback:
             print(traceback, file=sys.stderr)
 
         # and generate a complete report on disk
         try:
             report = open(report_name,'w')
         except:
             print('Could not create crash report on disk.', file=sys.stderr)
             return
 
         with report:
             # Inform user on stderr of what happened
             print('\n'+'*'*70+'\n', file=sys.stderr)
             print(self.message_template.format(**self.info), file=sys.stderr)
 
             # Construct report on disk
             report.write(self.make_report(traceback))
 
         input("Hit <Enter> to quit (your terminal may close):")
 
     def make_report(self,traceback):
         """Return a string containing a crash report."""
 
         sec_sep = self.section_sep
 
         report = ['*'*75+'\n\n'+'IPython post-mortem report\n\n']
         rpt_add = report.append
         rpt_add(sys_info())
 
         try:
             config = pformat(self.app.config)
             rpt_add(sec_sep)
             rpt_add('Application name: %s\n\n' % self.app_name)
             rpt_add('Current user configuration structure:\n\n')
             rpt_add(config)
         except:
             pass
         rpt_add(sec_sep+'Crash traceback:\n\n' + traceback)
 
         return ''.join(report)
 
 
 def crash_handler_lite(etype, evalue, tb):
     """a light excepthook, adding a small message to the usual traceback"""
     traceback.print_exception(etype, evalue, tb)
     
     from IPython.core.interactiveshell import InteractiveShell
     if InteractiveShell.initialized():
         # we are in a Shell environment, give %magic example
         config = "%config "
     else:
         # we are not in a shell, show generic config
         config = "c."
-    print(_lite_message_template.format(email=author_email, config=config), file=sys.stderr)
+    print(_lite_message_template.format(email=author_email, config=config, version=version), file=sys.stderr)
 

IPython/core/displayhook.py

@@ -1,325 +1,325 b''
 # -*- coding: utf-8 -*-
 """Displayhook for IPython.
 
 This defines a callable class that IPython uses for `sys.displayhook`.
 """
 
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
 import builtins as builtin_mod
 import sys
 import io as _io
 import tokenize
 
 from traitlets.config.configurable import Configurable
 from traitlets import Instance, Float
 from warnings import warn
 
 # TODO: Move the various attributes (cache_size, [others now moved]). Some
 # of these are also attributes of InteractiveShell. They should be on ONE object
 # only and the other objects should ask that one object for their values.
 
 class DisplayHook(Configurable):
     """The custom IPython displayhook to replace sys.displayhook.
 
     This class does many things, but the basic idea is that it is a callable
     that gets called anytime user code returns a value.
     """
 
     shell = Instance('IPython.core.interactiveshell.InteractiveShellABC',
                      allow_none=True)
     exec_result = Instance('IPython.core.interactiveshell.ExecutionResult',
                            allow_none=True)
     cull_fraction = Float(0.2)
 
     def __init__(self, shell=None, cache_size=1000, **kwargs):
         super(DisplayHook, self).__init__(shell=shell, **kwargs)
         cache_size_min = 3
         if cache_size <= 0:
             self.do_full_cache = 0
             cache_size = 0
         elif cache_size < cache_size_min:
             self.do_full_cache = 0
             cache_size = 0
             warn('caching was disabled (min value for cache size is %s).' %
                  cache_size_min,stacklevel=3)
         else:
             self.do_full_cache = 1
 
         self.cache_size = cache_size
 
         # we need a reference to the user-level namespace
         self.shell = shell
         
         self._,self.__,self.___ = '','',''
 
         # these are deliberately global:
         to_user_ns = {'_':self._,'__':self.__,'___':self.___}
         self.shell.user_ns.update(to_user_ns)
 
     @property
     def prompt_count(self):
         return self.shell.execution_count
 
     #-------------------------------------------------------------------------
     # Methods used in __call__. Override these methods to modify the behavior
     # of the displayhook.
     #-------------------------------------------------------------------------
 
     def check_for_underscore(self):
         """Check if the user has set the '_' variable by hand."""
         # If something injected a '_' variable in __builtin__, delete
         # ipython's automatic one so we don't clobber that.  gettext() in
         # particular uses _, so we need to stay away from it.
         if '_' in builtin_mod.__dict__:
             try:
                 user_value = self.shell.user_ns['_']
                 if user_value is not self._:
                     return
                 del self.shell.user_ns['_']
             except KeyError:
                 pass
 
     def quiet(self):
         """Should we silence the display hook because of ';'?"""
         # do not print output if input ends in ';'
         
         try:
             cell = self.shell.history_manager.input_hist_parsed[-1]
         except IndexError:
             # some uses of ipshellembed may fail here
             return False
         
         sio = _io.StringIO(cell)
         tokens = list(tokenize.generate_tokens(sio.readline))
 
         for token in reversed(tokens):
             if token[0] in (tokenize.ENDMARKER, tokenize.NL, tokenize.NEWLINE, tokenize.COMMENT):
                 continue
             if (token[0] == tokenize.OP) and (token[1] == ';'):
                 return True
             else:
                 return False
 
     def start_displayhook(self):
         """Start the displayhook, initializing resources."""
         pass
 
     def write_output_prompt(self):
         """Write the output prompt.
 
         The default implementation simply writes the prompt to
         ``sys.stdout``.
         """
         # Use write, not print which adds an extra space.
         sys.stdout.write(self.shell.separate_out)
         outprompt = 'Out[{}]: '.format(self.shell.execution_count)
         if self.do_full_cache:
             sys.stdout.write(outprompt)
 
     def compute_format_data(self, result):
         """Compute format data of the object to be displayed.
 
         The format data is a generalization of the :func:`repr` of an object.
         In the default implementation the format data is a :class:`dict` of
         key value pair where the keys are valid MIME types and the values
         are JSON'able data structure containing the raw data for that MIME
         type. It is up to frontends to determine pick a MIME to to use and
         display that data in an appropriate manner.
 
         This method only computes the format data for the object and should
         NOT actually print or write that to a stream.
 
         Parameters
         ----------
         result : object
             The Python object passed to the display hook, whose format will be
             computed.
 
         Returns
         -------
         (format_dict, md_dict) : dict
             format_dict is a :class:`dict` whose keys are valid MIME types and values are
             JSON'able raw data for that MIME type. It is recommended that
             all return values of this should always include the "text/plain"
             MIME type representation of the object.
             md_dict is a :class:`dict` with the same MIME type keys
             of metadata associated with each output.
             
         """
         return self.shell.display_formatter.format(result)
 
     # This can be set to True by the write_output_prompt method in a subclass
     prompt_end_newline = False
 
-    def write_format_data(self, format_dict, md_dict=None):
+    def write_format_data(self, format_dict, md_dict=None) -> None:
         """Write the format data dict to the frontend.
 
         This default version of this method simply writes the plain text
         representation of the object to ``sys.stdout``. Subclasses should
         override this method to send the entire `format_dict` to the
         frontends.
 
         Parameters
         ----------
         format_dict : dict
             The format dict for the object passed to `sys.displayhook`.
         md_dict : dict (optional)
             The metadata dict to be associated with the display data.
         """
         if 'text/plain' not in format_dict:
             # nothing to do
             return
         # We want to print because we want to always make sure we have a
         # newline, even if all the prompt separators are ''. This is the
         # standard IPython behavior.
         result_repr = format_dict['text/plain']
         if '\n' in result_repr:
             # So that multi-line strings line up with the left column of
             # the screen, instead of having the output prompt mess up
             # their first line.
             # We use the prompt template instead of the expanded prompt
             # because the expansion may add ANSI escapes that will interfere
             # with our ability to determine whether or not we should add
             # a newline.
             if not self.prompt_end_newline:
                 # But avoid extraneous empty lines.
                 result_repr = '\n' + result_repr
 
         try:
             print(result_repr)
         except UnicodeEncodeError:
             # If a character is not supported by the terminal encoding replace
             # it with its \u or \x representation
             print(result_repr.encode(sys.stdout.encoding,'backslashreplace').decode(sys.stdout.encoding))
 
     def update_user_ns(self, result):
         """Update user_ns with various things like _, __, _1, etc."""
 
         # Avoid recursive reference when displaying _oh/Out
         if self.cache_size and result is not self.shell.user_ns['_oh']:
             if len(self.shell.user_ns['_oh']) >= self.cache_size and self.do_full_cache:
                 self.cull_cache()
 
             # Don't overwrite '_' and friends if '_' is in __builtin__
             # (otherwise we cause buggy behavior for things like gettext). and
             # do not overwrite _, __ or ___ if one of these has been assigned
             # by the user.
             update_unders = True
             for unders in ['_'*i for i in range(1,4)]:
                 if not unders in self.shell.user_ns:
                     continue
                 if getattr(self, unders) is not self.shell.user_ns.get(unders):
                     update_unders = False
 
             self.___ = self.__
             self.__ = self._
             self._ = result
 
             if ('_' not in builtin_mod.__dict__) and (update_unders):
                 self.shell.push({'_':self._,
                                  '__':self.__,
                                 '___':self.___}, interactive=False)
 
             # hackish access to top-level  namespace to create _1,_2... dynamically
             to_main = {}
             if self.do_full_cache:
                 new_result = '_%s' % self.prompt_count
                 to_main[new_result] = result
                 self.shell.push(to_main, interactive=False)
                 self.shell.user_ns['_oh'][self.prompt_count] = result
 
     def fill_exec_result(self, result):
         if self.exec_result is not None:
             self.exec_result.result = result
 
     def log_output(self, format_dict):
         """Log the output."""
         if 'text/plain' not in format_dict:
             # nothing to do
             return
         if self.shell.logger.log_output:
             self.shell.logger.log_write(format_dict['text/plain'], 'output')
         self.shell.history_manager.output_hist_reprs[self.prompt_count] = \
                                                     format_dict['text/plain']
 
     def finish_displayhook(self):
         """Finish up all displayhook activities."""
         sys.stdout.write(self.shell.separate_out2)
         sys.stdout.flush()
 
     def __call__(self, result=None):
         """Printing with history cache management.
 
         This is invoked every time the interpreter needs to print, and is
         activated by setting the variable sys.displayhook to it.
         """
         self.check_for_underscore()
         if result is not None and not self.quiet():
             self.start_displayhook()
             self.write_output_prompt()
             format_dict, md_dict = self.compute_format_data(result)
             self.update_user_ns(result)
             self.fill_exec_result(result)
             if format_dict:
                 self.write_format_data(format_dict, md_dict)
                 self.log_output(format_dict)
             self.finish_displayhook()
 
     def cull_cache(self):
         """Output cache is full, cull the oldest entries"""
         oh = self.shell.user_ns.get('_oh', {})
         sz = len(oh)
         cull_count = max(int(sz * self.cull_fraction), 2)
         warn('Output cache limit (currently {sz} entries) hit.\n'
              'Flushing oldest {cull_count} entries.'.format(sz=sz, cull_count=cull_count))
         
         for i, n in enumerate(sorted(oh)):
             if i >= cull_count:
                 break
             self.shell.user_ns.pop('_%i' % n, None)
             oh.pop(n, None)
         
 
     def flush(self):
         if not self.do_full_cache:
             raise ValueError("You shouldn't have reached the cache flush "
                              "if full caching is not enabled!")
         # delete auto-generated vars from global namespace
 
         for n in range(1,self.prompt_count + 1):
             key = '_'+repr(n)
             try:
                 del self.shell.user_ns[key]
             except: pass
         # In some embedded circumstances, the user_ns doesn't have the
         # '_oh' key set up.
         oh = self.shell.user_ns.get('_oh', None)
         if oh is not None:
             oh.clear()
 
         # Release our own references to objects:
         self._, self.__, self.___ = '', '', ''
 
         if '_' not in builtin_mod.__dict__:
             self.shell.user_ns.update({'_':self._,'__':self.__,'___':self.___})
         import gc
         # TODO: Is this really needed?
         # IronPython blocks here forever
         if sys.platform != "cli":
             gc.collect()
 
 
 class CapturingDisplayHook(object):
     def __init__(self, shell, outputs=None):
         self.shell = shell
         if outputs is None:
             outputs = []
         self.outputs = outputs
 
     def __call__(self, result=None):
         if result is None:
             return
         format_dict, md_dict = self.shell.display_formatter.format(result)
         self.outputs.append({ 'data': format_dict, 'metadata': md_dict })

IPython/core/displaypub.py

@@ -1,125 +1,138 b''
 """An interface for publishing rich data to frontends.
 
 There are two components of the display system:
 
 * Display formatters, which take a Python object and compute the
   representation of the object in various formats (text, HTML, SVG, etc.).
 * The display publisher that is used to send the representation data to the
   various frontends.
 
 This module defines the logic display publishing. The display publisher uses
 the ``display_data`` message type that is defined in the IPython messaging
 spec.
 """
 
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
 
 import sys
 
 from traitlets.config.configurable import Configurable
-from traitlets import List
+from traitlets import List, Dict
 
 # This used to be defined here - it is imported for backwards compatibility
 from .display import publish_display_data
 
 #-----------------------------------------------------------------------------
 # Main payload class
 #-----------------------------------------------------------------------------
 
+
 class DisplayPublisher(Configurable):
     """A traited class that publishes display data to frontends.
 
     Instances of this class are created by the main IPython object and should
     be accessed there.
     """
 
+    def __init__(self, shell=None, *args, **kwargs):
+        self.shell = shell
+        super().__init__(*args, **kwargs)
+
     def _validate_data(self, data, metadata=None):
         """Validate the display data.
 
         Parameters
         ----------
         data : dict
             The formata data dictionary.
         metadata : dict
             Any metadata for the data.
         """
 
         if not isinstance(data, dict):
             raise TypeError('data must be a dict, got: %r' % data)
         if metadata is not None:
             if not isinstance(metadata, dict):
                 raise TypeError('metadata must be a dict, got: %r' % data)
 
     # use * to indicate transient, update are keyword-only
-    def publish(self, data, metadata=None, source=None, *, transient=None, update=False, **kwargs):
+    def publish(self, data, metadata=None, source=None, *, transient=None, update=False, **kwargs) -> None:
         """Publish data and metadata to all frontends.
 
         See the ``display_data`` message in the messaging documentation for
         more details about this message type.
 
         The following MIME types are currently implemented:
 
         * text/plain
         * text/html
         * text/markdown
         * text/latex
         * application/json
         * application/javascript
         * image/png
         * image/jpeg
         * image/svg+xml
 
         Parameters
         ----------
         data : dict
             A dictionary having keys that are valid MIME types (like
             'text/plain' or 'image/svg+xml') and values that are the data for
             that MIME type. The data itself must be a JSON'able data
             structure. Minimally all data should have the 'text/plain' data,
             which can be displayed by all frontends. If more than the plain
             text is given, it is up to the frontend to decide which
             representation to use.
         metadata : dict
             A dictionary for metadata related to the data. This can contain
             arbitrary key, value pairs that frontends can use to interpret
             the data.  Metadata specific to each mime-type can be specified
             in the metadata dict with the same mime-type keys as
             the data itself.
         source : str, deprecated
             Unused.
         transient: dict, keyword-only
             A dictionary for transient data.
             Data in this dictionary should not be persisted as part of saving this output.
             Examples include 'display_id'.
         update: bool, keyword-only, default: False
             If True, only update existing outputs with the same display_id,
             rather than creating a new output.
         """
 
-        # The default is to simply write the plain text data using sys.stdout.
+        handlers = {}
+        if self.shell is not None:
+            handlers = getattr(self.shell, 'mime_renderers', {})
+
+        for mime, handler in handlers.items():
+            if mime in data:
+                handler(data[mime], metadata.get(mime, None))
+                return
+
         if 'text/plain' in data:
             print(data['text/plain'])
 
     def clear_output(self, wait=False):
         """Clear the output of the cell receiving output."""
         print('\033[2K\r', end='')
         sys.stdout.flush()
         print('\033[2K\r', end='')
         sys.stderr.flush()
 
 
 class CapturingDisplayPublisher(DisplayPublisher):
     """A DisplayPublisher that stores"""
     outputs = List()
 
     def publish(self, data, metadata=None, source=None, *, transient=None, update=False):
         self.outputs.append({'data':data, 'metadata':metadata,
                              'transient':transient, 'update':update})
 
     def clear_output(self, wait=False):
         super(CapturingDisplayPublisher, self).clear_output(wait)
 
         # empty the list, *do not* reassign a new list
         self.outputs.clear()

IPython/core/interactiveshell.py

@@ -1,3701 +1,3701 b''
 # -*- coding: utf-8 -*-
 """Main IPython class."""
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2001 Janko Hauser <[e-mail]>
 #  Copyright (C) 2001-2007 Fernando Perez. <[e-mail]>
 #  Copyright (C) 2008-2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 
 import abc
 import ast
 import asyncio
 import atexit
 import builtins as builtin_mod
 import functools
 import inspect
 import os
 import re
 import runpy
 import sys
 import tempfile
 import traceback
 import types
 import subprocess
 import warnings
 from io import open as io_open
 
 from pickleshare import PickleShareDB
 
 from traitlets.config.configurable import SingletonConfigurable
 from traitlets.utils.importstring import import_item
 from IPython.core import oinspect
 from IPython.core import magic
 from IPython.core import page
 from IPython.core import prefilter
 from IPython.core import ultratb
 from IPython.core.alias import Alias, AliasManager
 from IPython.core.autocall import ExitAutocall
 from IPython.core.builtin_trap import BuiltinTrap
 from IPython.core.events import EventManager, available_events
 from IPython.core.compilerop import CachingCompiler, check_linecache_ipython
 from IPython.core.debugger import Pdb
 from IPython.core.display_trap import DisplayTrap
 from IPython.core.displayhook import DisplayHook
 from IPython.core.displaypub import DisplayPublisher
 from IPython.core.error import InputRejected, UsageError
 from IPython.core.extensions import ExtensionManager
 from IPython.core.formatters import DisplayFormatter
 from IPython.core.history import HistoryManager
 from IPython.core.inputtransformer2 import ESC_MAGIC, ESC_MAGIC2
 from IPython.core.logger import Logger
 from IPython.core.macro import Macro
 from IPython.core.payload import PayloadManager
 from IPython.core.prefilter import PrefilterManager
 from IPython.core.profiledir import ProfileDir
 from IPython.core.usage import default_banner
 from IPython.display import display
 from IPython.testing.skipdoctest import skip_doctest
 from IPython.utils import PyColorize
 from IPython.utils import io
 from IPython.utils import py3compat
 from IPython.utils import openpy
 from IPython.utils.decorators import undoc
 from IPython.utils.io import ask_yes_no
 from IPython.utils.ipstruct import Struct
 from IPython.paths import get_ipython_dir
 from IPython.utils.path import get_home_dir, get_py_filename, ensure_dir_exists
 from IPython.utils.process import system, getoutput
 from IPython.utils.strdispatch import StrDispatch
 from IPython.utils.syspathcontext import prepended_to_syspath
 from IPython.utils.text import format_screen, LSString, SList, DollarFormatter
 from IPython.utils.tempdir import TemporaryDirectory
 from traitlets import (
     Integer, Bool, CaselessStrEnum, Enum, List, Dict, Unicode, Instance, Type,
     observe, default, validate, Any
 )
 from warnings import warn
 from logging import error
 import IPython.core.hooks
 
 from typing import List as ListType, Tuple
 from ast import AST
 
 # NoOpContext is deprecated, but ipykernel imports it from here.
 # See https://github.com/ipython/ipykernel/issues/157
 from IPython.utils.contexts import NoOpContext
 
 try:
     import docrepr.sphinxify as sphx
 
     def sphinxify(doc):
         with TemporaryDirectory() as dirname:
             return {
                 'text/html': sphx.sphinxify(doc, dirname),
                 'text/plain': doc
             }
 except ImportError:
     sphinxify = None
 
 
 class ProvisionalWarning(DeprecationWarning):
     """
     Warning class for unstable features
     """
     pass
 
 if sys.version_info > (3,8):
     from ast import Module
 else :
     # mock the new API, ignore second argument
     # see https://github.com/ipython/ipython/issues/11590
     from ast import Module as OriginalModule
     Module = lambda nodelist, type_ignores: OriginalModule(nodelist)
 
 if sys.version_info > (3,6):
     _assign_nodes         = (ast.AugAssign, ast.AnnAssign, ast.Assign)
     _single_targets_nodes = (ast.AugAssign, ast.AnnAssign)
 else:
     _assign_nodes         = (ast.AugAssign, ast.Assign )
     _single_targets_nodes = (ast.AugAssign, )
 
 #-----------------------------------------------------------------------------
 # Await Helpers
 #-----------------------------------------------------------------------------
 
 def removed_co_newlocals(function:types.FunctionType) -> types.FunctionType:
     """Return a function that do not create a new local scope. 
 
     Given a function, create a clone of this function where the co_newlocal flag
     has been removed, making this function code actually run in the sourounding
     scope. 
 
     We need this in order to run asynchronous code in user level namespace.
     """
     from types import CodeType, FunctionType
     CO_NEWLOCALS = 0x0002
     code = function.__code__
     new_co_flags = code.co_flags & ~CO_NEWLOCALS
     if sys.version_info > (3, 8, 0, 'alpha', 3):
         new_code = code.replace(co_flags=new_co_flags)
     else:
         new_code = CodeType(
             code.co_argcount,
             code.co_kwonlyargcount,
             code.co_nlocals,
             code.co_stacksize,
             new_co_flags,
             code.co_code,
             code.co_consts,
             code.co_names,
             code.co_varnames,
             code.co_filename,
             code.co_name,
             code.co_firstlineno,
             code.co_lnotab,
             code.co_freevars,
             code.co_cellvars
         )
     return FunctionType(new_code, globals(), function.__name__, function.__defaults__)
 
 
 # we still need to run things using the asyncio eventloop, but there is no
 # async integration
 from .async_helpers import (_asyncio_runner,  _asyncify, _pseudo_sync_runner)
 from .async_helpers import _curio_runner, _trio_runner, _should_be_async
 
 
 def _ast_asyncify(cell:str, wrapper_name:str) -> ast.Module:
     """
     Parse a cell with top-level await and modify the AST to be able to run it later.
 
     Parameter
     ---------
 
     cell: str
         The code cell to asyncronify
     wrapper_name: str
         The name of the function to be used to wrap the passed `cell`. It is
         advised to **not** use a python identifier in order to not pollute the
         global namespace in which the function will be ran.
 
     Return
     ------
 
     A module object AST containing **one** function named `wrapper_name`.
 
     The given code is wrapped in a async-def function, parsed into an AST, and
     the resulting function definition AST is modified to return the last
     expression.
 
     The last expression or await node is moved into a return statement at the
     end of the function, and removed from its original location. If the last
     node is not Expr or Await nothing is done.
 
     The function `__code__` will need to be later modified  (by
     ``removed_co_newlocals``) in a subsequent step to not create new `locals()`
     meaning that the local and global scope are the same, ie as if the body of
     the function was at module level.
 
     Lastly a call to `locals()` is made just before the last expression of the
     function, or just after the last assignment or statement to make sure the
     global dict is updated as python function work with a local fast cache which
     is updated only on `local()` calls.
     """
 
     from ast import Expr, Await, Return
     if sys.version_info >= (3,8):
         return ast.parse(cell)
     tree = ast.parse(_asyncify(cell))
 
     function_def = tree.body[0]
     function_def.name = wrapper_name
     try_block = function_def.body[0]
     lastexpr = try_block.body[-1]
     if isinstance(lastexpr, (Expr, Await)):
         try_block.body[-1] = Return(lastexpr.value)
     ast.fix_missing_locations(tree)
     return tree
 #-----------------------------------------------------------------------------
 # Globals
 #-----------------------------------------------------------------------------
 
 # compiled regexps for autoindent management
 dedent_re = re.compile(r'^\s+raise|^\s+return|^\s+pass')
 
 #-----------------------------------------------------------------------------
 # Utilities
 #-----------------------------------------------------------------------------
 
 @undoc
 def softspace(file, newvalue):
     """Copied from code.py, to remove the dependency"""
 
     oldvalue = 0
     try:
         oldvalue = file.softspace
     except AttributeError:
         pass
     try:
         file.softspace = newvalue
     except (AttributeError, TypeError):
         # "attribute-less object" or "read-only attributes"
         pass
     return oldvalue
 
 @undoc
 def no_op(*a, **kw):
     pass
 
 
 class SpaceInInput(Exception): pass
 
 
 def get_default_colors():
     "DEPRECATED"
     warn('get_default_color is deprecated since IPython 5.0, and returns `Neutral` on all platforms.',
         DeprecationWarning, stacklevel=2)
     return 'Neutral'
 
 
 class SeparateUnicode(Unicode):
     r"""A Unicode subclass to validate separate_in, separate_out, etc.
 
     This is a Unicode based trait that converts '0'->'' and ``'\\n'->'\n'``.
     """
 
     def validate(self, obj, value):
         if value == '0': value = ''
         value = value.replace('\\n','\n')
         return super(SeparateUnicode, self).validate(obj, value)
 
 
 @undoc
 class DummyMod(object):
     """A dummy module used for IPython's interactive module when
     a namespace must be assigned to the module's __dict__."""
     __spec__ = None
 
 
 class ExecutionInfo(object):
     """The arguments used for a call to :meth:`InteractiveShell.run_cell`
 
     Stores information about what is going to happen.
     """
     raw_cell = None
     store_history = False
     silent = False
     shell_futures = True
 
     def __init__(self, raw_cell, store_history, silent, shell_futures):
         self.raw_cell = raw_cell
         self.store_history = store_history
         self.silent = silent
         self.shell_futures = shell_futures
 
     def __repr__(self):
         name = self.__class__.__qualname__
         raw_cell = ((self.raw_cell[:50] + '..')
                     if len(self.raw_cell) > 50 else self.raw_cell)
         return '<%s object at %x, raw_cell="%s" store_history=%s silent=%s shell_futures=%s>' %\
                (name, id(self), raw_cell, self.store_history, self.silent, self.shell_futures)
 
 
 class ExecutionResult(object):
     """The result of a call to :meth:`InteractiveShell.run_cell`
 
     Stores information about what took place.
     """
     execution_count = None
     error_before_exec = None
     error_in_exec = None
     info = None
     result = None
 
     def __init__(self, info):
         self.info = info
 
     @property
     def success(self):
         return (self.error_before_exec is None) and (self.error_in_exec is None)
 
     def raise_error(self):
         """Reraises error if `success` is `False`, otherwise does nothing"""
         if self.error_before_exec is not None:
             raise self.error_before_exec
         if self.error_in_exec is not None:
             raise self.error_in_exec
 
     def __repr__(self):
         name = self.__class__.__qualname__
         return '<%s object at %x, execution_count=%s error_before_exec=%s error_in_exec=%s info=%s result=%s>' %\
                 (name, id(self), self.execution_count, self.error_before_exec, self.error_in_exec, repr(self.info), repr(self.result))
 
 
 class InteractiveShell(SingletonConfigurable):
     """An enhanced, interactive shell for Python."""
 
     _instance = None
     
     ast_transformers = List([], help=
         """
         A list of ast.NodeTransformer subclass instances, which will be applied
         to user input before code is run.
         """
     ).tag(config=True)
 
     autocall = Enum((0,1,2), default_value=0, help=
         """
         Make IPython automatically call any callable object even if you didn't
         type explicit parentheses. For example, 'str 43' becomes 'str(43)'
         automatically. The value can be '0' to disable the feature, '1' for
         'smart' autocall, where it is not applied if there are no more
         arguments on the line, and '2' for 'full' autocall, where all callable
         objects are automatically called (even if no arguments are present).
         """
     ).tag(config=True)
 
     autoindent = Bool(True, help=
         """
         Autoindent IPython code entered interactively.
         """
     ).tag(config=True)
 
     autoawait = Bool(True, help=
         """
         Automatically run await statement in the top level repl.
         """
     ).tag(config=True)
 
     loop_runner_map ={
         'asyncio':(_asyncio_runner, True),
         'curio':(_curio_runner, True),
         'trio':(_trio_runner, True),
         'sync': (_pseudo_sync_runner, False)
     }
 
     loop_runner = Any(default_value="IPython.core.interactiveshell._asyncio_runner",
         allow_none=True,
         help="""Select the loop runner that will be used to execute top-level asynchronous code"""
     ).tag(config=True)
 
     @default('loop_runner')
     def _default_loop_runner(self):
         return import_item("IPython.core.interactiveshell._asyncio_runner")
 
     @validate('loop_runner')
     def _import_runner(self, proposal):
         if isinstance(proposal.value, str):
             if proposal.value in self.loop_runner_map:
                 runner, autoawait = self.loop_runner_map[proposal.value]
                 self.autoawait = autoawait
                 return runner
             runner = import_item(proposal.value)
             if not callable(runner):
                 raise ValueError('loop_runner must be callable')
             return runner
         if not callable(proposal.value):
             raise ValueError('loop_runner must be callable')
         return proposal.value
 
     automagic = Bool(True, help=
         """
         Enable magic commands to be called without the leading %.
         """
     ).tag(config=True)
     
     banner1 = Unicode(default_banner,
         help="""The part of the banner to be printed before the profile"""
     ).tag(config=True)
     banner2 = Unicode('',
         help="""The part of the banner to be printed after the profile"""
     ).tag(config=True)
 
     cache_size = Integer(1000, help=
         """
         Set the size of the output cache.  The default is 1000, you can
         change it permanently in your config file.  Setting it to 0 completely
         disables the caching system, and the minimum value accepted is 3 (if
         you provide a value less than 3, it is reset to 0 and a warning is
         issued).  This limit is defined because otherwise you'll spend more
         time re-flushing a too small cache than working
         """
     ).tag(config=True)
     color_info = Bool(True, help=
         """
         Use colors for displaying information about objects. Because this
         information is passed through a pager (like 'less'), and some pagers
         get confused with color codes, this capability can be turned off.
         """
     ).tag(config=True)
     colors = CaselessStrEnum(('Neutral', 'NoColor','LightBG','Linux'),
                              default_value='Neutral',
         help="Set the color scheme (NoColor, Neutral, Linux, or LightBG)."
     ).tag(config=True)
     debug = Bool(False).tag(config=True)
     disable_failing_post_execute = Bool(False,
         help="Don't call post-execute functions that have failed in the past."
     ).tag(config=True)
     display_formatter = Instance(DisplayFormatter, allow_none=True)
     displayhook_class = Type(DisplayHook)
     display_pub_class = Type(DisplayPublisher)
     
     sphinxify_docstring = Bool(False, help=
         """
         Enables rich html representation of docstrings. (This requires the
         docrepr module).
         """).tag(config=True)
 
     @observe("sphinxify_docstring")
     def _sphinxify_docstring_changed(self, change):
         if change['new']:
             warn("`sphinxify_docstring` is provisional since IPython 5.0 and might change in future versions." , ProvisionalWarning)
 
     enable_html_pager = Bool(False, help=
         """
         (Provisional API) enables html representation in mime bundles sent
         to pagers.
         """).tag(config=True)
 
     @observe("enable_html_pager")
     def _enable_html_pager_changed(self, change):
         if change['new']:
             warn("`enable_html_pager` is provisional since IPython 5.0 and might change in future versions.", ProvisionalWarning)
 
     data_pub_class = None
 
     exit_now = Bool(False)
     exiter = Instance(ExitAutocall)
     @default('exiter')
     def _exiter_default(self):
         return ExitAutocall(self)
     # Monotonically increasing execution counter
     execution_count = Integer(1)
     filename = Unicode("<ipython console>")
     ipython_dir= Unicode('').tag(config=True) # Set to get_ipython_dir() in __init__
 
     # Used to transform cells before running them, and check whether code is complete
     input_transformer_manager = Instance('IPython.core.inputtransformer2.TransformerManager',
                                          ())
 
     @property
     def input_transformers_cleanup(self):
         return self.input_transformer_manager.cleanup_transforms
 
     input_transformers_post = List([],
         help="A list of string input transformers, to be applied after IPython's "
              "own input transformations."
     )
 
     @property
     def input_splitter(self):
         """Make this available for backward compatibility (pre-7.0 release) with existing code.
 
         For example, ipykernel ipykernel currently uses
         `shell.input_splitter.check_complete`
         """
         from warnings import warn
         warn("`input_splitter` is deprecated since IPython 7.0, prefer `input_transformer_manager`.",
              DeprecationWarning, stacklevel=2
         )
         return self.input_transformer_manager
 
     logstart = Bool(False, help=
         """
         Start logging to the default log file in overwrite mode.
         Use `logappend` to specify a log file to **append** logs to.
         """
     ).tag(config=True)
     logfile = Unicode('', help=
         """
         The name of the logfile to use.
         """
     ).tag(config=True)
     logappend = Unicode('', help=
         """
         Start logging to the given file in append mode.
         Use `logfile` to specify a log file to **overwrite** logs to.
         """
     ).tag(config=True)
     object_info_string_level = Enum((0,1,2), default_value=0,
     ).tag(config=True)
     pdb = Bool(False, help=
         """
         Automatically call the pdb debugger after every exception.
         """
     ).tag(config=True)
     display_page = Bool(False,
         help="""If True, anything that would be passed to the pager
         will be displayed as regular output instead."""
     ).tag(config=True)
 
     # deprecated prompt traits:
     
     prompt_in1 = Unicode('In [\\#]: ',
         help="Deprecated since IPython 4.0 and ignored since 5.0, set TerminalInteractiveShell.prompts object directly."
     ).tag(config=True)
     prompt_in2 = Unicode('   .\\D.: ',
         help="Deprecated since IPython 4.0 and ignored since 5.0, set TerminalInteractiveShell.prompts object directly."
     ).tag(config=True)
     prompt_out = Unicode('Out[\\#]: ',
         help="Deprecated since IPython 4.0 and ignored since 5.0, set TerminalInteractiveShell.prompts object directly."
     ).tag(config=True)
     prompts_pad_left = Bool(True,
         help="Deprecated since IPython 4.0 and ignored since 5.0, set TerminalInteractiveShell.prompts object directly."
     ).tag(config=True)
     
     @observe('prompt_in1', 'prompt_in2', 'prompt_out', 'prompt_pad_left')
     def _prompt_trait_changed(self, change):
         name = change['name']
         warn("InteractiveShell.{name} is deprecated since IPython 4.0"
              " and ignored since 5.0, set TerminalInteractiveShell.prompts"
              " object directly.".format(name=name))
         
         # protect against weird cases where self.config may not exist:
 
     show_rewritten_input = Bool(True,
         help="Show rewritten input, e.g. for autocall."
     ).tag(config=True)
 
     quiet = Bool(False).tag(config=True)
 
     history_length = Integer(10000,
         help='Total length of command history'
     ).tag(config=True)
 
     history_load_length = Integer(1000, help=
         """
         The number of saved history entries to be loaded
         into the history buffer at startup.
         """
     ).tag(config=True)
 
     ast_node_interactivity = Enum(['all', 'last', 'last_expr', 'none', 'last_expr_or_assign'],
                                   default_value='last_expr',
                                   help="""
         'all', 'last', 'last_expr' or 'none', 'last_expr_or_assign' specifying
         which nodes should be run interactively (displaying output from expressions).
         """
     ).tag(config=True)
 
     # TODO: this part of prompt management should be moved to the frontends.
     # Use custom TraitTypes that convert '0'->'' and '\\n'->'\n'
     separate_in = SeparateUnicode('\n').tag(config=True)
     separate_out = SeparateUnicode('').tag(config=True)
     separate_out2 = SeparateUnicode('').tag(config=True)
     wildcards_case_sensitive = Bool(True).tag(config=True)
     xmode = CaselessStrEnum(('Context', 'Plain', 'Verbose', 'Minimal'),
                             default_value='Context',
                             help="Switch modes for the IPython exception handlers."
                             ).tag(config=True)
 
     # Subcomponents of InteractiveShell
     alias_manager = Instance('IPython.core.alias.AliasManager', allow_none=True)
     prefilter_manager = Instance('IPython.core.prefilter.PrefilterManager', allow_none=True)
     builtin_trap = Instance('IPython.core.builtin_trap.BuiltinTrap', allow_none=True)
     display_trap = Instance('IPython.core.display_trap.DisplayTrap', allow_none=True)
     extension_manager = Instance('IPython.core.extensions.ExtensionManager', allow_none=True)
     payload_manager = Instance('IPython.core.payload.PayloadManager', allow_none=True)
     history_manager = Instance('IPython.core.history.HistoryAccessorBase', allow_none=True)
     magics_manager = Instance('IPython.core.magic.MagicsManager', allow_none=True)
 
     profile_dir = Instance('IPython.core.application.ProfileDir', allow_none=True)
     @property
     def profile(self):
         if self.profile_dir is not None:
             name = os.path.basename(self.profile_dir.location)
             return name.replace('profile_','')
 
 
     # Private interface
     _post_execute = Dict()
 
     # Tracks any GUI loop loaded for pylab
     pylab_gui_select = None
 
     last_execution_succeeded = Bool(True, help='Did last executed command succeeded')
 
     last_execution_result = Instance('IPython.core.interactiveshell.ExecutionResult', help='Result of executing the last command', allow_none=True)
 
     def __init__(self, ipython_dir=None, profile_dir=None,
                  user_module=None, user_ns=None,
                  custom_exceptions=((), None), **kwargs):
 
         # This is where traits with a config_key argument are updated
         # from the values on config.
         super(InteractiveShell, self).__init__(**kwargs)
         if 'PromptManager' in self.config:
             warn('As of IPython 5.0 `PromptManager` config will have no effect'
                  ' and has been replaced by TerminalInteractiveShell.prompts_class')
         self.configurables = [self]
 
         # These are relatively independent and stateless
         self.init_ipython_dir(ipython_dir)
         self.init_profile_dir(profile_dir)
         self.init_instance_attrs()
         self.init_environment()
         
         # Check if we're in a virtualenv, and set up sys.path.
         self.init_virtualenv()
 
         # Create namespaces (user_ns, user_global_ns, etc.)
         self.init_create_namespaces(user_module, user_ns)
         # This has to be done after init_create_namespaces because it uses
         # something in self.user_ns, but before init_sys_modules, which
         # is the first thing to modify sys.
         # TODO: When we override sys.stdout and sys.stderr before this class
         # is created, we are saving the overridden ones here. Not sure if this
         # is what we want to do.
         self.save_sys_module_state()
         self.init_sys_modules()
 
         # While we're trying to have each part of the code directly access what
         # it needs without keeping redundant references to objects, we have too
         # much legacy code that expects ip.db to exist.
         self.db = PickleShareDB(os.path.join(self.profile_dir.location, 'db'))
 
         self.init_history()
         self.init_encoding()
         self.init_prefilter()
 
         self.init_syntax_highlighting()
         self.init_hooks()
         self.init_events()
         self.init_pushd_popd_magic()
         self.init_user_ns()
         self.init_logger()
         self.init_builtins()
 
         # The following was in post_config_initialization
         self.init_inspector()
         self.raw_input_original = input
         self.init_completer()
         # TODO: init_io() needs to happen before init_traceback handlers
         # because the traceback handlers hardcode the stdout/stderr streams.
         # This logic in in debugger.Pdb and should eventually be changed.
         self.init_io()
         self.init_traceback_handlers(custom_exceptions)
         self.init_prompts()
         self.init_display_formatter()
         self.init_display_pub()
         self.init_data_pub()
         self.init_displayhook()
         self.init_magics()
         self.init_alias()
         self.init_logstart()
         self.init_pdb()
         self.init_extension_manager()
         self.init_payload()
         self.init_deprecation_warnings()
         self.hooks.late_startup_hook()
         self.events.trigger('shell_initialized', self)
         atexit.register(self.atexit_operations)
 
     def get_ipython(self):
         """Return the currently running IPython instance."""
         return self
 
     #-------------------------------------------------------------------------
     # Trait changed handlers
     #-------------------------------------------------------------------------
     @observe('ipython_dir')
     def _ipython_dir_changed(self, change):
         ensure_dir_exists(change['new'])
 
     def set_autoindent(self,value=None):
         """Set the autoindent flag.
 
         If called with no arguments, it acts as a toggle."""
         if value is None:
             self.autoindent = not self.autoindent
         else:
             self.autoindent = value
 
     #-------------------------------------------------------------------------
     # init_* methods called by __init__
     #-------------------------------------------------------------------------
 
     def init_ipython_dir(self, ipython_dir):
         if ipython_dir is not None:
             self.ipython_dir = ipython_dir
             return
 
         self.ipython_dir = get_ipython_dir()
 
     def init_profile_dir(self, profile_dir):
         if profile_dir is not None:
             self.profile_dir = profile_dir
             return
         self.profile_dir =\
             ProfileDir.create_profile_dir_by_name(self.ipython_dir, 'default')
 
     def init_instance_attrs(self):
         self.more = False
 
         # command compiler
         self.compile = CachingCompiler()
 
         # Make an empty namespace, which extension writers can rely on both
         # existing and NEVER being used by ipython itself.  This gives them a
         # convenient location for storing additional information and state
         # their extensions may require, without fear of collisions with other
         # ipython names that may develop later.
         self.meta = Struct()
 
         # Temporary files used for various purposes.  Deleted at exit.
         self.tempfiles = []
         self.tempdirs = []
 
         # keep track of where we started running (mainly for crash post-mortem)
         # This is not being used anywhere currently.
         self.starting_dir = os.getcwd()
 
         # Indentation management
         self.indent_current_nsp = 0
 
         # Dict to track post-execution functions that have been registered
         self._post_execute = {}
 
     def init_environment(self):
         """Any changes we need to make to the user's environment."""
         pass
 
     def init_encoding(self):
         # Get system encoding at startup time.  Certain terminals (like Emacs
         # under Win32 have it set to None, and we need to have a known valid
         # encoding to use in the raw_input() method
         try:
             self.stdin_encoding = sys.stdin.encoding or 'ascii'
         except AttributeError:
             self.stdin_encoding = 'ascii'
 
 
     @observe('colors')
     def init_syntax_highlighting(self, changes=None):
         # Python source parser/formatter for syntax highlighting
         pyformat = PyColorize.Parser(style=self.colors, parent=self).format
         self.pycolorize = lambda src: pyformat(src,'str')
 
     def refresh_style(self):
         # No-op here, used in subclass
         pass
 
     def init_pushd_popd_magic(self):
         # for pushd/popd management
         self.home_dir = get_home_dir()
 
         self.dir_stack = []
 
     def init_logger(self):
         self.logger = Logger(self.home_dir, logfname='ipython_log.py',
                              logmode='rotate')
 
     def init_logstart(self):
         """Initialize logging in case it was requested at the command line.
         """
         if self.logappend:
             self.magic('logstart %s append' % self.logappend)
         elif self.logfile:
             self.magic('logstart %s' % self.logfile)
         elif self.logstart:
             self.magic('logstart')
 
     def init_deprecation_warnings(self):
         """
         register default filter for deprecation warning.
 
         This will allow deprecation warning of function used interactively to show
         warning to users, and still hide deprecation warning from libraries import.
         """
         if sys.version_info < (3,7):
             warnings.filterwarnings("default", category=DeprecationWarning, module=self.user_ns.get("__name__"))
 
 
     def init_builtins(self):
         # A single, static flag that we set to True.  Its presence indicates
         # that an IPython shell has been created, and we make no attempts at
         # removing on exit or representing the existence of more than one
         # IPython at a time.
         builtin_mod.__dict__['__IPYTHON__'] = True
         builtin_mod.__dict__['display'] = display
 
         self.builtin_trap = BuiltinTrap(shell=self)
 
     @observe('colors')
     def init_inspector(self, changes=None):
         # Object inspector
         self.inspector = oinspect.Inspector(oinspect.InspectColors,
                                             PyColorize.ANSICodeColors,
                                             self.colors,
                                             self.object_info_string_level)
 
     def init_io(self):
         # This will just use sys.stdout and sys.stderr. If you want to
         # override sys.stdout and sys.stderr themselves, you need to do that
         # *before* instantiating this class, because io holds onto
         # references to the underlying streams.
         # io.std* are deprecated, but don't show our own deprecation warnings
         # during initialization of the deprecated API.
         with warnings.catch_warnings():
             warnings.simplefilter('ignore', DeprecationWarning)
             io.stdout = io.IOStream(sys.stdout)
             io.stderr = io.IOStream(sys.stderr)
 
     def init_prompts(self):
         # Set system prompts, so that scripts can decide if they are running
         # interactively.
         sys.ps1 = 'In : '
         sys.ps2 = '...: '
         sys.ps3 = 'Out: '
 
     def init_display_formatter(self):
         self.display_formatter = DisplayFormatter(parent=self)
         self.configurables.append(self.display_formatter)
 
     def init_display_pub(self):
-        self.display_pub = self.display_pub_class(parent=self)
+        self.display_pub = self.display_pub_class(parent=self, shell=self)
         self.configurables.append(self.display_pub)
 
     def init_data_pub(self):
         if not self.data_pub_class:
             self.data_pub = None
             return
         self.data_pub = self.data_pub_class(parent=self)
         self.configurables.append(self.data_pub)
 
     def init_displayhook(self):
         # Initialize displayhook, set in/out prompts and printing system
         self.displayhook = self.displayhook_class(
             parent=self,
             shell=self,
             cache_size=self.cache_size,
         )
         self.configurables.append(self.displayhook)
         # This is a context manager that installs/revmoes the displayhook at
         # the appropriate time.
         self.display_trap = DisplayTrap(hook=self.displayhook)
 
     def init_virtualenv(self):
         """Add a virtualenv to sys.path so the user can import modules from it.
         This isn't perfect: it doesn't use the Python interpreter with which the
         virtualenv was built, and it ignores the --no-site-packages option. A
         warning will appear suggesting the user installs IPython in the
         virtualenv, but for many cases, it probably works well enough.
         
         Adapted from code snippets online.
         
         http://blog.ufsoft.org/2009/1/29/ipython-and-virtualenv
         """
         if 'VIRTUAL_ENV' not in os.environ:
             # Not in a virtualenv
             return
         
         p = os.path.normcase(sys.executable)
         p_venv = os.path.normcase(os.environ['VIRTUAL_ENV'])
 
         # executable path should end like /bin/python or \\scripts\\python.exe
         p_exe_up2 = os.path.dirname(os.path.dirname(p))
         if p_exe_up2 and os.path.exists(p_venv) and os.path.samefile(p_exe_up2, p_venv):
             # Our exe is inside the virtualenv, don't need to do anything.
             return
 
         # fallback venv detection:
         # stdlib venv may symlink sys.executable, so we can't use realpath.
         # but others can symlink *to* the venv Python, so we can't just use sys.executable.
         # So we just check every item in the symlink tree (generally <= 3)
         paths = [p]
         while os.path.islink(p):
             p = os.path.normcase(os.path.join(os.path.dirname(p), os.readlink(p)))
             paths.append(p)
         
         # In Cygwin paths like "c:\..." and '\cygdrive\c\...' are possible
         if p_venv.startswith('\\cygdrive'):
             p_venv = p_venv[11:]
         elif len(p_venv) >= 2 and p_venv[1] == ':':
             p_venv = p_venv[2:]
 
         if any(p_venv in p for p in paths):
             # Running properly in the virtualenv, don't need to do anything
             return
         
         warn("Attempting to work in a virtualenv. If you encounter problems, please "
              "install IPython inside the virtualenv.")
         if sys.platform == "win32":
             virtual_env = os.path.join(os.environ['VIRTUAL_ENV'], 'Lib', 'site-packages') 
         else:
             virtual_env = os.path.join(os.environ['VIRTUAL_ENV'], 'lib',
                        'python%d.%d' % sys.version_info[:2], 'site-packages')
         
         import site
         sys.path.insert(0, virtual_env)
         site.addsitedir(virtual_env)
 
     #-------------------------------------------------------------------------
     # Things related to injections into the sys module
     #-------------------------------------------------------------------------
 
     def save_sys_module_state(self):
         """Save the state of hooks in the sys module.
 
         This has to be called after self.user_module is created.
         """
         self._orig_sys_module_state = {'stdin': sys.stdin,
                                        'stdout': sys.stdout,
                                        'stderr': sys.stderr,
                                        'excepthook': sys.excepthook}
         self._orig_sys_modules_main_name = self.user_module.__name__
         self._orig_sys_modules_main_mod = sys.modules.get(self.user_module.__name__)
 
     def restore_sys_module_state(self):
         """Restore the state of the sys module."""
         try:
             for k, v in self._orig_sys_module_state.items():
                 setattr(sys, k, v)
         except AttributeError:
             pass
         # Reset what what done in self.init_sys_modules
         if self._orig_sys_modules_main_mod is not None:
             sys.modules[self._orig_sys_modules_main_name] = self._orig_sys_modules_main_mod
 
     #-------------------------------------------------------------------------
     # Things related to the banner
     #-------------------------------------------------------------------------
     
     @property
     def banner(self):
         banner = self.banner1
         if self.profile and self.profile != 'default':
             banner += '\nIPython profile: %s\n' % self.profile
         if self.banner2:
             banner += '\n' + self.banner2
         return banner
 
     def show_banner(self, banner=None):
         if banner is None:
             banner = self.banner
         sys.stdout.write(banner)
     
     #-------------------------------------------------------------------------
     # Things related to hooks
     #-------------------------------------------------------------------------
 
     def init_hooks(self):
         # hooks holds pointers used for user-side customizations
         self.hooks = Struct()
 
         self.strdispatchers = {}
 
         # Set all default hooks, defined in the IPython.hooks module.
         hooks = IPython.core.hooks
         for hook_name in hooks.__all__:
             # default hooks have priority 100, i.e. low; user hooks should have
             # 0-100 priority
             self.set_hook(hook_name,getattr(hooks,hook_name), 100, _warn_deprecated=False)
         
         if self.display_page:
             self.set_hook('show_in_pager', page.as_hook(page.display_page), 90)
     
     def set_hook(self,name,hook, priority=50, str_key=None, re_key=None,
                  _warn_deprecated=True):
         """set_hook(name,hook) -> sets an internal IPython hook.
 
         IPython exposes some of its internal API as user-modifiable hooks.  By
         adding your function to one of these hooks, you can modify IPython's
         behavior to call at runtime your own routines."""
 
         # At some point in the future, this should validate the hook before it
         # accepts it.  Probably at least check that the hook takes the number
         # of args it's supposed to.
 
         f = types.MethodType(hook,self)
 
         # check if the hook is for strdispatcher first
         if str_key is not None:
             sdp = self.strdispatchers.get(name, StrDispatch())
             sdp.add_s(str_key, f, priority )
             self.strdispatchers[name] = sdp
             return
         if re_key is not None:
             sdp = self.strdispatchers.get(name, StrDispatch())
             sdp.add_re(re.compile(re_key), f, priority )
             self.strdispatchers[name] = sdp
             return
 
         dp = getattr(self.hooks, name, None)
         if name not in IPython.core.hooks.__all__:
             print("Warning! Hook '%s' is not one of %s" % \
                   (name, IPython.core.hooks.__all__ ))
 
         if _warn_deprecated and (name in IPython.core.hooks.deprecated):
             alternative = IPython.core.hooks.deprecated[name]
             warn("Hook {} is deprecated. Use {} instead.".format(name, alternative), stacklevel=2)
 
         if not dp:
             dp = IPython.core.hooks.CommandChainDispatcher()
 
         try:
             dp.add(f,priority)
         except AttributeError:
             # it was not commandchain, plain old func - replace
             dp = f
 
         setattr(self.hooks,name, dp)
 
     #-------------------------------------------------------------------------
     # Things related to events
     #-------------------------------------------------------------------------
 
     def init_events(self):
         self.events = EventManager(self, available_events)
 
         self.events.register("pre_execute", self._clear_warning_registry)
 
     def register_post_execute(self, func):
         """DEPRECATED: Use ip.events.register('post_run_cell', func)
         
         Register a function for calling after code execution.
         """
         warn("ip.register_post_execute is deprecated, use "
              "ip.events.register('post_run_cell', func) instead.", stacklevel=2)
         self.events.register('post_run_cell', func)
     
     def _clear_warning_registry(self):
         # clear the warning registry, so that different code blocks with
         # overlapping line number ranges don't cause spurious suppression of
         # warnings (see gh-6611 for details)
         if "__warningregistry__" in self.user_global_ns:
             del self.user_global_ns["__warningregistry__"]
 
     #-------------------------------------------------------------------------
     # Things related to the "main" module
     #-------------------------------------------------------------------------
 
     def new_main_mod(self, filename, modname):
         """Return a new 'main' module object for user code execution.
         
         ``filename`` should be the path of the script which will be run in the
         module. Requests with the same filename will get the same module, with
         its namespace cleared.
         
         ``modname`` should be the module name - normally either '__main__' or
         the basename of the file without the extension.
         
         When scripts are executed via %run, we must keep a reference to their
         __main__ module around so that Python doesn't
         clear it, rendering references to module globals useless.
 
         This method keeps said reference in a private dict, keyed by the
         absolute path of the script. This way, for multiple executions of the
         same script we only keep one copy of the namespace (the last one),
         thus preventing memory leaks from old references while allowing the
         objects from the last execution to be accessible.
         """
         filename = os.path.abspath(filename)
         try:
             main_mod = self._main_mod_cache[filename]
         except KeyError:
             main_mod = self._main_mod_cache[filename] = types.ModuleType(
                         modname,
                         doc="Module created for script run in IPython")
         else:
             main_mod.__dict__.clear()
             main_mod.__name__ = modname
         
         main_mod.__file__ = filename
         # It seems pydoc (and perhaps others) needs any module instance to
         # implement a __nonzero__ method
         main_mod.__nonzero__ = lambda : True
         
         return main_mod
 
     def clear_main_mod_cache(self):
         """Clear the cache of main modules.
 
         Mainly for use by utilities like %reset.
 
         Examples
         --------
 
         In [15]: import IPython
 
         In [16]: m = _ip.new_main_mod(IPython.__file__, 'IPython')
 
         In [17]: len(_ip._main_mod_cache) > 0
         Out[17]: True
 
         In [18]: _ip.clear_main_mod_cache()
 
         In [19]: len(_ip._main_mod_cache) == 0
         Out[19]: True
         """
         self._main_mod_cache.clear()
 
     #-------------------------------------------------------------------------
     # Things related to debugging
     #-------------------------------------------------------------------------
 
     def init_pdb(self):
         # Set calling of pdb on exceptions
         # self.call_pdb is a property
         self.call_pdb = self.pdb
 
     def _get_call_pdb(self):
         return self._call_pdb
 
     def _set_call_pdb(self,val):
 
         if val not in (0,1,False,True):
             raise ValueError('new call_pdb value must be boolean')
 
         # store value in instance
         self._call_pdb = val
 
         # notify the actual exception handlers
         self.InteractiveTB.call_pdb = val
 
     call_pdb = property(_get_call_pdb,_set_call_pdb,None,
                         'Control auto-activation of pdb at exceptions')
 
     def debugger(self,force=False):
         """Call the pdb debugger.
 
         Keywords:
 
           - force(False): by default, this routine checks the instance call_pdb
             flag and does not actually invoke the debugger if the flag is false.
             The 'force' option forces the debugger to activate even if the flag
             is false.
         """
 
         if not (force or self.call_pdb):
             return
 
         if not hasattr(sys,'last_traceback'):
             error('No traceback has been produced, nothing to debug.')
             return
 
         self.InteractiveTB.debugger(force=True)
 
     #-------------------------------------------------------------------------
     # Things related to IPython's various namespaces
     #-------------------------------------------------------------------------
     default_user_namespaces = True
 
     def init_create_namespaces(self, user_module=None, user_ns=None):
         # Create the namespace where the user will operate.  user_ns is
         # normally the only one used, and it is passed to the exec calls as
         # the locals argument.  But we do carry a user_global_ns namespace
         # given as the exec 'globals' argument,  This is useful in embedding
         # situations where the ipython shell opens in a context where the
         # distinction between locals and globals is meaningful.  For
         # non-embedded contexts, it is just the same object as the user_ns dict.
 
         # FIXME. For some strange reason, __builtins__ is showing up at user
         # level as a dict instead of a module. This is a manual fix, but I
         # should really track down where the problem is coming from. Alex
         # Schmolck reported this problem first.
 
         # A useful post by Alex Martelli on this topic:
         # Re: inconsistent value from __builtins__
         # Von: Alex Martelli <[e-mail]>
         # Datum: Freitag 01 Oktober 2004 04:45:34 nachmittags/abends
         # Gruppen: comp.lang.python
 
         # Michael Hohn <[e-mail]> wrote:
         # > >>> print type(builtin_check.get_global_binding('__builtins__'))
         # > <type 'dict'>
         # > >>> print type(__builtins__)
         # > <type 'module'>
         # > Is this difference in return value intentional?
 
         # Well, it's documented that '__builtins__' can be either a dictionary
         # or a module, and it's been that way for a long time. Whether it's
         # intentional (or sensible), I don't know. In any case, the idea is
         # that if you need to access the built-in namespace directly, you
         # should start with "import __builtin__" (note, no 's') which will
         # definitely give you a module. Yeah, it's somewhat confusing:-(.
 
         # These routines return a properly built module and dict as needed by
         # the rest of the code, and can also be used by extension writers to
         # generate properly initialized namespaces.
         if (user_ns is not None) or (user_module is not None):
             self.default_user_namespaces = False
         self.user_module, self.user_ns = self.prepare_user_module(user_module, user_ns)
 
         # A record of hidden variables we have added to the user namespace, so
         # we can list later only variables defined in actual interactive use.
         self.user_ns_hidden = {}
 
         # Now that FakeModule produces a real module, we've run into a nasty
         # problem: after script execution (via %run), the module where the user
         # code ran is deleted.  Now that this object is a true module (needed
         # so doctest and other tools work correctly), the Python module
         # teardown mechanism runs over it, and sets to None every variable
         # present in that module.  Top-level references to objects from the
         # script survive, because the user_ns is updated with them.  However,
         # calling functions defined in the script that use other things from
         # the script will fail, because the function's closure had references
         # to the original objects, which are now all None.  So we must protect
         # these modules from deletion by keeping a cache.
         #
         # To avoid keeping stale modules around (we only need the one from the
         # last run), we use a dict keyed with the full path to the script, so
         # only the last version of the module is held in the cache.  Note,
         # however, that we must cache the module *namespace contents* (their
         # __dict__).  Because if we try to cache the actual modules, old ones
         # (uncached) could be destroyed while still holding references (such as
         # those held by GUI objects that tend to be long-lived)>
         #
         # The %reset command will flush this cache.  See the cache_main_mod()
         # and clear_main_mod_cache() methods for details on use.
 
         # This is the cache used for 'main' namespaces
         self._main_mod_cache = {}
 
         # A table holding all the namespaces IPython deals with, so that
         # introspection facilities can search easily.
         self.ns_table = {'user_global':self.user_module.__dict__,
                          'user_local':self.user_ns,
                          'builtin':builtin_mod.__dict__
                          }
     
     @property
     def user_global_ns(self):
         return self.user_module.__dict__
 
     def prepare_user_module(self, user_module=None, user_ns=None):
         """Prepare the module and namespace in which user code will be run.
         
         When IPython is started normally, both parameters are None: a new module
         is created automatically, and its __dict__ used as the namespace.
         
         If only user_module is provided, its __dict__ is used as the namespace.
         If only user_ns is provided, a dummy module is created, and user_ns
         becomes the global namespace. If both are provided (as they may be
         when embedding), user_ns is the local namespace, and user_module
         provides the global namespace.
 
         Parameters
         ----------
         user_module : module, optional
             The current user module in which IPython is being run. If None,
             a clean module will be created.
         user_ns : dict, optional
             A namespace in which to run interactive commands.
 
         Returns
         -------
         A tuple of user_module and user_ns, each properly initialised.
         """
         if user_module is None and user_ns is not None:
             user_ns.setdefault("__name__", "__main__")
             user_module = DummyMod()
             user_module.__dict__ = user_ns
             
         if user_module is None:
             user_module = types.ModuleType("__main__",
                 doc="Automatically created module for IPython interactive environment")
         
         # We must ensure that __builtin__ (without the final 's') is always
         # available and pointing to the __builtin__ *module*.  For more details:
         # http://mail.python.org/pipermail/python-dev/2001-April/014068.html
         user_module.__dict__.setdefault('__builtin__', builtin_mod)
         user_module.__dict__.setdefault('__builtins__', builtin_mod)
         
         if user_ns is None:
             user_ns = user_module.__dict__
 
         return user_module, user_ns
 
     def init_sys_modules(self):
         # We need to insert into sys.modules something that looks like a
         # module but which accesses the IPython namespace, for shelve and
         # pickle to work interactively. Normally they rely on getting
         # everything out of __main__, but for embedding purposes each IPython
         # instance has its own private namespace, so we can't go shoving
         # everything into __main__.
 
         # note, however, that we should only do this for non-embedded
         # ipythons, which really mimic the __main__.__dict__ with their own
         # namespace.  Embedded instances, on the other hand, should not do
         # this because they need to manage the user local/global namespaces
         # only, but they live within a 'normal' __main__ (meaning, they
         # shouldn't overtake the execution environment of the script they're
         # embedded in).
 
         # This is overridden in the InteractiveShellEmbed subclass to a no-op.
         main_name = self.user_module.__name__
         sys.modules[main_name] = self.user_module
 
     def init_user_ns(self):
         """Initialize all user-visible namespaces to their minimum defaults.
 
         Certain history lists are also initialized here, as they effectively
         act as user namespaces.
 
         Notes
         -----
         All data structures here are only filled in, they are NOT reset by this
         method.  If they were not empty before, data will simply be added to
         them.
         """
         # This function works in two parts: first we put a few things in
         # user_ns, and we sync that contents into user_ns_hidden so that these
         # initial variables aren't shown by %who.  After the sync, we add the
         # rest of what we *do* want the user to see with %who even on a new
         # session (probably nothing, so they really only see their own stuff)
 
         # The user dict must *always* have a __builtin__ reference to the
         # Python standard __builtin__ namespace,  which must be imported.
         # This is so that certain operations in prompt evaluation can be
         # reliably executed with builtins.  Note that we can NOT use
         # __builtins__ (note the 's'),  because that can either be a dict or a
         # module, and can even mutate at runtime, depending on the context
         # (Python makes no guarantees on it).  In contrast, __builtin__ is
         # always a module object, though it must be explicitly imported.
 
         # For more details:
         # http://mail.python.org/pipermail/python-dev/2001-April/014068.html
         ns = {}
         
         # make global variables for user access to the histories
         ns['_ih'] = self.history_manager.input_hist_parsed
         ns['_oh'] = self.history_manager.output_hist
         ns['_dh'] = self.history_manager.dir_hist
 
         # user aliases to input and output histories.  These shouldn't show up
         # in %who, as they can have very large reprs.
         ns['In']  = self.history_manager.input_hist_parsed
         ns['Out'] = self.history_manager.output_hist
 
         # Store myself as the public api!!!
         ns['get_ipython'] = self.get_ipython
         
         ns['exit'] = self.exiter
         ns['quit'] = self.exiter
 
         # Sync what we've added so far to user_ns_hidden so these aren't seen
         # by %who
         self.user_ns_hidden.update(ns)
 
         # Anything put into ns now would show up in %who.  Think twice before
         # putting anything here, as we really want %who to show the user their
         # stuff, not our variables.
 
         # Finally, update the real user's namespace
         self.user_ns.update(ns)
     
     @property
     def all_ns_refs(self):
         """Get a list of references to all the namespace dictionaries in which
         IPython might store a user-created object.
         
         Note that this does not include the displayhook, which also caches
         objects from the output."""
         return [self.user_ns, self.user_global_ns, self.user_ns_hidden] + \
                [m.__dict__ for m in self._main_mod_cache.values()]
 
     def reset(self, new_session=True):
         """Clear all internal namespaces, and attempt to release references to
         user objects.
 
         If new_session is True, a new history session will be opened.
         """
         # Clear histories
         self.history_manager.reset(new_session)
         # Reset counter used to index all histories
         if new_session:
             self.execution_count = 1
 
         # Reset last execution result
         self.last_execution_succeeded = True
         self.last_execution_result = None
         
         # Flush cached output items
         if self.displayhook.do_full_cache:
             self.displayhook.flush()
 
         # The main execution namespaces must be cleared very carefully,
         # skipping the deletion of the builtin-related keys, because doing so
         # would cause errors in many object's __del__ methods.
         if self.user_ns is not self.user_global_ns:
             self.user_ns.clear()
         ns = self.user_global_ns
         drop_keys = set(ns.keys())
         drop_keys.discard('__builtin__')
         drop_keys.discard('__builtins__')
         drop_keys.discard('__name__')
         for k in drop_keys:
             del ns[k]
 
         self.user_ns_hidden.clear()
 
         # Restore the user namespaces to minimal usability
         self.init_user_ns()
 
         # Restore the default and user aliases
         self.alias_manager.clear_aliases()
         self.alias_manager.init_aliases()
 
         # Now define aliases that only make sense on the terminal, because they
         # need direct access to the console in a way that we can't emulate in
         # GUI or web frontend
         if os.name == 'posix':
             for cmd in ('clear', 'more', 'less', 'man'):
                 if cmd not in self.magics_manager.magics['line']:
                     self.alias_manager.soft_define_alias(cmd, cmd)
 
         # Flush the private list of module references kept for script
         # execution protection
         self.clear_main_mod_cache()
 
     def del_var(self, varname, by_name=False):
         """Delete a variable from the various namespaces, so that, as
         far as possible, we're not keeping any hidden references to it.
 
         Parameters
         ----------
         varname : str
             The name of the variable to delete.
         by_name : bool
             If True, delete variables with the given name in each
             namespace. If False (default), find the variable in the user
             namespace, and delete references to it.
         """
         if varname in ('__builtin__', '__builtins__'):
             raise ValueError("Refusing to delete %s" % varname)
 
         ns_refs = self.all_ns_refs
         
         if by_name:                    # Delete by name
             for ns in ns_refs:
                 try:
                     del ns[varname]
                 except KeyError:
                     pass
         else:                         # Delete by object
             try:
                 obj = self.user_ns[varname]
             except KeyError:
                 raise NameError("name '%s' is not defined" % varname)
             # Also check in output history
             ns_refs.append(self.history_manager.output_hist)
             for ns in ns_refs:
                 to_delete = [n for n, o in ns.items() if o is obj]
                 for name in to_delete:
                     del ns[name]
 
             # Ensure it is removed from the last execution result
             if self.last_execution_result.result is obj:
                 self.last_execution_result = None
 
             # displayhook keeps extra references, but not in a dictionary
             for name in ('_', '__', '___'):
                 if getattr(self.displayhook, name) is obj:
                     setattr(self.displayhook, name, None)
 
     def reset_selective(self, regex=None):
         """Clear selective variables from internal namespaces based on a
         specified regular expression.
 
         Parameters
         ----------
         regex : string or compiled pattern, optional
             A regular expression pattern that will be used in searching
             variable names in the users namespaces.
         """
         if regex is not None:
             try:
                 m = re.compile(regex)
             except TypeError:
                 raise TypeError('regex must be a string or compiled pattern')
             # Search for keys in each namespace that match the given regex
             # If a match is found, delete the key/value pair.
             for ns in self.all_ns_refs:
                 for var in ns:
                     if m.search(var):
                         del ns[var]
 
     def push(self, variables, interactive=True):
         """Inject a group of variables into the IPython user namespace.
 
         Parameters
         ----------
         variables : dict, str or list/tuple of str
             The variables to inject into the user's namespace.  If a dict, a
             simple update is done.  If a str, the string is assumed to have
             variable names separated by spaces.  A list/tuple of str can also
             be used to give the variable names.  If just the variable names are
             give (list/tuple/str) then the variable values looked up in the
             callers frame.
         interactive : bool
             If True (default), the variables will be listed with the ``who``
             magic.
         """
         vdict = None
 
         # We need a dict of name/value pairs to do namespace updates.
         if isinstance(variables, dict):
             vdict = variables
         elif isinstance(variables, (str, list, tuple)):
             if isinstance(variables, str):
                 vlist = variables.split()
             else:
                 vlist = variables
             vdict = {}
             cf = sys._getframe(1)
             for name in vlist:
                 try:
                     vdict[name] = eval(name, cf.f_globals, cf.f_locals)
                 except:
                     print('Could not get variable %s from %s' %
                            (name,cf.f_code.co_name))
         else:
             raise ValueError('variables must be a dict/str/list/tuple')
 
         # Propagate variables to user namespace
         self.user_ns.update(vdict)
 
         # And configure interactive visibility
         user_ns_hidden = self.user_ns_hidden
         if interactive:
             for name in vdict:
                 user_ns_hidden.pop(name, None)
         else:
             user_ns_hidden.update(vdict)
 
     def drop_by_id(self, variables):
         """Remove a dict of variables from the user namespace, if they are the
         same as the values in the dictionary.
         
         This is intended for use by extensions: variables that they've added can
         be taken back out if they are unloaded, without removing any that the
         user has overwritten.
         
         Parameters
         ----------
         variables : dict
           A dictionary mapping object names (as strings) to the objects.
         """
         for name, obj in variables.items():
             if name in self.user_ns and self.user_ns[name] is obj:
                 del self.user_ns[name]
                 self.user_ns_hidden.pop(name, None)
 
     #-------------------------------------------------------------------------
     # Things related to object introspection
     #-------------------------------------------------------------------------
 
     def _ofind(self, oname, namespaces=None):
         """Find an object in the available namespaces.
 
         self._ofind(oname) -> dict with keys: found,obj,ospace,ismagic
 
         Has special code to detect magic functions.
         """
         oname = oname.strip()
         if not oname.startswith(ESC_MAGIC) and \
                 not oname.startswith(ESC_MAGIC2) and \
                 not all(a.isidentifier() for a in oname.split(".")):
             return {'found': False}
 
         if namespaces is None:
             # Namespaces to search in:
             # Put them in a list. The order is important so that we
             # find things in the same order that Python finds them.
             namespaces = [ ('Interactive', self.user_ns),
                            ('Interactive (global)', self.user_global_ns),
                            ('Python builtin', builtin_mod.__dict__),
                            ]
 
         ismagic = False
         isalias = False
         found = False
         ospace = None
         parent = None
         obj = None
 
 
         # Look for the given name by splitting it in parts.  If the head is
         # found, then we look for all the remaining parts as members, and only
         # declare success if we can find them all.
         oname_parts = oname.split('.')
         oname_head, oname_rest = oname_parts[0],oname_parts[1:]
         for nsname,ns in namespaces:
             try:
                 obj = ns[oname_head]
             except KeyError:
                 continue
             else:
                 for idx, part in enumerate(oname_rest):
                     try:
                         parent = obj
                         # The last part is looked up in a special way to avoid
                         # descriptor invocation as it may raise or have side
                         # effects.
                         if idx == len(oname_rest) - 1:
                             obj = self._getattr_property(obj, part)
                         else:
                             obj = getattr(obj, part)
                     except:
                         # Blanket except b/c some badly implemented objects
                         # allow __getattr__ to raise exceptions other than
                         # AttributeError, which then crashes IPython.
                         break
                 else:
                     # If we finish the for loop (no break), we got all members
                     found = True
                     ospace = nsname
                     break  # namespace loop
 
         # Try to see if it's magic
         if not found:
             obj = None
             if oname.startswith(ESC_MAGIC2):
                 oname = oname.lstrip(ESC_MAGIC2)
                 obj = self.find_cell_magic(oname)
             elif oname.startswith(ESC_MAGIC):
                 oname = oname.lstrip(ESC_MAGIC)
                 obj = self.find_line_magic(oname)
             else:
                 # search without prefix, so run? will find %run?
                 obj = self.find_line_magic(oname)
                 if obj is None:
                     obj = self.find_cell_magic(oname)
             if obj is not None:
                 found = True
                 ospace = 'IPython internal'
                 ismagic = True
                 isalias = isinstance(obj, Alias)
 
         # Last try: special-case some literals like '', [], {}, etc:
         if not found and oname_head in ["''",'""','[]','{}','()']:
             obj = eval(oname_head)
             found = True
             ospace = 'Interactive'
 
         return {
                 'obj':obj,
                 'found':found,
                 'parent':parent,
                 'ismagic':ismagic,
                 'isalias':isalias,
                 'namespace':ospace
                }
 
     @staticmethod
     def _getattr_property(obj, attrname):
         """Property-aware getattr to use in object finding.
 
         If attrname represents a property, return it unevaluated (in case it has
         side effects or raises an error.
 
         """
         if not isinstance(obj, type):
             try:
                 # `getattr(type(obj), attrname)` is not guaranteed to return
                 # `obj`, but does so for property:
                 #
                 # property.__get__(self, None, cls) -> self
                 #
                 # The universal alternative is to traverse the mro manually
                 # searching for attrname in class dicts.
                 attr = getattr(type(obj), attrname)
             except AttributeError:
                 pass
             else:
                 # This relies on the fact that data descriptors (with both
                 # __get__ & __set__ magic methods) take precedence over
                 # instance-level attributes:
                 #
                 #    class A(object):
                 #        @property
                 #        def foobar(self): return 123
                 #    a = A()
                 #    a.__dict__['foobar'] = 345
                 #    a.foobar  # == 123
                 #
                 # So, a property may be returned right away.
                 if isinstance(attr, property):
                     return attr
 
         # Nothing helped, fall back.
         return getattr(obj, attrname)
 
     def _object_find(self, oname, namespaces=None):
         """Find an object and return a struct with info about it."""
         return Struct(self._ofind(oname, namespaces))
 
     def _inspect(self, meth, oname, namespaces=None, **kw):
         """Generic interface to the inspector system.
 
         This function is meant to be called by pdef, pdoc & friends.
         """
         info = self._object_find(oname, namespaces)
         docformat = sphinxify if self.sphinxify_docstring else None
         if info.found:
             pmethod = getattr(self.inspector, meth)
             # TODO: only apply format_screen to the plain/text repr of the mime
             # bundle.
             formatter = format_screen if info.ismagic else docformat
             if meth == 'pdoc':
                 pmethod(info.obj, oname, formatter)
             elif meth == 'pinfo':
                 pmethod(info.obj, oname, formatter, info, 
                         enable_html_pager=self.enable_html_pager, **kw)
             else:
                 pmethod(info.obj, oname)
         else:
             print('Object `%s` not found.' % oname)
             return 'not found'  # so callers can take other action
 
     def object_inspect(self, oname, detail_level=0):
         """Get object info about oname"""
         with self.builtin_trap:
             info = self._object_find(oname)
             if info.found:
                 return self.inspector.info(info.obj, oname, info=info,
                             detail_level=detail_level
                 )
             else:
                 return oinspect.object_info(name=oname, found=False)
 
     def object_inspect_text(self, oname, detail_level=0):
         """Get object info as formatted text"""
         return self.object_inspect_mime(oname, detail_level)['text/plain']
 
     def object_inspect_mime(self, oname, detail_level=0):
         """Get object info as a mimebundle of formatted representations.
 
         A mimebundle is a dictionary, keyed by mime-type.
         It must always have the key `'text/plain'`.
         """
         with self.builtin_trap:
             info = self._object_find(oname)
             if info.found:
                 return self.inspector._get_info(info.obj, oname, info=info,
                             detail_level=detail_level
                 )
             else:
                 raise KeyError(oname)
 
     #-------------------------------------------------------------------------
     # Things related to history management
     #-------------------------------------------------------------------------
 
     def init_history(self):
         """Sets up the command history, and starts regular autosaves."""
         self.history_manager = HistoryManager(shell=self, parent=self)
         self.configurables.append(self.history_manager)
 
     #-------------------------------------------------------------------------
     # Things related to exception handling and tracebacks (not debugging)
     #-------------------------------------------------------------------------
 
     debugger_cls = Pdb
 
     def init_traceback_handlers(self, custom_exceptions):
         # Syntax error handler.
         self.SyntaxTB = ultratb.SyntaxTB(color_scheme='NoColor', parent=self)
 
         # The interactive one is initialized with an offset, meaning we always
         # want to remove the topmost item in the traceback, which is our own
         # internal code. Valid modes: ['Plain','Context','Verbose','Minimal']
         self.InteractiveTB = ultratb.AutoFormattedTB(mode = 'Plain',
                                                      color_scheme='NoColor',
                                                      tb_offset = 1,
                                    check_cache=check_linecache_ipython,
                                    debugger_cls=self.debugger_cls, parent=self)
 
         # The instance will store a pointer to the system-wide exception hook,
         # so that runtime code (such as magics) can access it.  This is because
         # during the read-eval loop, it may get temporarily overwritten.
         self.sys_excepthook = sys.excepthook
 
         # and add any custom exception handlers the user may have specified
         self.set_custom_exc(*custom_exceptions)
 
         # Set the exception mode
         self.InteractiveTB.set_mode(mode=self.xmode)
 
     def set_custom_exc(self, exc_tuple, handler):
         """set_custom_exc(exc_tuple, handler)
 
         Set a custom exception handler, which will be called if any of the
         exceptions in exc_tuple occur in the mainloop (specifically, in the
         run_code() method).
 
         Parameters
         ----------
 
         exc_tuple : tuple of exception classes
             A *tuple* of exception classes, for which to call the defined
             handler.  It is very important that you use a tuple, and NOT A
             LIST here, because of the way Python's except statement works.  If
             you only want to trap a single exception, use a singleton tuple::
 
                 exc_tuple == (MyCustomException,)
 
         handler : callable
             handler must have the following signature::
 
                 def my_handler(self, etype, value, tb, tb_offset=None):
                     ...
                     return structured_traceback
 
             Your handler must return a structured traceback (a list of strings),
             or None.
 
             This will be made into an instance method (via types.MethodType)
             of IPython itself, and it will be called if any of the exceptions
             listed in the exc_tuple are caught. If the handler is None, an
             internal basic one is used, which just prints basic info.
 
             To protect IPython from crashes, if your handler ever raises an
             exception or returns an invalid result, it will be immediately
             disabled.
 
         WARNING: by putting in your own exception handler into IPython's main
         execution loop, you run a very good chance of nasty crashes.  This
         facility should only be used if you really know what you are doing."""
         if not isinstance(exc_tuple, tuple):
             raise TypeError("The custom exceptions must be given as a tuple.")
 
         def dummy_handler(self, etype, value, tb, tb_offset=None):
             print('*** Simple custom exception handler ***')
             print('Exception type :', etype)
             print('Exception value:', value)
             print('Traceback      :', tb)
         
         def validate_stb(stb):
             """validate structured traceback return type
             
             return type of CustomTB *should* be a list of strings, but allow
             single strings or None, which are harmless.
             
             This function will *always* return a list of strings,
             and will raise a TypeError if stb is inappropriate.
             """
             msg = "CustomTB must return list of strings, not %r" % stb
             if stb is None:
                 return []
             elif isinstance(stb, str):
                 return [stb]
             elif not isinstance(stb, list):
                 raise TypeError(msg)
             # it's a list
             for line in stb:
                 # check every element
                 if not isinstance(line, str):
                     raise TypeError(msg)
             return stb
 
         if handler is None:
             wrapped = dummy_handler
         else:
             def wrapped(self,etype,value,tb,tb_offset=None):
                 """wrap CustomTB handler, to protect IPython from user code
                 
                 This makes it harder (but not impossible) for custom exception
                 handlers to crash IPython.
                 """
                 try:
                     stb = handler(self,etype,value,tb,tb_offset=tb_offset)
                     return validate_stb(stb)
                 except:
                     # clear custom handler immediately
                     self.set_custom_exc((), None)
                     print("Custom TB Handler failed, unregistering", file=sys.stderr)
                     # show the exception in handler first
                     stb = self.InteractiveTB.structured_traceback(*sys.exc_info())
                     print(self.InteractiveTB.stb2text(stb))
                     print("The original exception:")
                     stb = self.InteractiveTB.structured_traceback(
                                             (etype,value,tb), tb_offset=tb_offset
                     )
                 return stb
 
         self.CustomTB = types.MethodType(wrapped,self)
         self.custom_exceptions = exc_tuple
 
     def excepthook(self, etype, value, tb):
         """One more defense for GUI apps that call sys.excepthook.
 
         GUI frameworks like wxPython trap exceptions and call
         sys.excepthook themselves.  I guess this is a feature that
         enables them to keep running after exceptions that would
         otherwise kill their mainloop. This is a bother for IPython
         which excepts to catch all of the program exceptions with a try:
         except: statement.
 
         Normally, IPython sets sys.excepthook to a CrashHandler instance, so if
         any app directly invokes sys.excepthook, it will look to the user like
         IPython crashed.  In order to work around this, we can disable the
         CrashHandler and replace it with this excepthook instead, which prints a
         regular traceback using our InteractiveTB.  In this fashion, apps which
         call sys.excepthook will generate a regular-looking exception from
         IPython, and the CrashHandler will only be triggered by real IPython
         crashes.
 
         This hook should be used sparingly, only in places which are not likely
         to be true IPython errors.
         """
         self.showtraceback((etype, value, tb), tb_offset=0)
 
     def _get_exc_info(self, exc_tuple=None):
         """get exc_info from a given tuple, sys.exc_info() or sys.last_type etc.
         
         Ensures sys.last_type,value,traceback hold the exc_info we found,
         from whichever source.
         
         raises ValueError if none of these contain any information
         """
         if exc_tuple is None:
             etype, value, tb = sys.exc_info()
         else:
             etype, value, tb = exc_tuple
 
         if etype is None:
             if hasattr(sys, 'last_type'):
                 etype, value, tb = sys.last_type, sys.last_value, \
                                    sys.last_traceback
         
         if etype is None:
             raise ValueError("No exception to find")
         
         # Now store the exception info in sys.last_type etc.
         # WARNING: these variables are somewhat deprecated and not
         # necessarily safe to use in a threaded environment, but tools
         # like pdb depend on their existence, so let's set them.  If we
         # find problems in the field, we'll need to revisit their use.
         sys.last_type = etype
         sys.last_value = value
         sys.last_traceback = tb
         
         return etype, value, tb
     
     def show_usage_error(self, exc):
         """Show a short message for UsageErrors
         
         These are special exceptions that shouldn't show a traceback.
         """
         print("UsageError: %s" % exc, file=sys.stderr)
     
     def get_exception_only(self, exc_tuple=None):
         """
         Return as a string (ending with a newline) the exception that
         just occurred, without any traceback.
         """
         etype, value, tb = self._get_exc_info(exc_tuple)
         msg = traceback.format_exception_only(etype, value)
         return ''.join(msg)
 
     def showtraceback(self, exc_tuple=None, filename=None, tb_offset=None,
                       exception_only=False, running_compiled_code=False):
         """Display the exception that just occurred.
 
         If nothing is known about the exception, this is the method which
         should be used throughout the code for presenting user tracebacks,
         rather than directly invoking the InteractiveTB object.
 
         A specific showsyntaxerror() also exists, but this method can take
         care of calling it if needed, so unless you are explicitly catching a
         SyntaxError exception, don't try to analyze the stack manually and
         simply call this method."""
 
         try:
             try:
                 etype, value, tb = self._get_exc_info(exc_tuple)
             except ValueError:
                 print('No traceback available to show.', file=sys.stderr)
                 return
 
             if issubclass(etype, SyntaxError):
                 # Though this won't be called by syntax errors in the input
                 # line, there may be SyntaxError cases with imported code.
                 self.showsyntaxerror(filename, running_compiled_code)
             elif etype is UsageError:
                 self.show_usage_error(value)
             else:
                 if exception_only:
                     stb = ['An exception has occurred, use %tb to see '
                            'the full traceback.\n']
                     stb.extend(self.InteractiveTB.get_exception_only(etype,
                                                                      value))
                 else:
                     try:
                         # Exception classes can customise their traceback - we
                         # use this in IPython.parallel for exceptions occurring
                         # in the engines. This should return a list of strings.
                         stb = value._render_traceback_()
                     except Exception:
                         stb = self.InteractiveTB.structured_traceback(etype,
                                             value, tb, tb_offset=tb_offset)
 
                     self._showtraceback(etype, value, stb)
                     if self.call_pdb:
                         # drop into debugger
                         self.debugger(force=True)
                     return
 
                 # Actually show the traceback
                 self._showtraceback(etype, value, stb)
 
         except KeyboardInterrupt:
             print('\n' + self.get_exception_only(), file=sys.stderr)
 
     def _showtraceback(self, etype, evalue, stb):
         """Actually show a traceback.
 
         Subclasses may override this method to put the traceback on a different
         place, like a side channel.
         """
         print(self.InteractiveTB.stb2text(stb))
 
     def showsyntaxerror(self, filename=None, running_compiled_code=False):
         """Display the syntax error that just occurred.
 
         This doesn't display a stack trace because there isn't one.
 
         If a filename is given, it is stuffed in the exception instead
         of what was there before (because Python's parser always uses
         "<string>" when reading from a string).
 
         If the syntax error occurred when running a compiled code (i.e. running_compile_code=True),
         longer stack trace will be displayed.
          """
         etype, value, last_traceback = self._get_exc_info()
 
         if filename and issubclass(etype, SyntaxError):
             try:
                 value.filename = filename
             except:
                 # Not the format we expect; leave it alone
                 pass
 
         # If the error occurred when executing compiled code, we should provide full stacktrace.
         elist = traceback.extract_tb(last_traceback) if running_compiled_code else []
         stb = self.SyntaxTB.structured_traceback(etype, value, elist)
         self._showtraceback(etype, value, stb)
 
     # This is overridden in TerminalInteractiveShell to show a message about
     # the %paste magic.
     def showindentationerror(self):
         """Called by _run_cell when there's an IndentationError in code entered
         at the prompt.
 
         This is overridden in TerminalInteractiveShell to show a message about
         the %paste magic."""
         self.showsyntaxerror()
 
     #-------------------------------------------------------------------------
     # Things related to readline
     #-------------------------------------------------------------------------
 
     def init_readline(self):
         """DEPRECATED
         
         Moved to terminal subclass, here only to simplify the init logic."""
         # Set a number of methods that depend on readline to be no-op
         warnings.warn('`init_readline` is no-op since IPython 5.0 and is Deprecated',
                 DeprecationWarning, stacklevel=2)
         self.set_custom_completer = no_op
 
     @skip_doctest
     def set_next_input(self, s, replace=False):
         """ Sets the 'default' input string for the next command line.
 
         Example::
 
             In [1]: _ip.set_next_input("Hello Word")
             In [2]: Hello Word_  # cursor is here
         """
         self.rl_next_input = s
 
     def _indent_current_str(self):
         """return the current level of indentation as a string"""
         return self.input_splitter.get_indent_spaces() * ' '
 
     #-------------------------------------------------------------------------
     # Things related to text completion
     #-------------------------------------------------------------------------
 
     def init_completer(self):
         """Initialize the completion machinery.
 
         This creates completion machinery that can be used by client code,
         either interactively in-process (typically triggered by the readline
         library), programmatically (such as in test suites) or out-of-process
         (typically over the network by remote frontends).
         """
         from IPython.core.completer import IPCompleter
         from IPython.core.completerlib import (module_completer,
                 magic_run_completer, cd_completer, reset_completer)
 
         self.Completer = IPCompleter(shell=self,
                                      namespace=self.user_ns,
                                      global_namespace=self.user_global_ns,
                                      parent=self,
                                      )
         self.configurables.append(self.Completer)
 
         # Add custom completers to the basic ones built into IPCompleter
         sdisp = self.strdispatchers.get('complete_command', StrDispatch())
         self.strdispatchers['complete_command'] = sdisp
         self.Completer.custom_completers = sdisp
 
         self.set_hook('complete_command', module_completer, str_key = 'import')
         self.set_hook('complete_command', module_completer, str_key = 'from')
         self.set_hook('complete_command', module_completer, str_key = '%aimport')
         self.set_hook('complete_command', magic_run_completer, str_key = '%run')
         self.set_hook('complete_command', cd_completer, str_key = '%cd')
         self.set_hook('complete_command', reset_completer, str_key = '%reset')
 
     @skip_doctest
     def complete(self, text, line=None, cursor_pos=None):
         """Return the completed text and a list of completions.
 
         Parameters
         ----------
 
            text : string
              A string of text to be completed on.  It can be given as empty and
              instead a line/position pair are given.  In this case, the
              completer itself will split the line like readline does.
 
            line : string, optional
              The complete line that text is part of.
 
            cursor_pos : int, optional
              The position of the cursor on the input line.
 
         Returns
         -------
           text : string
             The actual text that was completed.
 
           matches : list
             A sorted list with all possible completions.
 
         The optional arguments allow the completion to take more context into
         account, and are part of the low-level completion API.
 
         This is a wrapper around the completion mechanism, similar to what
         readline does at the command line when the TAB key is hit.  By
         exposing it as a method, it can be used by other non-readline
         environments (such as GUIs) for text completion.
 
         Simple usage example:
 
         In [1]: x = 'hello'
 
         In [2]: _ip.complete('x.l')
         Out[2]: ('x.l', ['x.ljust', 'x.lower', 'x.lstrip'])
         """
 
         # Inject names into __builtin__ so we can complete on the added names.
         with self.builtin_trap:
             return self.Completer.complete(text, line, cursor_pos)
 
     def set_custom_completer(self, completer, pos=0):
         """Adds a new custom completer function.
 
         The position argument (defaults to 0) is the index in the completers
         list where you want the completer to be inserted."""
 
         newcomp = types.MethodType(completer,self.Completer)
         self.Completer.matchers.insert(pos,newcomp)
 
     def set_completer_frame(self, frame=None):
         """Set the frame of the completer."""
         if frame:
             self.Completer.namespace = frame.f_locals
             self.Completer.global_namespace = frame.f_globals
         else:
             self.Completer.namespace = self.user_ns
             self.Completer.global_namespace = self.user_global_ns
 
     #-------------------------------------------------------------------------
     # Things related to magics
     #-------------------------------------------------------------------------
 
     def init_magics(self):
         from IPython.core import magics as m
         self.magics_manager = magic.MagicsManager(shell=self,
                                    parent=self,
                                    user_magics=m.UserMagics(self))
         self.configurables.append(self.magics_manager)
 
         # Expose as public API from the magics manager
         self.register_magics = self.magics_manager.register
 
         self.register_magics(m.AutoMagics, m.BasicMagics, m.CodeMagics,
             m.ConfigMagics, m.DisplayMagics, m.ExecutionMagics,
             m.ExtensionMagics, m.HistoryMagics, m.LoggingMagics,
             m.NamespaceMagics, m.OSMagics, m.PackagingMagics,
             m.PylabMagics, m.ScriptMagics,
         )
         self.register_magics(m.AsyncMagics)
 
         # Register Magic Aliases
         mman = self.magics_manager
         # FIXME: magic aliases should be defined by the Magics classes
         # or in MagicsManager, not here
         mman.register_alias('ed', 'edit')
         mman.register_alias('hist', 'history')
         mman.register_alias('rep', 'recall')
         mman.register_alias('SVG', 'svg', 'cell')
         mman.register_alias('HTML', 'html', 'cell')
         mman.register_alias('file', 'writefile', 'cell')
 
         # FIXME: Move the color initialization to the DisplayHook, which
         # should be split into a prompt manager and displayhook. We probably
         # even need a centralize colors management object.
         self.run_line_magic('colors', self.colors)
     
     # Defined here so that it's included in the documentation
     @functools.wraps(magic.MagicsManager.register_function)
     def register_magic_function(self, func, magic_kind='line', magic_name=None):
         self.magics_manager.register_function(func, 
                                   magic_kind=magic_kind, magic_name=magic_name)
 
     def run_line_magic(self, magic_name, line, _stack_depth=1):
         """Execute the given line magic.
 
         Parameters
         ----------
         magic_name : str
           Name of the desired magic function, without '%' prefix.
 
         line : str
           The rest of the input line as a single string.
           
         _stack_depth : int
           If run_line_magic() is called from magic() then _stack_depth=2.
           This is added to ensure backward compatibility for use of 'get_ipython().magic()'
         """
         fn = self.find_line_magic(magic_name)
         if fn is None:
             cm = self.find_cell_magic(magic_name)
             etpl = "Line magic function `%%%s` not found%s."
             extra = '' if cm is None else (' (But cell magic `%%%%%s` exists, '
                                     'did you mean that instead?)' % magic_name )
             raise UsageError(etpl % (magic_name, extra))
         else:
             # Note: this is the distance in the stack to the user's frame.
             # This will need to be updated if the internal calling logic gets
             # refactored, or else we'll be expanding the wrong variables.
 
             # Determine stack_depth depending on where run_line_magic() has been called
             stack_depth = _stack_depth
             if getattr(fn, magic.MAGIC_NO_VAR_EXPAND_ATTR, False):
                 # magic has opted out of var_expand
                 magic_arg_s = line
             else:
                 magic_arg_s = self.var_expand(line, stack_depth)
             # Put magic args in a list so we can call with f(*a) syntax
             args = [magic_arg_s]
             kwargs = {}
             # Grab local namespace if we need it:
             if getattr(fn, "needs_local_scope", False):
                 kwargs['local_ns'] = sys._getframe(stack_depth).f_locals
             with self.builtin_trap:
                 result = fn(*args, **kwargs)
             return result
 
     def run_cell_magic(self, magic_name, line, cell):
         """Execute the given cell magic.
 
         Parameters
         ----------
         magic_name : str
           Name of the desired magic function, without '%' prefix.
 
         line : str
           The rest of the first input line as a single string.
 
         cell : str
           The body of the cell as a (possibly multiline) string.
         """
         fn = self.find_cell_magic(magic_name)
         if fn is None:
             lm = self.find_line_magic(magic_name)
             etpl = "Cell magic `%%{0}` not found{1}."
             extra = '' if lm is None else (' (But line magic `%{0}` exists, '
                             'did you mean that instead?)'.format(magic_name))
             raise UsageError(etpl.format(magic_name, extra))
         elif cell == '':
             message = '%%{0} is a cell magic, but the cell body is empty.'.format(magic_name)
             if self.find_line_magic(magic_name) is not None:
                 message += ' Did you mean the line magic %{0} (single %)?'.format(magic_name)
             raise UsageError(message)
         else:
             # Note: this is the distance in the stack to the user's frame.
             # This will need to be updated if the internal calling logic gets
             # refactored, or else we'll be expanding the wrong variables.
             stack_depth = 2
             if getattr(fn, magic.MAGIC_NO_VAR_EXPAND_ATTR, False):
                 # magic has opted out of var_expand
                 magic_arg_s = line
             else:
                 magic_arg_s = self.var_expand(line, stack_depth)
             kwargs = {}
             if getattr(fn, "needs_local_scope", False):
                 kwargs['local_ns'] = self.user_ns
 
             with self.builtin_trap:
                 args = (magic_arg_s, cell)
                 result = fn(*args, **kwargs)
             return result
 
     def find_line_magic(self, magic_name):
         """Find and return a line magic by name.
 
         Returns None if the magic isn't found."""
         return self.magics_manager.magics['line'].get(magic_name)
 
     def find_cell_magic(self, magic_name):
         """Find and return a cell magic by name.
 
         Returns None if the magic isn't found."""
         return self.magics_manager.magics['cell'].get(magic_name)
 
     def find_magic(self, magic_name, magic_kind='line'):
         """Find and return a magic of the given type by name.
 
         Returns None if the magic isn't found."""
         return self.magics_manager.magics[magic_kind].get(magic_name)
 
     def magic(self, arg_s):
         """DEPRECATED. Use run_line_magic() instead.
 
         Call a magic function by name.
 
         Input: a string containing the name of the magic function to call and
         any additional arguments to be passed to the magic.
 
         magic('name -opt foo bar') is equivalent to typing at the ipython
         prompt:
 
         In[1]: %name -opt foo bar
 
         To call a magic without arguments, simply use magic('name').
 
         This provides a proper Python function to call IPython's magics in any
         valid Python code you can type at the interpreter, including loops and
         compound statements.
         """
         # TODO: should we issue a loud deprecation warning here?
         magic_name, _, magic_arg_s = arg_s.partition(' ')
         magic_name = magic_name.lstrip(prefilter.ESC_MAGIC)
         return self.run_line_magic(magic_name, magic_arg_s, _stack_depth=2)
 
     #-------------------------------------------------------------------------
     # Things related to macros
     #-------------------------------------------------------------------------
 
     def define_macro(self, name, themacro):
         """Define a new macro
 
         Parameters
         ----------
         name : str
             The name of the macro.
         themacro : str or Macro
             The action to do upon invoking the macro.  If a string, a new
             Macro object is created by passing the string to it.
         """
 
         from IPython.core import macro
 
         if isinstance(themacro, str):
             themacro = macro.Macro(themacro)
         if not isinstance(themacro, macro.Macro):
             raise ValueError('A macro must be a string or a Macro instance.')
         self.user_ns[name] = themacro
 
     #-------------------------------------------------------------------------
     # Things related to the running of system commands
     #-------------------------------------------------------------------------
 
     def system_piped(self, cmd):
         """Call the given cmd in a subprocess, piping stdout/err
 
         Parameters
         ----------
         cmd : str
           Command to execute (can not end in '&', as background processes are
           not supported.  Should not be a command that expects input
           other than simple text.
         """
         if cmd.rstrip().endswith('&'):
             # this is *far* from a rigorous test
             # We do not support backgrounding processes because we either use
             # pexpect or pipes to read from.  Users can always just call
             # os.system() or use ip.system=ip.system_raw
             # if they really want a background process.
             raise OSError("Background processes not supported.")
 
         # we explicitly do NOT return the subprocess status code, because
         # a non-None value would trigger :func:`sys.displayhook` calls.
         # Instead, we store the exit_code in user_ns.
         self.user_ns['_exit_code'] = system(self.var_expand(cmd, depth=1))
 
     def system_raw(self, cmd):
         """Call the given cmd in a subprocess using os.system on Windows or
         subprocess.call using the system shell on other platforms.
 
         Parameters
         ----------
         cmd : str
           Command to execute.
         """
         cmd = self.var_expand(cmd, depth=1)
         # protect os.system from UNC paths on Windows, which it can't handle:
         if sys.platform == 'win32':
             from IPython.utils._process_win32 import AvoidUNCPath
             with AvoidUNCPath() as path:
                 if path is not None:
                     cmd = '"pushd %s &&"%s' % (path, cmd)
                 try:
                     ec = os.system(cmd)
                 except KeyboardInterrupt:
                     print('\n' + self.get_exception_only(), file=sys.stderr)
                     ec = -2
         else:
             # For posix the result of the subprocess.call() below is an exit
             # code, which by convention is zero for success, positive for
             # program failure.  Exit codes above 128 are reserved for signals,
             # and the formula for converting a signal to an exit code is usually
             # signal_number+128.  To more easily differentiate between exit
             # codes and signals, ipython uses negative numbers.  For instance
             # since control-c is signal 2 but exit code 130, ipython's
             # _exit_code variable will read -2.  Note that some shells like
             # csh and fish don't follow sh/bash conventions for exit codes.
             executable = os.environ.get('SHELL', None)
             try:
                 # Use env shell instead of default /bin/sh
                 ec = subprocess.call(cmd, shell=True, executable=executable)
             except KeyboardInterrupt:
                 # intercept control-C; a long traceback is not useful here
                 print('\n' + self.get_exception_only(), file=sys.stderr)
                 ec = 130
             if ec > 128:
                 ec = -(ec - 128)
         
         # We explicitly do NOT return the subprocess status code, because
         # a non-None value would trigger :func:`sys.displayhook` calls.
         # Instead, we store the exit_code in user_ns.  Note the semantics
         # of _exit_code: for control-c, _exit_code == -signal.SIGNIT,
         # but raising SystemExit(_exit_code) will give status 254!
         self.user_ns['_exit_code'] = ec
 
     # use piped system by default, because it is better behaved
     system = system_piped
 
     def getoutput(self, cmd, split=True, depth=0):
         """Get output (possibly including stderr) from a subprocess.
 
         Parameters
         ----------
         cmd : str
           Command to execute (can not end in '&', as background processes are
           not supported.
         split : bool, optional
           If True, split the output into an IPython SList.  Otherwise, an
           IPython LSString is returned.  These are objects similar to normal
           lists and strings, with a few convenience attributes for easier
           manipulation of line-based output.  You can use '?' on them for
           details.
         depth : int, optional
           How many frames above the caller are the local variables which should
           be expanded in the command string? The default (0) assumes that the
           expansion variables are in the stack frame calling this function.
         """
         if cmd.rstrip().endswith('&'):
             # this is *far* from a rigorous test
             raise OSError("Background processes not supported.")
         out = getoutput(self.var_expand(cmd, depth=depth+1))
         if split:
             out = SList(out.splitlines())
         else:
             out = LSString(out)
         return out
 
     #-------------------------------------------------------------------------
     # Things related to aliases
     #-------------------------------------------------------------------------
 
     def init_alias(self):
         self.alias_manager = AliasManager(shell=self, parent=self)
         self.configurables.append(self.alias_manager)
 
     #-------------------------------------------------------------------------
     # Things related to extensions
     #-------------------------------------------------------------------------
 
     def init_extension_manager(self):
         self.extension_manager = ExtensionManager(shell=self, parent=self)
         self.configurables.append(self.extension_manager)
 
     #-------------------------------------------------------------------------
     # Things related to payloads
     #-------------------------------------------------------------------------
 
     def init_payload(self):
         self.payload_manager = PayloadManager(parent=self)
         self.configurables.append(self.payload_manager)
     
     #-------------------------------------------------------------------------
     # Things related to the prefilter
     #-------------------------------------------------------------------------
 
     def init_prefilter(self):
         self.prefilter_manager = PrefilterManager(shell=self, parent=self)
         self.configurables.append(self.prefilter_manager)
         # Ultimately this will be refactored in the new interpreter code, but
         # for now, we should expose the main prefilter method (there's legacy
         # code out there that may rely on this).
         self.prefilter = self.prefilter_manager.prefilter_lines
 
     def auto_rewrite_input(self, cmd):
         """Print to the screen the rewritten form of the user's command.
 
         This shows visual feedback by rewriting input lines that cause
         automatic calling to kick in, like::
 
           /f x
 
         into::
 
           ------> f(x)
 
         after the user's input prompt.  This helps the user understand that the
         input line was transformed automatically by IPython.
         """
         if not self.show_rewritten_input:
             return
 
         # This is overridden in TerminalInteractiveShell to use fancy prompts
         print("------> " + cmd)
 
     #-------------------------------------------------------------------------
     # Things related to extracting values/expressions from kernel and user_ns
     #-------------------------------------------------------------------------
 
     def _user_obj_error(self):
         """return simple exception dict
         
         for use in user_expressions
         """
         
         etype, evalue, tb = self._get_exc_info()
         stb = self.InteractiveTB.get_exception_only(etype, evalue)
         
         exc_info = {
             u'status' : 'error',
             u'traceback' : stb,
             u'ename' : etype.__name__,
             u'evalue' : py3compat.safe_unicode(evalue),
         }
 
         return exc_info
     
     def _format_user_obj(self, obj):
         """format a user object to display dict
         
         for use in user_expressions
         """
         
         data, md = self.display_formatter.format(obj)
         value = {
             'status' : 'ok',
             'data' : data,
             'metadata' : md,
         }
         return value
     
     def user_expressions(self, expressions):
         """Evaluate a dict of expressions in the user's namespace.
 
         Parameters
         ----------
         expressions : dict
           A dict with string keys and string values.  The expression values
           should be valid Python expressions, each of which will be evaluated
           in the user namespace.
 
         Returns
         -------
         A dict, keyed like the input expressions dict, with the rich mime-typed
         display_data of each value.
         """
         out = {}
         user_ns = self.user_ns
         global_ns = self.user_global_ns
         
         for key, expr in expressions.items():
             try:
                 value = self._format_user_obj(eval(expr, global_ns, user_ns))
             except:
                 value = self._user_obj_error()
             out[key] = value
         return out
 
     #-------------------------------------------------------------------------
     # Things related to the running of code
     #-------------------------------------------------------------------------
 
     def ex(self, cmd):
         """Execute a normal python statement in user namespace."""
         with self.builtin_trap:
             exec(cmd, self.user_global_ns, self.user_ns)
 
     def ev(self, expr):
         """Evaluate python expression expr in user namespace.
 
         Returns the result of evaluation
         """
         with self.builtin_trap:
             return eval(expr, self.user_global_ns, self.user_ns)
 
     def safe_execfile(self, fname, *where, exit_ignore=False, raise_exceptions=False, shell_futures=False):
         """A safe version of the builtin execfile().
 
         This version will never throw an exception, but instead print
         helpful error messages to the screen.  This only works on pure
         Python files with the .py extension.
 
         Parameters
         ----------
         fname : string
             The name of the file to be executed.
         where : tuple
             One or two namespaces, passed to execfile() as (globals,locals).
             If only one is given, it is passed as both.
         exit_ignore : bool (False)
             If True, then silence SystemExit for non-zero status (it is always
             silenced for zero status, as it is so common).
         raise_exceptions : bool (False)
             If True raise exceptions everywhere. Meant for testing.
         shell_futures : bool (False)
             If True, the code will share future statements with the interactive
             shell. It will both be affected by previous __future__ imports, and
             any __future__ imports in the code will affect the shell. If False,
             __future__ imports are not shared in either direction.
 
         """
         fname = os.path.abspath(os.path.expanduser(fname))
 
         # Make sure we can open the file
         try:
             with open(fname):
                 pass
         except:
             warn('Could not open file <%s> for safe execution.' % fname)
             return
 
         # Find things also in current directory.  This is needed to mimic the
         # behavior of running a script from the system command line, where
         # Python inserts the script's directory into sys.path
         dname = os.path.dirname(fname)
 
         with prepended_to_syspath(dname), self.builtin_trap:
             try:
                 glob, loc = (where + (None, ))[:2]
                 py3compat.execfile(
                     fname, glob, loc,
                     self.compile if shell_futures else None)
             except SystemExit as status:
                 # If the call was made with 0 or None exit status (sys.exit(0)
                 # or sys.exit() ), don't bother showing a traceback, as both of
                 # these are considered normal by the OS:
                 # > python -c'import sys;sys.exit(0)'; echo $?
                 # 0
                 # > python -c'import sys;sys.exit()'; echo $?
                 # 0
                 # For other exit status, we show the exception unless
                 # explicitly silenced, but only in short form.
                 if status.code:
                     if raise_exceptions:
                         raise
                     if not exit_ignore:
                         self.showtraceback(exception_only=True)
             except:
                 if raise_exceptions:
                     raise
                 # tb offset is 2 because we wrap execfile
                 self.showtraceback(tb_offset=2)
 
     def safe_execfile_ipy(self, fname, shell_futures=False, raise_exceptions=False):
         """Like safe_execfile, but for .ipy or .ipynb files with IPython syntax.
 
         Parameters
         ----------
         fname : str
             The name of the file to execute.  The filename must have a
             .ipy or .ipynb extension.
         shell_futures : bool (False)
             If True, the code will share future statements with the interactive
             shell. It will both be affected by previous __future__ imports, and
             any __future__ imports in the code will affect the shell. If False,
             __future__ imports are not shared in either direction.
         raise_exceptions : bool (False)
             If True raise exceptions everywhere.  Meant for testing.
         """
         fname = os.path.abspath(os.path.expanduser(fname))
 
         # Make sure we can open the file
         try:
             with open(fname):
                 pass
         except:
             warn('Could not open file <%s> for safe execution.' % fname)
             return
 
         # Find things also in current directory.  This is needed to mimic the
         # behavior of running a script from the system command line, where
         # Python inserts the script's directory into sys.path
         dname = os.path.dirname(fname)
         
         def get_cells():
             """generator for sequence of code blocks to run"""
             if fname.endswith('.ipynb'):
                 from nbformat import read
                 nb = read(fname, as_version=4)
                 if not nb.cells:
                     return
                 for cell in nb.cells:
                     if cell.cell_type == 'code':
                         yield cell.source
             else:
                 with open(fname) as f:
                     yield f.read()
 
         with prepended_to_syspath(dname):
             try:
                 for cell in get_cells():
                     result = self.run_cell(cell, silent=True, shell_futures=shell_futures)
                     if raise_exceptions:
                         result.raise_error()
                     elif not result.success:
                         break
             except:
                 if raise_exceptions:
                     raise
                 self.showtraceback()
                 warn('Unknown failure executing file: <%s>' % fname)
 
     def safe_run_module(self, mod_name, where):
         """A safe version of runpy.run_module().
 
         This version will never throw an exception, but instead print
         helpful error messages to the screen.
 
         `SystemExit` exceptions with status code 0 or None are ignored.
 
         Parameters
         ----------
         mod_name : string
             The name of the module to be executed.
         where : dict
             The globals namespace.
         """
         try:
             try:
                 where.update(
                     runpy.run_module(str(mod_name), run_name="__main__",
                                      alter_sys=True)
                             )
             except SystemExit as status:
                 if status.code:
                     raise
         except:
             self.showtraceback()
             warn('Unknown failure executing module: <%s>' % mod_name)
 
     def run_cell(self, raw_cell, store_history=False, silent=False, shell_futures=True):
         """Run a complete IPython cell.
 
         Parameters
         ----------
         raw_cell : str
           The code (including IPython code such as %magic functions) to run.
         store_history : bool
           If True, the raw and translated cell will be stored in IPython's
           history. For user code calling back into IPython's machinery, this
           should be set to False.
         silent : bool
           If True, avoid side-effects, such as implicit displayhooks and
           and logging.  silent=True forces store_history=False.
         shell_futures : bool
           If True, the code will share future statements with the interactive
           shell. It will both be affected by previous __future__ imports, and
           any __future__ imports in the code will affect the shell. If False,
           __future__ imports are not shared in either direction.
 
         Returns
         -------
         result : :class:`ExecutionResult`
         """
         result = None
         try:
             result = self._run_cell(
                 raw_cell, store_history, silent, shell_futures)
         finally:
             self.events.trigger('post_execute')
             if not silent:
                 self.events.trigger('post_run_cell', result)
         return result
 
     def _run_cell(self, raw_cell:str, store_history:bool, silent:bool, shell_futures:bool):
         """Internal method to run a complete IPython cell."""
         coro = self.run_cell_async(
             raw_cell,
             store_history=store_history,
             silent=silent,
             shell_futures=shell_futures,
         )
 
         # run_cell_async is async, but may not actually need an eventloop.
         # when this is the case, we want to run it using the pseudo_sync_runner
         # so that code can invoke eventloops (for example via the %run , and
         # `%paste` magic.
         if self.should_run_async(raw_cell):
             runner = self.loop_runner
         else:
             runner = _pseudo_sync_runner
 
         try:
             return runner(coro)
         except BaseException as e:
             info = ExecutionInfo(raw_cell, store_history, silent, shell_futures)
             result = ExecutionResult(info)
             result.error_in_exec = e
             self.showtraceback(running_compiled_code=True)
             return result
         return
 
     def should_run_async(self, raw_cell: str) -> bool:
         """Return whether a cell should be run asynchronously via a coroutine runner
 
         Parameters
         ----------
         raw_cell: str
             The code to be executed
 
         Returns
         -------
         result: bool
             Whether the code needs to be run with a coroutine runner or not
 
         .. versionadded: 7.0
         """
         if not self.autoawait:
             return False
         try:
             cell = self.transform_cell(raw_cell)
         except Exception:
             # any exception during transform will be raised
             # prior to execution
             return False
         return _should_be_async(cell)
 
     async def run_cell_async(self, raw_cell: str, store_history=False, silent=False, shell_futures=True) -> ExecutionResult:
         """Run a complete IPython cell asynchronously.
 
         Parameters
         ----------
         raw_cell : str
           The code (including IPython code such as %magic functions) to run.
         store_history : bool
           If True, the raw and translated cell will be stored in IPython's
           history. For user code calling back into IPython's machinery, this
           should be set to False.
         silent : bool
           If True, avoid side-effects, such as implicit displayhooks and
           and logging.  silent=True forces store_history=False.
         shell_futures : bool
           If True, the code will share future statements with the interactive
           shell. It will both be affected by previous __future__ imports, and
           any __future__ imports in the code will affect the shell. If False,
           __future__ imports are not shared in either direction.
 
         Returns
         -------
         result : :class:`ExecutionResult`
 
         .. versionadded: 7.0
         """
         info = ExecutionInfo(
             raw_cell, store_history, silent, shell_futures)
         result = ExecutionResult(info)
 
         if (not raw_cell) or raw_cell.isspace():
             self.last_execution_succeeded = True
             self.last_execution_result = result
             return result
 
         if silent:
             store_history = False
 
         if store_history:
             result.execution_count = self.execution_count
 
         def error_before_exec(value):
             if store_history:
                 self.execution_count += 1
             result.error_before_exec = value
             self.last_execution_succeeded = False
             self.last_execution_result = result
             return result
 
         self.events.trigger('pre_execute')
         if not silent:
             self.events.trigger('pre_run_cell', info)
 
         # If any of our input transformation (input_transformer_manager or
         # prefilter_manager) raises an exception, we store it in this variable
         # so that we can display the error after logging the input and storing
         # it in the history.
         try:
             cell = self.transform_cell(raw_cell)
         except Exception:
             preprocessing_exc_tuple = sys.exc_info()
             cell = raw_cell  # cell has to exist so it can be stored/logged
         else:
             preprocessing_exc_tuple = None
 
         # Store raw and processed history
         if store_history:
             self.history_manager.store_inputs(self.execution_count,
                                               cell, raw_cell)
         if not silent:
             self.logger.log(cell, raw_cell)
 
         # Display the exception if input processing failed.
         if preprocessing_exc_tuple is not None:
             self.showtraceback(preprocessing_exc_tuple)
             if store_history:
                 self.execution_count += 1
             return error_before_exec(preprocessing_exc_tuple[1])
 
         # Our own compiler remembers the __future__ environment. If we want to
         # run code with a separate __future__ environment, use the default
         # compiler
         compiler = self.compile if shell_futures else CachingCompiler()
 
         _run_async = False
 
         with self.builtin_trap:
             cell_name = self.compile.cache(cell, self.execution_count)
 
             with self.display_trap:
                 # Compile to bytecode
                 try:
                     if sys.version_info < (3,8) and self.autoawait:
                         if _should_be_async(cell):
                             # the code AST below will not be user code: we wrap it
                             # in an `async def`. This will likely make some AST
                             # transformer below miss some transform opportunity and
                             # introduce a small coupling to run_code (in which we
                             # bake some assumptions of what _ast_asyncify returns.
                             # they are ways around (like grafting part of the ast
                             # later:
                             #    - Here, return code_ast.body[0].body[1:-1], as well
                             #    as last expression in  return statement which is
                             #    the user code part.
                             #    - Let it go through the AST transformers, and graft
                             #    - it back after the AST transform
                             # But that seem unreasonable, at least while we
                             # do not need it.
                             code_ast = _ast_asyncify(cell, 'async-def-wrapper')
                             _run_async = True
                         else:
                             code_ast = compiler.ast_parse(cell, filename=cell_name)
                     else:
                         code_ast = compiler.ast_parse(cell, filename=cell_name)
                 except self.custom_exceptions as e:
                     etype, value, tb = sys.exc_info()
                     self.CustomTB(etype, value, tb)
                     return error_before_exec(e)
                 except IndentationError as e:
                     self.showindentationerror()
                     return error_before_exec(e)
                 except (OverflowError, SyntaxError, ValueError, TypeError,
                         MemoryError) as e:
                     self.showsyntaxerror()
                     return error_before_exec(e)
 
                 # Apply AST transformations
                 try:
                     code_ast = self.transform_ast(code_ast)
                 except InputRejected as e:
                     self.showtraceback()
                     return error_before_exec(e)
 
                 # Give the displayhook a reference to our ExecutionResult so it
                 # can fill in the output value.
                 self.displayhook.exec_result = result
 
                 # Execute the user code
                 interactivity = "none" if silent else self.ast_node_interactivity
                 if _run_async:
                     interactivity = 'async'
 
                 has_raised = await self.run_ast_nodes(code_ast.body, cell_name,
                        interactivity=interactivity, compiler=compiler, result=result)
 
                 self.last_execution_succeeded = not has_raised
                 self.last_execution_result = result
 
                 # Reset this so later displayed values do not modify the
                 # ExecutionResult
                 self.displayhook.exec_result = None
 
         if store_history:
             # Write output to the database. Does nothing unless
             # history output logging is enabled.
             self.history_manager.store_output(self.execution_count)
             # Each cell is a *single* input, regardless of how many lines it has
             self.execution_count += 1
 
         return result
 
     def transform_cell(self, raw_cell):
         """Transform an input cell before parsing it.
 
         Static transformations, implemented in IPython.core.inputtransformer2,
         deal with things like ``%magic`` and ``!system`` commands.
         These run on all input.
         Dynamic transformations, for things like unescaped magics and the exit
         autocall, depend on the state of the interpreter.
         These only apply to single line inputs.
 
         These string-based transformations are followed by AST transformations;
         see :meth:`transform_ast`.
         """
         # Static input transformations
         cell = self.input_transformer_manager.transform_cell(raw_cell)
 
         if len(cell.splitlines()) == 1:
             # Dynamic transformations - only applied for single line commands
             with self.builtin_trap:
                 # use prefilter_lines to handle trailing newlines
                 # restore trailing newline for ast.parse
                 cell = self.prefilter_manager.prefilter_lines(cell) + '\n'
 
         lines = cell.splitlines(keepends=True)
         for transform in self.input_transformers_post:
             lines = transform(lines)
         cell = ''.join(lines)
 
         return cell
 
     def transform_ast(self, node):
         """Apply the AST transformations from self.ast_transformers
         
         Parameters
         ----------
         node : ast.Node
           The root node to be transformed. Typically called with the ast.Module
           produced by parsing user input.
         
         Returns
         -------
         An ast.Node corresponding to the node it was called with. Note that it
         may also modify the passed object, so don't rely on references to the
         original AST.
         """
         for transformer in self.ast_transformers:
             try:
                 node = transformer.visit(node)
             except InputRejected:
                 # User-supplied AST transformers can reject an input by raising
                 # an InputRejected.  Short-circuit in this case so that we
                 # don't unregister the transform.
                 raise
             except Exception:
                 warn("AST transformer %r threw an error. It will be unregistered." % transformer)
                 self.ast_transformers.remove(transformer)
 
         if self.ast_transformers:
             ast.fix_missing_locations(node)
         return node
 
     async def run_ast_nodes(self, nodelist:ListType[AST], cell_name:str, interactivity='last_expr',
                         compiler=compile, result=None):
         """Run a sequence of AST nodes. The execution mode depends on the
         interactivity parameter.
 
         Parameters
         ----------
         nodelist : list
           A sequence of AST nodes to run.
         cell_name : str
           Will be passed to the compiler as the filename of the cell. Typically
           the value returned by ip.compile.cache(cell).
         interactivity : str
           'all', 'last', 'last_expr' , 'last_expr_or_assign' or 'none',
           specifying which nodes should be run interactively (displaying output
           from expressions). 'last_expr' will run the last node interactively
           only if it is an expression (i.e. expressions in loops or other blocks
           are not displayed) 'last_expr_or_assign' will run the last expression
           or the last assignment. Other values for this parameter will raise a
           ValueError.
 
           Experimental value: 'async' Will try to run top level interactive
           async/await code in default runner, this will not respect the
           interactivity setting and will only run the last node if it is an
           expression. 
 
         compiler : callable
           A function with the same interface as the built-in compile(), to turn
           the AST nodes into code objects. Default is the built-in compile().
         result : ExecutionResult, optional
           An object to store exceptions that occur during execution.
 
         Returns
         -------
         True if an exception occurred while running code, False if it finished
         running.
         """
         if not nodelist:
             return
 
         if interactivity == 'last_expr_or_assign':
             if isinstance(nodelist[-1], _assign_nodes):
                 asg = nodelist[-1]
                 if isinstance(asg, ast.Assign) and len(asg.targets) == 1:
                     target = asg.targets[0]
                 elif isinstance(asg, _single_targets_nodes):
                     target = asg.target
                 else:
                     target = None
                 if isinstance(target, ast.Name):
                     nnode = ast.Expr(ast.Name(target.id, ast.Load()))
                     ast.fix_missing_locations(nnode)
                     nodelist.append(nnode)
             interactivity = 'last_expr'
 
         _async = False
         if interactivity == 'last_expr':
             if isinstance(nodelist[-1], ast.Expr):
                 interactivity = "last"
             else:
                 interactivity = "none"
 
         if interactivity == 'none':
             to_run_exec, to_run_interactive = nodelist, []
         elif interactivity == 'last':
             to_run_exec, to_run_interactive = nodelist[:-1], nodelist[-1:]
         elif interactivity == 'all':
             to_run_exec, to_run_interactive = [], nodelist
         elif interactivity == 'async':
             to_run_exec, to_run_interactive = [], nodelist
             _async = True
         else:
             raise ValueError("Interactivity was %r" % interactivity)
 
         try:
             if _async and sys.version_info > (3,8):
                 raise ValueError("This branch should never happen on Python 3.8 and above, "
                                  "please try to upgrade IPython and open a bug report with your case.")
             if _async:
                 # If interactivity is async the semantics of run_code are
                 # completely different Skip usual machinery.
                 mod = Module(nodelist, [])
                 async_wrapper_code = compiler(mod, cell_name, 'exec')
                 exec(async_wrapper_code, self.user_global_ns, self.user_ns)
                 async_code = removed_co_newlocals(self.user_ns.pop('async-def-wrapper')).__code__
                 if (await self.run_code(async_code, result, async_=True)):
                     return True
             else:
                 if sys.version_info > (3, 8):
                     def compare(code):
                         is_async = (inspect.CO_COROUTINE & code.co_flags == inspect.CO_COROUTINE)
                         return is_async
                 else:
                     def compare(code):
                         return _async
 
                 # refactor that to just change the mod constructor.
                 to_run = []
                 for node in to_run_exec:
                     to_run.append((node, 'exec'))
 
                 for node in to_run_interactive:
                     to_run.append((node, 'single'))
 
                 for node,mode in to_run:
                     if mode == 'exec':
                         mod = Module([node], [])
                     elif mode == 'single':
                         mod = ast.Interactive([node])
                     with compiler.extra_flags(getattr(ast, 'PyCF_ALLOW_TOP_LEVEL_AWAIT', 0x0) if self.autoawait else 0x0):
                         code = compiler(mod, cell_name, mode)
                         asy = compare(code)
                     if (await self.run_code(code, result,  async_=asy)):
                         return True
 
             # Flush softspace
             if softspace(sys.stdout, 0):
                 print()
 
         except:
             # It's possible to have exceptions raised here, typically by
             # compilation of odd code (such as a naked 'return' outside a
             # function) that did parse but isn't valid. Typically the exception
             # is a SyntaxError, but it's safest just to catch anything and show
             # the user a traceback.
 
             # We do only one try/except outside the loop to minimize the impact
             # on runtime, and also because if any node in the node list is
             # broken, we should stop execution completely.
             if result:
                 result.error_before_exec = sys.exc_info()[1]
             self.showtraceback()
             return True
 
         return False
 
     def _async_exec(self, code_obj: types.CodeType, user_ns: dict):
         """
         Evaluate an asynchronous code object using a code runner
 
         Fake asynchronous execution of code_object in a namespace via a proxy namespace.
 
         Returns coroutine object, which can be executed via async loop runner
 
         WARNING: The semantics of `async_exec` are quite different from `exec`,
         in particular you can only pass a single namespace. It also return a
         handle to the value of the last things returned by code_object.
         """
 
         return eval(code_obj, user_ns)
 
     async def run_code(self, code_obj, result=None, *, async_=False):
         """Execute a code object.
 
         When an exception occurs, self.showtraceback() is called to display a
         traceback.
 
         Parameters
         ----------
         code_obj : code object
           A compiled code object, to be executed
         result : ExecutionResult, optional
           An object to store exceptions that occur during execution.
         async_ :  Bool (Experimental)
           Attempt to run top-level asynchronous code in a default loop.
 
         Returns
         -------
         False : successful execution.
         True : an error occurred.
         """
         # Set our own excepthook in case the user code tries to call it
         # directly, so that the IPython crash handler doesn't get triggered
         old_excepthook, sys.excepthook = sys.excepthook, self.excepthook
 
         # we save the original sys.excepthook in the instance, in case config
         # code (such as magics) needs access to it.
         self.sys_excepthook = old_excepthook
         outflag = True  # happens in more places, so it's easier as default
         try:
             try:
                 self.hooks.pre_run_code_hook()
                 if async_ and sys.version_info < (3,8):
                     last_expr = (await self._async_exec(code_obj, self.user_ns))
                     code = compile('last_expr', 'fake', "single")
                     exec(code, {'last_expr': last_expr})
                 elif async_ :
                     await eval(code_obj, self.user_global_ns, self.user_ns)
                 else:
                     exec(code_obj, self.user_global_ns, self.user_ns)
             finally:
                 # Reset our crash handler in place
                 sys.excepthook = old_excepthook
         except SystemExit as e:
             if result is not None:
                 result.error_in_exec = e
             self.showtraceback(exception_only=True)
             warn("To exit: use 'exit', 'quit', or Ctrl-D.", stacklevel=1)
         except self.custom_exceptions:
             etype, value, tb = sys.exc_info()
             if result is not None:
                 result.error_in_exec = value
             self.CustomTB(etype, value, tb)
         except:
             if result is not None:
                 result.error_in_exec = sys.exc_info()[1]
             self.showtraceback(running_compiled_code=True)
         else:
             outflag = False
         return outflag
 
     # For backwards compatibility
     runcode = run_code
 
     def check_complete(self, code: str) -> Tuple[str, str]:
         """Return whether a block of code is ready to execute, or should be continued
 
         Parameters
         ----------
         source : string
           Python input code, which can be multiline.
 
         Returns
         -------
         status : str
           One of 'complete', 'incomplete', or 'invalid' if source is not a
           prefix of valid code.
         indent : str
           When status is 'incomplete', this is some whitespace to insert on
           the next line of the prompt.
         """
         status, nspaces = self.input_transformer_manager.check_complete(code)
         return status, ' ' * (nspaces or 0)
 
     #-------------------------------------------------------------------------
     # Things related to GUI support and pylab
     #-------------------------------------------------------------------------
 
     active_eventloop = None
 
     def enable_gui(self, gui=None):
         raise NotImplementedError('Implement enable_gui in a subclass')
     
     def enable_matplotlib(self, gui=None):
         """Enable interactive matplotlib and inline figure support.
         
         This takes the following steps:
         
         1. select the appropriate eventloop and matplotlib backend
         2. set up matplotlib for interactive use with that backend
         3. configure formatters for inline figure display
         4. enable the selected gui eventloop
         
         Parameters
         ----------
         gui : optional, string
           If given, dictates the choice of matplotlib GUI backend to use
           (should be one of IPython's supported backends, 'qt', 'osx', 'tk',
           'gtk', 'wx' or 'inline'), otherwise we use the default chosen by
           matplotlib (as dictated by the matplotlib build-time options plus the
           user's matplotlibrc configuration file).  Note that not all backends
           make sense in all contexts, for example a terminal ipython can't
           display figures inline.
         """
         from IPython.core import pylabtools as pt
         gui, backend = pt.find_gui_and_backend(gui, self.pylab_gui_select)
     
         if gui != 'inline':
             # If we have our first gui selection, store it
             if self.pylab_gui_select is None:
                 self.pylab_gui_select = gui
             # Otherwise if they are different
             elif gui != self.pylab_gui_select:
                 print('Warning: Cannot change to a different GUI toolkit: %s.'
                         ' Using %s instead.' % (gui, self.pylab_gui_select))
                 gui, backend = pt.find_gui_and_backend(self.pylab_gui_select)
         
         pt.activate_matplotlib(backend)
         pt.configure_inline_support(self, backend)
         
         # Now we must activate the gui pylab wants to use, and fix %run to take
         # plot updates into account
         self.enable_gui(gui)
         self.magics_manager.registry['ExecutionMagics'].default_runner = \
             pt.mpl_runner(self.safe_execfile)
         
         return gui, backend
 
     def enable_pylab(self, gui=None, import_all=True, welcome_message=False):
         """Activate pylab support at runtime.
 
         This turns on support for matplotlib, preloads into the interactive
         namespace all of numpy and pylab, and configures IPython to correctly
         interact with the GUI event loop.  The GUI backend to be used can be
         optionally selected with the optional ``gui`` argument.
         
         This method only adds preloading the namespace to InteractiveShell.enable_matplotlib.
 
         Parameters
         ----------
         gui : optional, string
           If given, dictates the choice of matplotlib GUI backend to use
           (should be one of IPython's supported backends, 'qt', 'osx', 'tk',
           'gtk', 'wx' or 'inline'), otherwise we use the default chosen by
           matplotlib (as dictated by the matplotlib build-time options plus the
           user's matplotlibrc configuration file).  Note that not all backends
           make sense in all contexts, for example a terminal ipython can't
           display figures inline.
         import_all : optional, bool, default: True
           Whether to do `from numpy import *` and `from pylab import *`
           in addition to module imports.
         welcome_message : deprecated
           This argument is ignored, no welcome message will be displayed.
         """
         from IPython.core.pylabtools import import_pylab
         
         gui, backend = self.enable_matplotlib(gui)
         
         # We want to prevent the loading of pylab to pollute the user's
         # namespace as shown by the %who* magics, so we execute the activation
         # code in an empty namespace, and we update *both* user_ns and
         # user_ns_hidden with this information.
         ns = {}
         import_pylab(ns, import_all)
         # warn about clobbered names
         ignored = {"__builtins__"}
         both = set(ns).intersection(self.user_ns).difference(ignored)
         clobbered = [ name for name in both if self.user_ns[name] is not ns[name] ]
         self.user_ns.update(ns)
         self.user_ns_hidden.update(ns)
         return gui, backend, clobbered
 
     #-------------------------------------------------------------------------
     # Utilities
     #-------------------------------------------------------------------------
 
     def var_expand(self, cmd, depth=0, formatter=DollarFormatter()):
         """Expand python variables in a string.
 
         The depth argument indicates how many frames above the caller should
         be walked to look for the local namespace where to expand variables.
 
         The global namespace for expansion is always the user's interactive
         namespace.
         """
         ns = self.user_ns.copy()
         try:
             frame = sys._getframe(depth+1)
         except ValueError:
             # This is thrown if there aren't that many frames on the stack,
             # e.g. if a script called run_line_magic() directly.
             pass
         else:
             ns.update(frame.f_locals)
 
         try:
             # We have to use .vformat() here, because 'self' is a valid and common
             # name, and expanding **ns for .format() would make it collide with
             # the 'self' argument of the method.
             cmd = formatter.vformat(cmd, args=[], kwargs=ns)
         except Exception:
             # if formatter couldn't format, just let it go untransformed
             pass
         return cmd
 
     def mktempfile(self, data=None, prefix='ipython_edit_'):
         """Make a new tempfile and return its filename.
 
         This makes a call to tempfile.mkstemp (created in a tempfile.mkdtemp),
         but it registers the created filename internally so ipython cleans it up
         at exit time.
 
         Optional inputs:
 
           - data(None): if data is given, it gets written out to the temp file
             immediately, and the file is closed again."""
 
         dirname = tempfile.mkdtemp(prefix=prefix)
         self.tempdirs.append(dirname)
 
         handle, filename = tempfile.mkstemp('.py', prefix, dir=dirname)
         os.close(handle)  # On Windows, there can only be one open handle on a file
         self.tempfiles.append(filename)
 
         if data:
             with open(filename, 'w') as tmp_file:
                 tmp_file.write(data)
         return filename
 
     @undoc
     def write(self,data):
         """DEPRECATED: Write a string to the default output"""
         warn('InteractiveShell.write() is deprecated, use sys.stdout instead',
              DeprecationWarning, stacklevel=2)
         sys.stdout.write(data)
 
     @undoc
     def write_err(self,data):
         """DEPRECATED: Write a string to the default error output"""
         warn('InteractiveShell.write_err() is deprecated, use sys.stderr instead',
              DeprecationWarning, stacklevel=2)
         sys.stderr.write(data)
 
     def ask_yes_no(self, prompt, default=None, interrupt=None):
         if self.quiet:
             return True
         return ask_yes_no(prompt,default,interrupt)
 
     def show_usage(self):
         """Show a usage message"""
         page.page(IPython.core.usage.interactive_usage)
 
     def extract_input_lines(self, range_str, raw=False):
         """Return as a string a set of input history slices.
 
         Parameters
         ----------
         range_str : string
             The set of slices is given as a string, like "~5/6-~4/2 4:8 9",
             since this function is for use by magic functions which get their
             arguments as strings. The number before the / is the session
             number: ~n goes n back from the current session.
 
         raw : bool, optional
             By default, the processed input is used.  If this is true, the raw
             input history is used instead.
 
         Notes
         -----
 
         Slices can be described with two notations:
 
         * ``N:M`` -> standard python form, means including items N...(M-1).
         * ``N-M`` -> include items N..M (closed endpoint).
         """
         lines = self.history_manager.get_range_by_str(range_str, raw=raw)
         return "\n".join(x for _, _, x in lines)
 
     def find_user_code(self, target, raw=True, py_only=False, skip_encoding_cookie=True, search_ns=False):
         """Get a code string from history, file, url, or a string or macro.
 
         This is mainly used by magic functions.
 
         Parameters
         ----------
 
         target : str
 
           A string specifying code to retrieve. This will be tried respectively
           as: ranges of input history (see %history for syntax), url,
           corresponding .py file, filename, or an expression evaluating to a
           string or Macro in the user namespace.
 
         raw : bool
           If true (default), retrieve raw history. Has no effect on the other
           retrieval mechanisms.
 
         py_only : bool (default False)
           Only try to fetch python code, do not try alternative methods to decode file
           if unicode fails.
 
         Returns
         -------
         A string of code.
 
         ValueError is raised if nothing is found, and TypeError if it evaluates
         to an object of another type. In each case, .args[0] is a printable
         message.
         """
         code = self.extract_input_lines(target, raw=raw)  # Grab history
         if code:
             return code
         try:
             if target.startswith(('http://', 'https://')):
                 return openpy.read_py_url(target, skip_encoding_cookie=skip_encoding_cookie)
         except UnicodeDecodeError:
             if not py_only :
                 # Deferred import
                 from urllib.request import urlopen
                 response = urlopen(target)
                 return response.read().decode('latin1')
             raise ValueError(("'%s' seem to be unreadable.") % target)
 
         potential_target = [target]
         try :
             potential_target.insert(0,get_py_filename(target))
         except IOError:
             pass
 
         for tgt in potential_target :
             if os.path.isfile(tgt):                        # Read file
                 try :
                     return openpy.read_py_file(tgt, skip_encoding_cookie=skip_encoding_cookie)
                 except UnicodeDecodeError :
                     if not py_only :
                         with io_open(tgt,'r', encoding='latin1') as f :
                             return f.read()
                     raise ValueError(("'%s' seem to be unreadable.") % target)
             elif os.path.isdir(os.path.expanduser(tgt)):
                 raise ValueError("'%s' is a directory, not a regular file." % target)
 
         if search_ns:
             # Inspect namespace to load object source
             object_info = self.object_inspect(target, detail_level=1)
             if object_info['found'] and object_info['source']:
                 return object_info['source']
 
         try:                                              # User namespace
             codeobj = eval(target, self.user_ns)
         except Exception:
             raise ValueError(("'%s' was not found in history, as a file, url, "
                                 "nor in the user namespace.") % target)
 
         if isinstance(codeobj, str):
             return codeobj
         elif isinstance(codeobj, Macro):
             return codeobj.value
 
         raise TypeError("%s is neither a string nor a macro." % target,
                         codeobj)
 
     #-------------------------------------------------------------------------
     # Things related to IPython exiting
     #-------------------------------------------------------------------------
     def atexit_operations(self):
         """This will be executed at the time of exit.
 
         Cleanup operations and saving of persistent data that is done
         unconditionally by IPython should be performed here.
 
         For things that may depend on startup flags or platform specifics (such
         as having readline or not), register a separate atexit function in the
         code that has the appropriate information, rather than trying to
         clutter
         """
         # Close the history session (this stores the end time and line count)
         # this must be *before* the tempfile cleanup, in case of temporary
         # history db
         self.history_manager.end_session()
 
         # Cleanup all tempfiles and folders left around
         for tfile in self.tempfiles:
             try:
                 os.unlink(tfile)
             except OSError:
                 pass
 
         for tdir in self.tempdirs:
             try:
                 os.rmdir(tdir)
             except OSError:
                 pass
 
         # Clear all user namespaces to release all references cleanly.
         self.reset(new_session=False)
 
         # Run user hooks
         self.hooks.shutdown_hook()
 
     def cleanup(self):
         self.restore_sys_module_state()
 
 
     # Overridden in terminal subclass to change prompts
     def switch_doctest_mode(self, mode):
         pass
 
 
 class InteractiveShellABC(metaclass=abc.ABCMeta):
     """An abstract base class for InteractiveShell."""
 
 InteractiveShellABC.register(InteractiveShell)

IPython/core/magics/execution.py

@@ -1,1501 +1,1503 b''
 # -*- coding: utf-8 -*-
 """Implementation of execution-related magic functions."""
 
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
 
 import ast
 import bdb
 import builtins as builtin_mod
 import gc
 import itertools
 import os
 import shlex
 import sys
 import time
 import timeit
 import math
 import re
 from pdb import Restart
 
 # cProfile was added in Python2.5
 try:
     import cProfile as profile
     import pstats
 except ImportError:
     # profile isn't bundled by default in Debian for license reasons
     try:
         import profile, pstats
     except ImportError:
         profile = pstats = None
 
 from IPython.core import oinspect
 from IPython.core import magic_arguments
 from IPython.core import page
 from IPython.core.error import UsageError
 from IPython.core.macro import Macro
 from IPython.core.magic import (Magics, magics_class, line_magic, cell_magic,
                                 line_cell_magic, on_off, needs_local_scope,
                                 no_var_expand)
 from IPython.testing.skipdoctest import skip_doctest
 from IPython.utils.contexts import preserve_keys
 from IPython.utils.capture import capture_output
 from IPython.utils.ipstruct import Struct
 from IPython.utils.module_paths import find_mod
 from IPython.utils.path import get_py_filename, shellglob
 from IPython.utils.timing import clock, clock2
 from warnings import warn
 from logging import error
 from io import StringIO
 
 if sys.version_info > (3,8):
     from ast import Module
 else :
     # mock the new API, ignore second argument
     # see https://github.com/ipython/ipython/issues/11590
     from ast import Module as OriginalModule
     Module = lambda nodelist, type_ignores: OriginalModule(nodelist)
 
 
 #-----------------------------------------------------------------------------
 # Magic implementation classes
 #-----------------------------------------------------------------------------
 
 
 class TimeitResult(object):
     """
     Object returned by the timeit magic with info about the run.
 
     Contains the following attributes :
 
     loops: (int) number of loops done per measurement
     repeat: (int) number of times the measurement has been repeated
     best: (float) best execution time / number
     all_runs: (list of float) execution time of each run (in s)
     compile_time: (float) time of statement compilation (s)
 
     """
     def __init__(self, loops, repeat, best, worst, all_runs, compile_time, precision):
         self.loops = loops
         self.repeat = repeat
         self.best = best
         self.worst = worst
         self.all_runs = all_runs
         self.compile_time = compile_time
         self._precision = precision
         self.timings = [ dt / self.loops for dt in all_runs]
 
     @property
     def average(self):
         return math.fsum(self.timings) / len(self.timings)
 
     @property
     def stdev(self):
         mean = self.average
         return (math.fsum([(x - mean) ** 2 for x in self.timings]) / len(self.timings)) ** 0.5
 
     def __str__(self):
         pm = '+-'
         if hasattr(sys.stdout, 'encoding') and sys.stdout.encoding:
             try:
                 u'\xb1'.encode(sys.stdout.encoding)
                 pm = u'\xb1'
             except:
                 pass
         return (
             u"{mean} {pm} {std} per loop (mean {pm} std. dev. of {runs} run{run_plural}, {loops} loop{loop_plural} each)"
                 .format(
                     pm = pm,
                     runs = self.repeat,
                     loops = self.loops,
                     loop_plural = "" if self.loops == 1 else "s",
                     run_plural = "" if self.repeat == 1 else "s",
                     mean = _format_time(self.average, self._precision),
                     std = _format_time(self.stdev, self._precision))
                 )
 
     def _repr_pretty_(self, p , cycle):
         unic = self.__str__()
         p.text(u'<TimeitResult : '+unic+u'>')
 
 
 class TimeitTemplateFiller(ast.NodeTransformer):
     """Fill in the AST template for timing execution.
 
     This is quite closely tied to the template definition, which is in
     :meth:`ExecutionMagics.timeit`.
     """
     def __init__(self, ast_setup, ast_stmt):
         self.ast_setup = ast_setup
         self.ast_stmt = ast_stmt
 
     def visit_FunctionDef(self, node):
         "Fill in the setup statement"
         self.generic_visit(node)
         if node.name == "inner":
             node.body[:1] = self.ast_setup.body
 
         return node
 
     def visit_For(self, node):
         "Fill in the statement to be timed"
         if getattr(getattr(node.body[0], 'value', None), 'id', None) == 'stmt':
             node.body = self.ast_stmt.body
         return node
 
 
 class Timer(timeit.Timer):
     """Timer class that explicitly uses self.inner
     
     which is an undocumented implementation detail of CPython,
     not shared by PyPy.
     """
     # Timer.timeit copied from CPython 3.4.2
     def timeit(self, number=timeit.default_number):
         """Time 'number' executions of the main statement.
 
         To be precise, this executes the setup statement once, and
         then returns the time it takes to execute the main statement
         a number of times, as a float measured in seconds.  The
         argument is the number of times through the loop, defaulting
         to one million.  The main statement, the setup statement and
         the timer function to be used are passed to the constructor.
         """
         it = itertools.repeat(None, number)
         gcold = gc.isenabled()
         gc.disable()
         try:
             timing = self.inner(it, self.timer)
         finally:
             if gcold:
                 gc.enable()
         return timing
 
 
 @magics_class
 class ExecutionMagics(Magics):
     """Magics related to code execution, debugging, profiling, etc.
 
     """
 
     def __init__(self, shell):
         super(ExecutionMagics, self).__init__(shell)
         if profile is None:
             self.prun = self.profile_missing_notice
         # Default execution function used to actually run user code.
         self.default_runner = None
 
     def profile_missing_notice(self, *args, **kwargs):
         error("""\
 The profile module could not be found. It has been removed from the standard
 python packages because of its non-free license. To use profiling, install the
 python-profiler package from non-free.""")
 
     @skip_doctest
     @no_var_expand
     @line_cell_magic
     def prun(self, parameter_s='', cell=None):
 
         """Run a statement through the python code profiler.
 
         Usage, in line mode:
           %prun [options] statement
 
         Usage, in cell mode:
           %%prun [options] [statement]
           code...
           code...
 
         In cell mode, the additional code lines are appended to the (possibly
         empty) statement in the first line.  Cell mode allows you to easily
         profile multiline blocks without having to put them in a separate
         function.
 
         The given statement (which doesn't require quote marks) is run via the
         python profiler in a manner similar to the profile.run() function.
         Namespaces are internally managed to work correctly; profile.run
         cannot be used in IPython because it makes certain assumptions about
         namespaces which do not hold under IPython.
 
         Options:
 
         -l <limit>
           you can place restrictions on what or how much of the
           profile gets printed. The limit value can be:
 
              * A string: only information for function names containing this string
                is printed.
 
              * An integer: only these many lines are printed.
 
              * A float (between 0 and 1): this fraction of the report is printed
                (for example, use a limit of 0.4 to see the topmost 40% only).
 
           You can combine several limits with repeated use of the option. For
           example, ``-l __init__ -l 5`` will print only the topmost 5 lines of
           information about class constructors.
 
         -r
           return the pstats.Stats object generated by the profiling. This
           object has all the information about the profile in it, and you can
           later use it for further analysis or in other functions.
 
         -s <key>
           sort profile by given key. You can provide more than one key
           by using the option several times: '-s key1 -s key2 -s key3...'. The
           default sorting key is 'time'.
 
           The following is copied verbatim from the profile documentation
           referenced below:
 
           When more than one key is provided, additional keys are used as
           secondary criteria when the there is equality in all keys selected
           before them.
 
           Abbreviations can be used for any key names, as long as the
           abbreviation is unambiguous.  The following are the keys currently
           defined:
 
           ============  =====================
           Valid Arg     Meaning
           ============  =====================
           "calls"       call count
           "cumulative"  cumulative time
           "file"        file name
           "module"      file name
           "pcalls"      primitive call count
           "line"        line number
           "name"        function name
           "nfl"         name/file/line
           "stdname"     standard name
           "time"        internal time
           ============  =====================
 
           Note that all sorts on statistics are in descending order (placing
           most time consuming items first), where as name, file, and line number
           searches are in ascending order (i.e., alphabetical). The subtle
           distinction between "nfl" and "stdname" is that the standard name is a
           sort of the name as printed, which means that the embedded line
           numbers get compared in an odd way.  For example, lines 3, 20, and 40
           would (if the file names were the same) appear in the string order
           "20" "3" and "40".  In contrast, "nfl" does a numeric compare of the
           line numbers.  In fact, sort_stats("nfl") is the same as
           sort_stats("name", "file", "line").
 
         -T <filename>
           save profile results as shown on screen to a text
           file. The profile is still shown on screen.
 
         -D <filename>
           save (via dump_stats) profile statistics to given
           filename. This data is in a format understood by the pstats module, and
           is generated by a call to the dump_stats() method of profile
           objects. The profile is still shown on screen.
 
         -q
           suppress output to the pager.  Best used with -T and/or -D above.
 
         If you want to run complete programs under the profiler's control, use
         ``%run -p [prof_opts] filename.py [args to program]`` where prof_opts
         contains profiler specific options as described here.
 
         You can read the complete documentation for the profile module with::
 
           In [1]: import profile; profile.help()
 
         .. versionchanged:: 7.3
             User variables are no longer expanded,
             the magic line is always left unmodified.
 
         """
         opts, arg_str = self.parse_options(parameter_s, 'D:l:rs:T:q',
                                            list_all=True, posix=False)
         if cell is not None:
             arg_str += '\n' + cell
         arg_str = self.shell.transform_cell(arg_str)
         return self._run_with_profiler(arg_str, opts, self.shell.user_ns)
 
     def _run_with_profiler(self, code, opts, namespace):
         """
         Run `code` with profiler.  Used by ``%prun`` and ``%run -p``.
 
         Parameters
         ----------
         code : str
             Code to be executed.
         opts : Struct
             Options parsed by `self.parse_options`.
         namespace : dict
             A dictionary for Python namespace (e.g., `self.shell.user_ns`).
 
         """
 
         # Fill default values for unspecified options:
         opts.merge(Struct(D=[''], l=[], s=['time'], T=['']))
 
         prof = profile.Profile()
         try:
             prof = prof.runctx(code, namespace, namespace)
             sys_exit = ''
         except SystemExit:
             sys_exit = """*** SystemExit exception caught in code being profiled."""
 
         stats = pstats.Stats(prof).strip_dirs().sort_stats(*opts.s)
 
         lims = opts.l
         if lims:
             lims = []  # rebuild lims with ints/floats/strings
             for lim in opts.l:
                 try:
                     lims.append(int(lim))
                 except ValueError:
                     try:
                         lims.append(float(lim))
                     except ValueError:
                         lims.append(lim)
 
         # Trap output.
         stdout_trap = StringIO()
         stats_stream = stats.stream
         try:
             stats.stream = stdout_trap
             stats.print_stats(*lims)
         finally:
             stats.stream = stats_stream
 
         output = stdout_trap.getvalue()
         output = output.rstrip()
 
         if 'q' not in opts:
             page.page(output)
         print(sys_exit, end=' ')
 
         dump_file = opts.D[0]
         text_file = opts.T[0]
         if dump_file:
             prof.dump_stats(dump_file)
             print('\n*** Profile stats marshalled to file',\
                   repr(dump_file)+'.',sys_exit)
         if text_file:
             with open(text_file, 'w') as pfile:
                 pfile.write(output)
             print('\n*** Profile printout saved to text file',\
                   repr(text_file)+'.',sys_exit)
 
         if 'r' in opts:
             return stats
         else:
             return None
 
     @line_magic
     def pdb(self, parameter_s=''):
         """Control the automatic calling of the pdb interactive debugger.
 
         Call as '%pdb on', '%pdb 1', '%pdb off' or '%pdb 0'. If called without
         argument it works as a toggle.
 
         When an exception is triggered, IPython can optionally call the
         interactive pdb debugger after the traceback printout. %pdb toggles
         this feature on and off.
 
         The initial state of this feature is set in your configuration
         file (the option is ``InteractiveShell.pdb``).
 
         If you want to just activate the debugger AFTER an exception has fired,
         without having to type '%pdb on' and rerunning your code, you can use
         the %debug magic."""
 
         par = parameter_s.strip().lower()
 
         if par:
             try:
                 new_pdb = {'off':0,'0':0,'on':1,'1':1}[par]
             except KeyError:
                 print ('Incorrect argument. Use on/1, off/0, '
                        'or nothing for a toggle.')
                 return
         else:
             # toggle
             new_pdb = not self.shell.call_pdb
 
         # set on the shell
         self.shell.call_pdb = new_pdb
         print('Automatic pdb calling has been turned',on_off(new_pdb))
 
     @skip_doctest
     @magic_arguments.magic_arguments()
     @magic_arguments.argument('--breakpoint', '-b', metavar='FILE:LINE',
         help="""
         Set break point at LINE in FILE.
         """
     )
     @magic_arguments.argument('statement', nargs='*',
         help="""
         Code to run in debugger.
         You can omit this in cell magic mode.
         """
     )
     @no_var_expand
     @line_cell_magic
     def debug(self, line='', cell=None):
         """Activate the interactive debugger.
 
         This magic command support two ways of activating debugger.
         One is to activate debugger before executing code.  This way, you
         can set a break point, to step through the code from the point.
         You can use this mode by giving statements to execute and optionally
         a breakpoint.
 
         The other one is to activate debugger in post-mortem mode.  You can
         activate this mode simply running %debug without any argument.
         If an exception has just occurred, this lets you inspect its stack
         frames interactively.  Note that this will always work only on the last
         traceback that occurred, so you must call this quickly after an
         exception that you wish to inspect has fired, because if another one
         occurs, it clobbers the previous one.
 
         If you want IPython to automatically do this on every exception, see
         the %pdb magic for more details.
 
         .. versionchanged:: 7.3
             When running code, user variables are no longer expanded,
             the magic line is always left unmodified.
 
         """
         args = magic_arguments.parse_argstring(self.debug, line)
 
         if not (args.breakpoint or args.statement or cell):
             self._debug_post_mortem()
         else:
             code = "\n".join(args.statement)
             if cell:
                 code += "\n" + cell
             self._debug_exec(code, args.breakpoint)
 
     def _debug_post_mortem(self):
         self.shell.debugger(force=True)
 
     def _debug_exec(self, code, breakpoint):
         if breakpoint:
             (filename, bp_line) = breakpoint.rsplit(':', 1)
             bp_line = int(bp_line)
         else:
             (filename, bp_line) = (None, None)
         self._run_with_debugger(code, self.shell.user_ns, filename, bp_line)
 
     @line_magic
     def tb(self, s):
         """Print the last traceback.
 
         Optionally, specify an exception reporting mode, tuning the
         verbosity of the traceback. By default the currently-active exception
         mode is used. See %xmode for changing exception reporting modes.
 
         Valid modes: Plain, Context, Verbose, and Minimal.
         """
         interactive_tb = self.shell.InteractiveTB
         if s:
             # Switch exception reporting mode for this one call.
             # Ensure it is switched back.
             def xmode_switch_err(name):
                 warn('Error changing %s exception modes.\n%s' %
                     (name,sys.exc_info()[1]))
 
             new_mode = s.strip().capitalize()
             original_mode = interactive_tb.mode
             try:
                 try:
                     interactive_tb.set_mode(mode=new_mode)
                 except Exception:
                     xmode_switch_err('user')
                 else:
                     self.shell.showtraceback()
             finally:
                 interactive_tb.set_mode(mode=original_mode)
         else:
             self.shell.showtraceback()
 
     @skip_doctest
     @line_magic
     def run(self, parameter_s='', runner=None,
                   file_finder=get_py_filename):
         """Run the named file inside IPython as a program.
 
         Usage::
         
           %run [-n -i -e -G]
                [( -t [-N<N>] | -d [-b<N>] | -p [profile options] )]
                ( -m mod | file ) [args]
 
         Parameters after the filename are passed as command-line arguments to
         the program (put in sys.argv). Then, control returns to IPython's
         prompt.
 
         This is similar to running at a system prompt ``python file args``,
         but with the advantage of giving you IPython's tracebacks, and of
         loading all variables into your interactive namespace for further use
         (unless -p is used, see below).
 
         The file is executed in a namespace initially consisting only of
         ``__name__=='__main__'`` and sys.argv constructed as indicated. It thus
         sees its environment as if it were being run as a stand-alone program
         (except for sharing global objects such as previously imported
         modules). But after execution, the IPython interactive namespace gets
         updated with all variables defined in the program (except for __name__
         and sys.argv). This allows for very convenient loading of code for
         interactive work, while giving each program a 'clean sheet' to run in.
 
         Arguments are expanded using shell-like glob match.  Patterns
         '*', '?', '[seq]' and '[!seq]' can be used.  Additionally,
         tilde '~' will be expanded into user's home directory.  Unlike
         real shells, quotation does not suppress expansions.  Use
         *two* back slashes (e.g. ``\\\\*``) to suppress expansions.
         To completely disable these expansions, you can use -G flag.
 
         On Windows systems, the use of single quotes `'` when specifying 
         a file is not supported. Use double quotes `"`.
 
         Options:
 
         -n
           __name__ is NOT set to '__main__', but to the running file's name
           without extension (as python does under import).  This allows running
           scripts and reloading the definitions in them without calling code
           protected by an ``if __name__ == "__main__"`` clause.
 
         -i
           run the file in IPython's namespace instead of an empty one. This
           is useful if you are experimenting with code written in a text editor
           which depends on variables defined interactively.
 
         -e
           ignore sys.exit() calls or SystemExit exceptions in the script
           being run.  This is particularly useful if IPython is being used to
           run unittests, which always exit with a sys.exit() call.  In such
           cases you are interested in the output of the test results, not in
           seeing a traceback of the unittest module.
 
         -t
           print timing information at the end of the run.  IPython will give
           you an estimated CPU time consumption for your script, which under
           Unix uses the resource module to avoid the wraparound problems of
           time.clock().  Under Unix, an estimate of time spent on system tasks
           is also given (for Windows platforms this is reported as 0.0).
 
         If -t is given, an additional ``-N<N>`` option can be given, where <N>
         must be an integer indicating how many times you want the script to
         run.  The final timing report will include total and per run results.
 
         For example (testing the script uniq_stable.py)::
 
             In [1]: run -t uniq_stable
 
             IPython CPU timings (estimated):
               User  :    0.19597 s.
               System:        0.0 s.
 
             In [2]: run -t -N5 uniq_stable
 
             IPython CPU timings (estimated):
             Total runs performed: 5
               Times :      Total       Per run
               User  :   0.910862 s,  0.1821724 s.
               System:        0.0 s,        0.0 s.
 
         -d
           run your program under the control of pdb, the Python debugger.
           This allows you to execute your program step by step, watch variables,
           etc.  Internally, what IPython does is similar to calling::
 
               pdb.run('execfile("YOURFILENAME")')
 
           with a breakpoint set on line 1 of your file.  You can change the line
           number for this automatic breakpoint to be <N> by using the -bN option
           (where N must be an integer). For example::
 
               %run -d -b40 myscript
 
           will set the first breakpoint at line 40 in myscript.py.  Note that
           the first breakpoint must be set on a line which actually does
           something (not a comment or docstring) for it to stop execution.
 
           Or you can specify a breakpoint in a different file::
 
               %run -d -b myotherfile.py:20 myscript
 
           When the pdb debugger starts, you will see a (Pdb) prompt.  You must
           first enter 'c' (without quotes) to start execution up to the first
           breakpoint.
 
           Entering 'help' gives information about the use of the debugger.  You
           can easily see pdb's full documentation with "import pdb;pdb.help()"
           at a prompt.
 
         -p
           run program under the control of the Python profiler module (which
           prints a detailed report of execution times, function calls, etc).
 
           You can pass other options after -p which affect the behavior of the
           profiler itself. See the docs for %prun for details.
 
           In this mode, the program's variables do NOT propagate back to the
           IPython interactive namespace (because they remain in the namespace
           where the profiler executes them).
 
           Internally this triggers a call to %prun, see its documentation for
           details on the options available specifically for profiling.
 
         There is one special usage for which the text above doesn't apply:
         if the filename ends with .ipy[nb], the file is run as ipython script,
         just as if the commands were written on IPython prompt.
 
         -m
           specify module name to load instead of script path. Similar to
           the -m option for the python interpreter. Use this option last if you
           want to combine with other %run options. Unlike the python interpreter
           only source modules are allowed no .pyc or .pyo files.
           For example::
 
               %run -m example
 
           will run the example module.
 
         -G
           disable shell-like glob expansion of arguments.
 
         """
 
         # Logic to handle issue #3664
         # Add '--' after '-m <module_name>' to ignore additional args passed to a module.
         if '-m' in parameter_s and '--' not in parameter_s:
             argv = shlex.split(parameter_s, posix=(os.name == 'posix'))
             for idx, arg in enumerate(argv):
                 if arg and arg.startswith('-') and arg != '-':
                     if arg == '-m':
                         argv.insert(idx + 2, '--')
                         break
                 else:
                     # Positional arg, break
                     break
             parameter_s = ' '.join(shlex.quote(arg) for arg in argv)
 
         # get arguments and set sys.argv for program to be run.
         opts, arg_lst = self.parse_options(parameter_s,
                                            'nidtN:b:pD:l:rs:T:em:G',
                                            mode='list', list_all=1)
         if "m" in opts:
             modulename = opts["m"][0]
             modpath = find_mod(modulename)
             if modpath is None:
                 warn('%r is not a valid modulename on sys.path'%modulename)
                 return
             arg_lst = [modpath] + arg_lst
         try:
             fpath = None # initialize to make sure fpath is in scope later
             fpath = arg_lst[0]
             filename = file_finder(fpath)
         except IndexError:
             warn('you must provide at least a filename.')
             print('\n%run:\n', oinspect.getdoc(self.run))
             return
         except IOError as e:
             try:
                 msg = str(e)
             except UnicodeError:
                 msg = e.message
             if os.name == 'nt' and re.match(r"^'.*'$",fpath):
                 warn('For Windows, use double quotes to wrap a filename: %run "mypath\\myfile.py"')
             error(msg)
             return
 
         if filename.lower().endswith(('.ipy', '.ipynb')):
             with preserve_keys(self.shell.user_ns, '__file__'):
                 self.shell.user_ns['__file__'] = filename
                 self.shell.safe_execfile_ipy(filename)
             return
 
         # Control the response to exit() calls made by the script being run
         exit_ignore = 'e' in opts
 
         # Make sure that the running script gets a proper sys.argv as if it
         # were run from a system shell.
         save_argv = sys.argv # save it for later restoring
 
         if 'G' in opts:
             args = arg_lst[1:]
         else:
             # tilde and glob expansion
             args = shellglob(map(os.path.expanduser,  arg_lst[1:]))
 
         sys.argv = [filename] + args  # put in the proper filename
 
         if 'n' in opts:
             name = os.path.splitext(os.path.basename(filename))[0]
         else:
             name = '__main__'
 
         if 'i' in opts:
             # Run in user's interactive namespace
             prog_ns = self.shell.user_ns
             __name__save = self.shell.user_ns['__name__']
             prog_ns['__name__'] = name
             main_mod = self.shell.user_module
 
             # Since '%run foo' emulates 'python foo.py' at the cmd line, we must
             # set the __file__ global in the script's namespace
             # TK: Is this necessary in interactive mode?
             prog_ns['__file__'] = filename
         else:
             # Run in a fresh, empty namespace
 
             # The shell MUST hold a reference to prog_ns so after %run
             # exits, the python deletion mechanism doesn't zero it out
             # (leaving dangling references). See interactiveshell for details
             main_mod = self.shell.new_main_mod(filename, name)
             prog_ns = main_mod.__dict__
 
         # pickle fix.  See interactiveshell for an explanation.  But we need to
         # make sure that, if we overwrite __main__, we replace it at the end
         main_mod_name = prog_ns['__name__']
 
         if main_mod_name == '__main__':
             restore_main = sys.modules['__main__']
         else:
             restore_main = False
 
         # This needs to be undone at the end to prevent holding references to
         # every single object ever created.
         sys.modules[main_mod_name] = main_mod
 
         if 'p' in opts or 'd' in opts:
             if 'm' in opts:
                 code = 'run_module(modulename, prog_ns)'
                 code_ns = {
                     'run_module': self.shell.safe_run_module,
                     'prog_ns': prog_ns,
                     'modulename': modulename,
                 }
             else:
                 if 'd' in opts:
                     # allow exceptions to raise in debug mode
                     code = 'execfile(filename, prog_ns, raise_exceptions=True)'
                 else:
                     code = 'execfile(filename, prog_ns)'
                 code_ns = {
                     'execfile': self.shell.safe_execfile,
                     'prog_ns': prog_ns,
                     'filename': get_py_filename(filename),
                 }
 
         try:
             stats = None
             if 'p' in opts:
                 stats = self._run_with_profiler(code, opts, code_ns)
             else:
                 if 'd' in opts:
                     bp_file, bp_line = parse_breakpoint(
                         opts.get('b', ['1'])[0], filename)
                     self._run_with_debugger(
                         code, code_ns, filename, bp_line, bp_file)
                 else:
                     if 'm' in opts:
                         def run():
                             self.shell.safe_run_module(modulename, prog_ns)
                     else:
                         if runner is None:
                             runner = self.default_runner
                         if runner is None:
                             runner = self.shell.safe_execfile
 
                         def run():
                             runner(filename, prog_ns, prog_ns,
                                     exit_ignore=exit_ignore)
 
                     if 't' in opts:
                         # timed execution
                         try:
                             nruns = int(opts['N'][0])
                             if nruns < 1:
                                 error('Number of runs must be >=1')
                                 return
                         except (KeyError):
                             nruns = 1
                         self._run_with_timing(run, nruns)
                     else:
                         # regular execution
                         run()
 
             if 'i' in opts:
                 self.shell.user_ns['__name__'] = __name__save
             else:
                 # update IPython interactive namespace
 
                 # Some forms of read errors on the file may mean the
                 # __name__ key was never set; using pop we don't have to
                 # worry about a possible KeyError.
                 prog_ns.pop('__name__', None)
 
                 with preserve_keys(self.shell.user_ns, '__file__'):
                     self.shell.user_ns.update(prog_ns)
         finally:
             # It's a bit of a mystery why, but __builtins__ can change from
             # being a module to becoming a dict missing some key data after
             # %run.  As best I can see, this is NOT something IPython is doing
             # at all, and similar problems have been reported before:
             # http://coding.derkeiler.com/Archive/Python/comp.lang.python/2004-10/0188.html
             # Since this seems to be done by the interpreter itself, the best
             # we can do is to at least restore __builtins__ for the user on
             # exit.
             self.shell.user_ns['__builtins__'] = builtin_mod
 
             # Ensure key global structures are restored
             sys.argv = save_argv
             if restore_main:
                 sys.modules['__main__'] = restore_main
+                if '__mp_main__' in sys.modules:
+                    sys.modules['__mp_main__'] = restore_main
             else:
                 # Remove from sys.modules the reference to main_mod we'd
                 # added.  Otherwise it will trap references to objects
                 # contained therein.
                 del sys.modules[main_mod_name]
 
         return stats
 
     def _run_with_debugger(self, code, code_ns, filename=None,
                            bp_line=None, bp_file=None):
         """
         Run `code` in debugger with a break point.
 
         Parameters
         ----------
         code : str
             Code to execute.
         code_ns : dict
             A namespace in which `code` is executed.
         filename : str
             `code` is ran as if it is in `filename`.
         bp_line : int, optional
             Line number of the break point.
         bp_file : str, optional
             Path to the file in which break point is specified.
             `filename` is used if not given.
 
         Raises
         ------
         UsageError
             If the break point given by `bp_line` is not valid.
 
         """
         deb = self.shell.InteractiveTB.pdb
         if not deb:
             self.shell.InteractiveTB.pdb = self.shell.InteractiveTB.debugger_cls()
             deb = self.shell.InteractiveTB.pdb
 
         # deb.checkline() fails if deb.curframe exists but is None; it can
         # handle it not existing. https://github.com/ipython/ipython/issues/10028
         if hasattr(deb, 'curframe'):
             del deb.curframe
 
         # reset Breakpoint state, which is moronically kept
         # in a class
         bdb.Breakpoint.next = 1
         bdb.Breakpoint.bplist = {}
         bdb.Breakpoint.bpbynumber = [None]
         deb.clear_all_breaks()
         if bp_line is not None:
             # Set an initial breakpoint to stop execution
             maxtries = 10
             bp_file = bp_file or filename
             checkline = deb.checkline(bp_file, bp_line)
             if not checkline:
                 for bp in range(bp_line + 1, bp_line + maxtries + 1):
                     if deb.checkline(bp_file, bp):
                         break
                 else:
                     msg = ("\nI failed to find a valid line to set "
                            "a breakpoint\n"
                            "after trying up to line: %s.\n"
                            "Please set a valid breakpoint manually "
                            "with the -b option." % bp)
                     raise UsageError(msg)
             # if we find a good linenumber, set the breakpoint
             deb.do_break('%s:%s' % (bp_file, bp_line))
 
         if filename:
             # Mimic Pdb._runscript(...)
             deb._wait_for_mainpyfile = True
             deb.mainpyfile = deb.canonic(filename)
 
         # Start file run
         print("NOTE: Enter 'c' at the %s prompt to continue execution." % deb.prompt)
         try:
             if filename:
                 # save filename so it can be used by methods on the deb object
                 deb._exec_filename = filename
             while True:
                 try:
                     trace = sys.gettrace()
                     deb.run(code, code_ns)
                 except Restart:
                     print("Restarting")
                     if filename:
                         deb._wait_for_mainpyfile = True
                         deb.mainpyfile = deb.canonic(filename)
                     continue
                 else:
                     break
                 finally:
                     sys.settrace(trace)
             
 
         except:
             etype, value, tb = sys.exc_info()
             # Skip three frames in the traceback: the %run one,
             # one inside bdb.py, and the command-line typed by the
             # user (run by exec in pdb itself).
             self.shell.InteractiveTB(etype, value, tb, tb_offset=3)
 
     @staticmethod
     def _run_with_timing(run, nruns):
         """
         Run function `run` and print timing information.
 
         Parameters
         ----------
         run : callable
             Any callable object which takes no argument.
         nruns : int
             Number of times to execute `run`.
 
         """
         twall0 = time.perf_counter()
         if nruns == 1:
             t0 = clock2()
             run()
             t1 = clock2()
             t_usr = t1[0] - t0[0]
             t_sys = t1[1] - t0[1]
             print("\nIPython CPU timings (estimated):")
             print("  User   : %10.2f s." % t_usr)
             print("  System : %10.2f s." % t_sys)
         else:
             runs = range(nruns)
             t0 = clock2()
             for nr in runs:
                 run()
             t1 = clock2()
             t_usr = t1[0] - t0[0]
             t_sys = t1[1] - t0[1]
             print("\nIPython CPU timings (estimated):")
             print("Total runs performed:", nruns)
             print("  Times  : %10s   %10s" % ('Total', 'Per run'))
             print("  User   : %10.2f s, %10.2f s." % (t_usr, t_usr / nruns))
             print("  System : %10.2f s, %10.2f s." % (t_sys, t_sys / nruns))
         twall1 = time.perf_counter()
         print("Wall time: %10.2f s." % (twall1 - twall0))
 
     @skip_doctest
     @no_var_expand
     @line_cell_magic
     @needs_local_scope
     def timeit(self, line='', cell=None, local_ns=None):
         """Time execution of a Python statement or expression
 
         Usage, in line mode:
           %timeit [-n<N> -r<R> [-t|-c] -q -p<P> -o] statement
         or in cell mode:
           %%timeit [-n<N> -r<R> [-t|-c] -q -p<P> -o] setup_code
           code
           code...
 
         Time execution of a Python statement or expression using the timeit
         module.  This function can be used both as a line and cell magic:
 
         - In line mode you can time a single-line statement (though multiple
           ones can be chained with using semicolons).
 
         - In cell mode, the statement in the first line is used as setup code
           (executed but not timed) and the body of the cell is timed.  The cell
           body has access to any variables created in the setup code.
 
         Options:
         -n<N>: execute the given statement <N> times in a loop. If <N> is not
         provided, <N> is determined so as to get sufficient accuracy.
 
         -r<R>: number of repeats <R>, each consisting of <N> loops, and take the
         best result.
         Default: 7
 
         -t: use time.time to measure the time, which is the default on Unix.
         This function measures wall time.
 
         -c: use time.clock to measure the time, which is the default on
         Windows and measures wall time. On Unix, resource.getrusage is used
         instead and returns the CPU user time.
 
         -p<P>: use a precision of <P> digits to display the timing result.
         Default: 3
 
         -q: Quiet, do not print result.
 
         -o: return a TimeitResult that can be stored in a variable to inspect
             the result in more details.
 
         .. versionchanged:: 7.3
             User variables are no longer expanded,
             the magic line is always left unmodified.
 
         Examples
         --------
         ::
 
           In [1]: %timeit pass
           8.26 ns ± 0.12 ns per loop (mean ± std. dev. of 7 runs, 100000000 loops each)
 
           In [2]: u = None
 
           In [3]: %timeit u is None
           29.9 ns ± 0.643 ns per loop (mean ± std. dev. of 7 runs, 10000000 loops each)
 
           In [4]: %timeit -r 4 u == None
 
           In [5]: import time
 
           In [6]: %timeit -n1 time.sleep(2)
 
 
         The times reported by %timeit will be slightly higher than those
         reported by the timeit.py script when variables are accessed. This is
         due to the fact that %timeit executes the statement in the namespace
         of the shell, compared with timeit.py, which uses a single setup
         statement to import function or create variables. Generally, the bias
         does not matter as long as results from timeit.py are not mixed with
         those from %timeit."""
 
         opts, stmt = self.parse_options(line,'n:r:tcp:qo',
                                         posix=False, strict=False)
         if stmt == "" and cell is None:
             return
         
         timefunc = timeit.default_timer
         number = int(getattr(opts, "n", 0))
         default_repeat = 7 if timeit.default_repeat < 7 else timeit.default_repeat
         repeat = int(getattr(opts, "r", default_repeat))
         precision = int(getattr(opts, "p", 3))
         quiet = 'q' in opts
         return_result = 'o' in opts
         if hasattr(opts, "t"):
             timefunc = time.time
         if hasattr(opts, "c"):
             timefunc = clock
 
         timer = Timer(timer=timefunc)
         # this code has tight coupling to the inner workings of timeit.Timer,
         # but is there a better way to achieve that the code stmt has access
         # to the shell namespace?
         transform  = self.shell.transform_cell
 
         if cell is None:
             # called as line magic
             ast_setup = self.shell.compile.ast_parse("pass")
             ast_stmt = self.shell.compile.ast_parse(transform(stmt))
         else:
             ast_setup = self.shell.compile.ast_parse(transform(stmt))
             ast_stmt = self.shell.compile.ast_parse(transform(cell))
 
         ast_setup = self.shell.transform_ast(ast_setup)
         ast_stmt = self.shell.transform_ast(ast_stmt)
 
         # Check that these compile to valid Python code *outside* the timer func
         # Invalid code may become valid when put inside the function & loop,
         # which messes up error messages.
         # https://github.com/ipython/ipython/issues/10636
         self.shell.compile(ast_setup, "<magic-timeit-setup>", "exec")
         self.shell.compile(ast_stmt, "<magic-timeit-stmt>", "exec")
 
         # This codestring is taken from timeit.template - we fill it in as an
         # AST, so that we can apply our AST transformations to the user code
         # without affecting the timing code.
         timeit_ast_template = ast.parse('def inner(_it, _timer):\n'
                                         '    setup\n'
                                         '    _t0 = _timer()\n'
                                         '    for _i in _it:\n'
                                         '        stmt\n'
                                         '    _t1 = _timer()\n'
                                         '    return _t1 - _t0\n')
 
         timeit_ast = TimeitTemplateFiller(ast_setup, ast_stmt).visit(timeit_ast_template)
         timeit_ast = ast.fix_missing_locations(timeit_ast)
 
         # Track compilation time so it can be reported if too long
         # Minimum time above which compilation time will be reported
         tc_min = 0.1
 
         t0 = clock()
         code = self.shell.compile(timeit_ast, "<magic-timeit>", "exec")
         tc = clock()-t0
 
         ns = {}
         glob = self.shell.user_ns
         # handles global vars with same name as local vars. We store them in conflict_globs.
         conflict_globs = {}
         if local_ns and cell is None:
             for var_name, var_val in glob.items():
                 if var_name in local_ns:
                     conflict_globs[var_name] = var_val
             glob.update(local_ns)
             
         exec(code, glob, ns)
         timer.inner = ns["inner"]
 
         # This is used to check if there is a huge difference between the
         # best and worst timings.
         # Issue: https://github.com/ipython/ipython/issues/6471
         if number == 0:
             # determine number so that 0.2 <= total time < 2.0
             for index in range(0, 10):
                 number = 10 ** index
                 time_number = timer.timeit(number)
                 if time_number >= 0.2:
                     break
 
         all_runs = timer.repeat(repeat, number)
         best = min(all_runs) / number
         worst = max(all_runs) / number
         timeit_result = TimeitResult(number, repeat, best, worst, all_runs, tc, precision)
 
         # Restore global vars from conflict_globs
         if conflict_globs:
            glob.update(conflict_globs)
                 
         if not quiet :
             # Check best timing is greater than zero to avoid a
             # ZeroDivisionError.
             # In cases where the slowest timing is lesser than a microsecond
             # we assume that it does not really matter if the fastest
             # timing is 4 times faster than the slowest timing or not.
             if worst > 4 * best and best > 0 and worst > 1e-6:
                 print("The slowest run took %0.2f times longer than the "
                       "fastest. This could mean that an intermediate result "
                       "is being cached." % (worst / best))
            
             print( timeit_result )
 
             if tc > tc_min:
                 print("Compiler time: %.2f s" % tc)
         if return_result:
             return timeit_result
 
     @skip_doctest
     @no_var_expand
     @needs_local_scope
     @line_cell_magic
     def time(self,line='', cell=None, local_ns=None):
         """Time execution of a Python statement or expression.
 
         The CPU and wall clock times are printed, and the value of the
         expression (if any) is returned.  Note that under Win32, system time
         is always reported as 0, since it can not be measured.
         
         This function can be used both as a line and cell magic:
 
         - In line mode you can time a single-line statement (though multiple
           ones can be chained with using semicolons).
 
         - In cell mode, you can time the cell body (a directly
           following statement raises an error).
 
         This function provides very basic timing functionality.  Use the timeit
         magic for more control over the measurement.
 
         .. versionchanged:: 7.3
             User variables are no longer expanded,
             the magic line is always left unmodified.
 
         Examples
         --------
         ::
 
           In [1]: %time 2**128
           CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s
           Wall time: 0.00
           Out[1]: 340282366920938463463374607431768211456L
 
           In [2]: n = 1000000
 
           In [3]: %time sum(range(n))
           CPU times: user 1.20 s, sys: 0.05 s, total: 1.25 s
           Wall time: 1.37
           Out[3]: 499999500000L
 
           In [4]: %time print 'hello world'
           hello world
           CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s
           Wall time: 0.00
 
           Note that the time needed by Python to compile the given expression
           will be reported if it is more than 0.1s.  In this example, the
           actual exponentiation is done by Python at compilation time, so while
           the expression can take a noticeable amount of time to compute, that
           time is purely due to the compilation:
 
           In [5]: %time 3**9999;
           CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s
           Wall time: 0.00 s
 
           In [6]: %time 3**999999;
           CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s
           Wall time: 0.00 s
           Compiler : 0.78 s
           """
 
         # fail immediately if the given expression can't be compiled
         
         if line and cell:
             raise UsageError("Can't use statement directly after '%%time'!")
         
         if cell:
             expr = self.shell.transform_cell(cell)
         else:
             expr = self.shell.transform_cell(line)
 
         # Minimum time above which parse time will be reported
         tp_min = 0.1
 
         t0 = clock()
         expr_ast = self.shell.compile.ast_parse(expr)
         tp = clock()-t0
 
         # Apply AST transformations
         expr_ast = self.shell.transform_ast(expr_ast)
 
         # Minimum time above which compilation time will be reported
         tc_min = 0.1
 
         expr_val=None
         if len(expr_ast.body)==1 and isinstance(expr_ast.body[0], ast.Expr):
             mode = 'eval'
             source = '<timed eval>'
             expr_ast = ast.Expression(expr_ast.body[0].value)
         else:
             mode = 'exec'
             source = '<timed exec>'
             # multi-line %%time case
             if len(expr_ast.body) > 1 and isinstance(expr_ast.body[-1], ast.Expr):
                 expr_val= expr_ast.body[-1]
                 expr_ast = expr_ast.body[:-1]
                 expr_ast = Module(expr_ast, [])
                 expr_val = ast.Expression(expr_val.value)
 
         t0 = clock()
         code = self.shell.compile(expr_ast, source, mode)
         tc = clock()-t0
 
         # skew measurement as little as possible
         glob = self.shell.user_ns
         wtime = time.time
         # time execution
         wall_st = wtime()
         if mode=='eval':
             st = clock2()
             try:
                 out = eval(code, glob, local_ns)
             except:
                 self.shell.showtraceback()
                 return
             end = clock2()
         else:
             st = clock2()
             try:
                 exec(code, glob, local_ns)
                 out=None
                 # multi-line %%time case
                 if expr_val is not None:
                     code_2 = self.shell.compile(expr_val, source, 'eval')
                     out = eval(code_2, glob, local_ns)
             except:
                 self.shell.showtraceback()
                 return
             end = clock2()
 
         wall_end = wtime()
         # Compute actual times and report
         wall_time = wall_end-wall_st
         cpu_user = end[0]-st[0]
         cpu_sys = end[1]-st[1]
         cpu_tot = cpu_user+cpu_sys
         # On windows cpu_sys is always zero, so no new information to the next print 
         if sys.platform != 'win32':
             print("CPU times: user %s, sys: %s, total: %s" % \
                 (_format_time(cpu_user),_format_time(cpu_sys),_format_time(cpu_tot)))
         print("Wall time: %s" % _format_time(wall_time))
         if tc > tc_min:
             print("Compiler : %s" % _format_time(tc))
         if tp > tp_min:
             print("Parser   : %s" % _format_time(tp))
         return out
 
     @skip_doctest
     @line_magic
     def macro(self, parameter_s=''):
         """Define a macro for future re-execution. It accepts ranges of history,
         filenames or string objects.
 
         Usage:\\
           %macro [options] name n1-n2 n3-n4 ... n5 .. n6 ...
 
         Options:
 
           -r: use 'raw' input.  By default, the 'processed' history is used,
           so that magics are loaded in their transformed version to valid
           Python.  If this option is given, the raw input as typed at the
           command line is used instead.
           
           -q: quiet macro definition.  By default, a tag line is printed 
           to indicate the macro has been created, and then the contents of 
           the macro are printed.  If this option is given, then no printout
           is produced once the macro is created.
 
         This will define a global variable called `name` which is a string
         made of joining the slices and lines you specify (n1,n2,... numbers
         above) from your input history into a single string. This variable
         acts like an automatic function which re-executes those lines as if
         you had typed them. You just type 'name' at the prompt and the code
         executes.
 
         The syntax for indicating input ranges is described in %history.
 
         Note: as a 'hidden' feature, you can also use traditional python slice
         notation, where N:M means numbers N through M-1.
 
         For example, if your history contains (print using %hist -n )::
 
           44: x=1
           45: y=3
           46: z=x+y
           47: print x
           48: a=5
           49: print 'x',x,'y',y
 
         you can create a macro with lines 44 through 47 (included) and line 49
         called my_macro with::
 
           In [55]: %macro my_macro 44-47 49
 
         Now, typing `my_macro` (without quotes) will re-execute all this code
         in one pass.
 
         You don't need to give the line-numbers in order, and any given line
         number can appear multiple times. You can assemble macros with any
         lines from your input history in any order.
 
         The macro is a simple object which holds its value in an attribute,
         but IPython's display system checks for macros and executes them as
         code instead of printing them when you type their name.
 
         You can view a macro's contents by explicitly printing it with::
 
           print macro_name
 
         """
         opts,args = self.parse_options(parameter_s,'rq',mode='list')
         if not args:   # List existing macros
             return sorted(k for k,v in self.shell.user_ns.items() if isinstance(v, Macro))
         if len(args) == 1:
             raise UsageError(
                 "%macro insufficient args; usage '%macro name n1-n2 n3-4...")
         name, codefrom = args[0], " ".join(args[1:])
 
         #print 'rng',ranges  # dbg
         try:
             lines = self.shell.find_user_code(codefrom, 'r' in opts)
         except (ValueError, TypeError) as e:
             print(e.args[0])
             return
         macro = Macro(lines)
         self.shell.define_macro(name, macro)
         if not ( 'q' in opts) : 
             print('Macro `%s` created. To execute, type its name (without quotes).' % name)
             print('=== Macro contents: ===')
             print(macro, end=' ')
 
     @magic_arguments.magic_arguments()
     @magic_arguments.argument('output', type=str, default='', nargs='?',
         help="""The name of the variable in which to store output.
         This is a utils.io.CapturedIO object with stdout/err attributes
         for the text of the captured output.
 
         CapturedOutput also has a show() method for displaying the output,
         and __call__ as well, so you can use that to quickly display the
         output.
 
         If unspecified, captured output is discarded.
         """
     )
     @magic_arguments.argument('--no-stderr', action="store_true",
         help="""Don't capture stderr."""
     )
     @magic_arguments.argument('--no-stdout', action="store_true",
         help="""Don't capture stdout."""
     )
     @magic_arguments.argument('--no-display', action="store_true",
         help="""Don't capture IPython's rich display."""
     )
     @cell_magic
     def capture(self, line, cell):
         """run the cell, capturing stdout, stderr, and IPython's rich display() calls."""
         args = magic_arguments.parse_argstring(self.capture, line)
         out = not args.no_stdout
         err = not args.no_stderr
         disp = not args.no_display
         with capture_output(out, err, disp) as io:
             self.shell.run_cell(cell)
         if args.output:
             self.shell.user_ns[args.output] = io
 
 def parse_breakpoint(text, current_file):
     '''Returns (file, line) for file:line and (current_file, line) for line'''
     colon = text.find(':')
     if colon == -1:
         return current_file, int(text)
     else:
         return text[:colon], int(text[colon+1:])
     
 def _format_time(timespan, precision=3):
     """Formats the timespan in a human readable form"""
 
     if timespan >= 60.0:
         # we have more than a minute, format that in a human readable form
         # Idea from http://snipplr.com/view/5713/
         parts = [("d", 60*60*24),("h", 60*60),("min", 60), ("s", 1)]
         time = []
         leftover = timespan
         for suffix, length in parts:
             value = int(leftover / length)
             if value > 0:
                 leftover = leftover % length
                 time.append(u'%s%s' % (str(value), suffix))
             if leftover < 1:
                 break
         return " ".join(time)
 
     
     # Unfortunately the unicode 'micro' symbol can cause problems in
     # certain terminals.  
     # See bug: https://bugs.launchpad.net/ipython/+bug/348466
     # Try to prevent crashes by being more secure than it needs to
     # E.g. eclipse is able to print a µ, but has no sys.stdout.encoding set.
     units = [u"s", u"ms",u'us',"ns"] # the save value   
     if hasattr(sys.stdout, 'encoding') and sys.stdout.encoding:
         try:
             u'\xb5'.encode(sys.stdout.encoding)
             units = [u"s", u"ms",u'\xb5s',"ns"]
         except:
             pass
     scaling = [1, 1e3, 1e6, 1e9]
         
     if timespan > 0.0:
         order = min(-int(math.floor(math.log10(timespan)) // 3), 3)
     else:
         order = 3
     return u"%.*g %s" % (precision, timespan * scaling[order], units[order])

IPython/core/oinspect.py

@@ -1,1072 +1,1058 b''
 # -*- coding: utf-8 -*-
 """Tools for inspecting Python objects.
 
 Uses syntax highlighting for presenting the various information elements.
 
 Similar in spirit to the inspect module, but all calls take a name argument to
 reference the name under which an object is being read.
 """
 
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
 __all__ = ['Inspector','InspectColors']
 
 # stdlib modules
 import ast
 import inspect
 from inspect import signature
 import linecache
 import warnings
 import os
 from textwrap import dedent
 import types
 import io as stdlib_io
 from itertools import zip_longest
 
 # IPython's own
 from IPython.core import page
 from IPython.lib.pretty import pretty
 from IPython.testing.skipdoctest import skip_doctest
 from IPython.utils import PyColorize
 from IPython.utils import openpy
 from IPython.utils import py3compat
 from IPython.utils.dir2 import safe_hasattr
 from IPython.utils.path import compress_user
 from IPython.utils.text import indent
 from IPython.utils.wildcard import list_namespace
 from IPython.utils.wildcard import typestr2type
 from IPython.utils.coloransi import TermColors, ColorScheme, ColorSchemeTable
 from IPython.utils.py3compat import cast_unicode
 from IPython.utils.colorable import Colorable
 from IPython.utils.decorators import undoc
 
 from pygments import highlight
 from pygments.lexers import PythonLexer
 from pygments.formatters import HtmlFormatter
 
 def pylight(code):
     return highlight(code, PythonLexer(), HtmlFormatter(noclasses=True))
 
 # builtin docstrings to ignore
 _func_call_docstring = types.FunctionType.__call__.__doc__
 _object_init_docstring = object.__init__.__doc__
 _builtin_type_docstrings = {
     inspect.getdoc(t) for t in (types.ModuleType, types.MethodType,
                                 types.FunctionType, property)
 }
 
 _builtin_func_type = type(all)
 _builtin_meth_type = type(str.upper)  # Bound methods have the same type as builtin functions
 #****************************************************************************
 # Builtin color schemes
 
 Colors = TermColors  # just a shorthand
 
 InspectColors = PyColorize.ANSICodeColors
 
 #****************************************************************************
 # Auxiliary functions and objects
 
 # See the messaging spec for the definition of all these fields.  This list
 # effectively defines the order of display
 info_fields = ['type_name', 'base_class', 'string_form', 'namespace',
                'length', 'file', 'definition', 'docstring', 'source',
                'init_definition', 'class_docstring', 'init_docstring',
                'call_def', 'call_docstring',
                # These won't be printed but will be used to determine how to
                # format the object
-               'ismagic', 'isalias', 'isclass', 'argspec', 'found', 'name'
+               'ismagic', 'isalias', 'isclass', 'found', 'name'
                ]
 
 
 def object_info(**kw):
     """Make an object info dict with all fields present."""
     infodict = dict(zip_longest(info_fields, [None]))
     infodict.update(kw)
     return infodict
 
 
 def get_encoding(obj):
     """Get encoding for python source file defining obj
 
     Returns None if obj is not defined in a sourcefile.
     """
     ofile = find_file(obj)
     # run contents of file through pager starting at line where the object
     # is defined, as long as the file isn't binary and is actually on the
     # filesystem.
     if ofile is None:
         return None
     elif ofile.endswith(('.so', '.dll', '.pyd')):
         return None
     elif not os.path.isfile(ofile):
         return None
     else:
         # Print only text files, not extension binaries.  Note that
         # getsourcelines returns lineno with 1-offset and page() uses
         # 0-offset, so we must adjust.
         with stdlib_io.open(ofile, 'rb') as buffer:   # Tweaked to use io.open for Python 2
             encoding, lines = openpy.detect_encoding(buffer.readline)
         return encoding
 
 def getdoc(obj):
     """Stable wrapper around inspect.getdoc.
 
     This can't crash because of attribute problems.
 
     It also attempts to call a getdoc() method on the given object.  This
     allows objects which provide their docstrings via non-standard mechanisms
     (like Pyro proxies) to still be inspected by ipython's ? system.
     """
     # Allow objects to offer customized documentation via a getdoc method:
     try:
         ds = obj.getdoc()
     except Exception:
         pass
     else:
         if isinstance(ds, str):
             return inspect.cleandoc(ds)
     docstr = inspect.getdoc(obj)
     encoding = get_encoding(obj)
     return py3compat.cast_unicode(docstr, encoding=encoding)
 
 
 def getsource(obj, oname=''):
     """Wrapper around inspect.getsource.
 
     This can be modified by other projects to provide customized source
     extraction.
 
     Parameters
     ----------
     obj : object
         an object whose source code we will attempt to extract
     oname : str
         (optional) a name under which the object is known
 
     Returns
     -------
     src : unicode or None
 
     """
 
     if isinstance(obj, property):
         sources = []
         for attrname in ['fget', 'fset', 'fdel']:
             fn = getattr(obj, attrname)
             if fn is not None:
                 encoding = get_encoding(fn)
                 oname_prefix = ('%s.' % oname) if oname else ''
                 sources.append(cast_unicode(
                     ''.join(('# ', oname_prefix, attrname)),
                     encoding=encoding))
                 if inspect.isfunction(fn):
                     sources.append(dedent(getsource(fn)))
                 else:
                     # Default str/repr only prints function name,
                     # pretty.pretty prints module name too.
                     sources.append(cast_unicode(
                         '%s%s = %s\n' % (
                             oname_prefix, attrname, pretty(fn)),
                         encoding=encoding))
         if sources:
             return '\n'.join(sources)
         else:
             return None
 
     else:
         # Get source for non-property objects.
 
         obj = _get_wrapped(obj)
 
         try:
             src = inspect.getsource(obj)
         except TypeError:
             # The object itself provided no meaningful source, try looking for
             # its class definition instead.
             if hasattr(obj, '__class__'):
                 try:
                     src = inspect.getsource(obj.__class__)
                 except TypeError:
                     return None
 
         encoding = get_encoding(obj)
         return cast_unicode(src, encoding=encoding)
 
 
 def is_simple_callable(obj):
     """True if obj is a function ()"""
     return (inspect.isfunction(obj) or inspect.ismethod(obj) or \
             isinstance(obj, _builtin_func_type) or isinstance(obj, _builtin_meth_type))
 
-
+@undoc
 def getargspec(obj):
     """Wrapper around :func:`inspect.getfullargspec` on Python 3, and
     :func:inspect.getargspec` on Python 2.
 
     In addition to functions and methods, this can also handle objects with a
     ``__call__`` attribute.
+
+    DEPRECATED: Deprecated since 7.10. Do not use, will be removed.
     """
+
+    warnings.warn('`getargspec` function is deprecated as of IPython 7.10'
+                  'and will be removed in future versions.', DeprecationWarning, stacklevel=2)
+
     if safe_hasattr(obj, '__call__') and not is_simple_callable(obj):
         obj = obj.__call__
 
     return inspect.getfullargspec(obj)
 
-
+@undoc
 def format_argspec(argspec):
     """Format argspect, convenience wrapper around inspect's.
 
     This takes a dict instead of ordered arguments and calls
     inspect.format_argspec with the arguments in the necessary order.
+
+    DEPRECATED: Do not use; will be removed in future versions.
     """
+    
+    warnings.warn('`format_argspec` function is deprecated as of IPython 7.10'
+                  'and will be removed in future versions.', DeprecationWarning, stacklevel=2)
+
+
     return inspect.formatargspec(argspec['args'], argspec['varargs'],
                                  argspec['varkw'], argspec['defaults'])
 
 @undoc
 def call_tip(oinfo, format_call=True):
     """DEPRECATED. Extract call tip data from an oinfo dict.
     """
     warnings.warn('`call_tip` function is deprecated as of IPython 6.0'
                   'and will be removed in future versions.', DeprecationWarning, stacklevel=2)
     # Get call definition
     argspec = oinfo.get('argspec')
     if argspec is None:
         call_line = None
     else:
         # Callable objects will have 'self' as their first argument, prune
         # it out if it's there for clarity (since users do *not* pass an
         # extra first argument explicitly).
         try:
             has_self = argspec['args'][0] == 'self'
         except (KeyError, IndexError):
             pass
         else:
             if has_self:
                 argspec['args'] = argspec['args'][1:]
 
         call_line = oinfo['name']+format_argspec(argspec)
 
     # Now get docstring.
     # The priority is: call docstring, constructor docstring, main one.
     doc = oinfo.get('call_docstring')
     if doc is None:
         doc = oinfo.get('init_docstring')
     if doc is None:
         doc = oinfo.get('docstring','')
 
     return call_line, doc
 
 
 def _get_wrapped(obj):
     """Get the original object if wrapped in one or more @decorators
 
     Some objects automatically construct similar objects on any unrecognised
     attribute access (e.g. unittest.mock.call). To protect against infinite loops,
     this will arbitrarily cut off after 100 levels of obj.__wrapped__
     attribute access. --TK, Jan 2016
     """
     orig_obj = obj
     i = 0
     while safe_hasattr(obj, '__wrapped__'):
         obj = obj.__wrapped__
         i += 1
         if i > 100:
             # __wrapped__ is probably a lie, so return the thing we started with
             return orig_obj
     return obj
 
 def find_file(obj):
     """Find the absolute path to the file where an object was defined.
 
     This is essentially a robust wrapper around `inspect.getabsfile`.
 
     Returns None if no file can be found.
 
     Parameters
     ----------
     obj : any Python object
 
     Returns
     -------
     fname : str
       The absolute path to the file where the object was defined.
     """
     obj = _get_wrapped(obj)
 
     fname = None
     try:
         fname = inspect.getabsfile(obj)
     except TypeError:
         # For an instance, the file that matters is where its class was
         # declared.
         if hasattr(obj, '__class__'):
             try:
                 fname = inspect.getabsfile(obj.__class__)
             except TypeError:
                 # Can happen for builtins
                 pass
     except:
         pass
     return cast_unicode(fname)
 
 
 def find_source_lines(obj):
     """Find the line number in a file where an object was defined.
 
     This is essentially a robust wrapper around `inspect.getsourcelines`.
 
     Returns None if no file can be found.
 
     Parameters
     ----------
     obj : any Python object
 
     Returns
     -------
     lineno : int
       The line number where the object definition starts.
     """
     obj = _get_wrapped(obj)
 
     try:
         try:
             lineno = inspect.getsourcelines(obj)[1]
         except TypeError:
             # For instances, try the class object like getsource() does
             if hasattr(obj, '__class__'):
                 lineno = inspect.getsourcelines(obj.__class__)[1]
             else:
                 lineno = None
     except:
         return None
 
     return lineno
 
 class Inspector(Colorable):
 
     def __init__(self, color_table=InspectColors,
                  code_color_table=PyColorize.ANSICodeColors,
                  scheme=None,
                  str_detail_level=0,
                  parent=None, config=None):
         super(Inspector, self).__init__(parent=parent, config=config)
         self.color_table = color_table
         self.parser = PyColorize.Parser(out='str', parent=self, style=scheme)
         self.format = self.parser.format
         self.str_detail_level = str_detail_level
         self.set_active_scheme(scheme)
 
     def _getdef(self,obj,oname=''):
         """Return the call signature for any callable object.
 
         If any exception is generated, None is returned instead and the
         exception is suppressed."""
         try:
             hdef = _render_signature(signature(obj), oname)
             return cast_unicode(hdef)
         except:
             return None
 
     def __head(self,h):
         """Return a header string with proper colors."""
         return '%s%s%s' % (self.color_table.active_colors.header,h,
                            self.color_table.active_colors.normal)
 
     def set_active_scheme(self, scheme):
         if scheme is not None:
             self.color_table.set_active_scheme(scheme)
             self.parser.color_table.set_active_scheme(scheme)
 
     def noinfo(self, msg, oname):
         """Generic message when no information is found."""
         print('No %s found' % msg, end=' ')
         if oname:
             print('for %s' % oname)
         else:
             print()
 
     def pdef(self, obj, oname=''):
         """Print the call signature for any callable object.
 
         If the object is a class, print the constructor information."""
 
         if not callable(obj):
             print('Object is not callable.')
             return
 
         header = ''
 
         if inspect.isclass(obj):
             header = self.__head('Class constructor information:\n')
 
 
         output = self._getdef(obj,oname)
         if output is None:
             self.noinfo('definition header',oname)
         else:
             print(header,self.format(output), end=' ')
 
     # In Python 3, all classes are new-style, so they all have __init__.
     @skip_doctest
     def pdoc(self, obj, oname='', formatter=None):
         """Print the docstring for any object.
 
         Optional:
         -formatter: a function to run the docstring through for specially
         formatted docstrings.
 
         Examples
         --------
 
         In [1]: class NoInit:
            ...:     pass
 
         In [2]: class NoDoc:
            ...:     def __init__(self):
            ...:         pass
 
         In [3]: %pdoc NoDoc
         No documentation found for NoDoc
 
         In [4]: %pdoc NoInit
         No documentation found for NoInit
 
         In [5]: obj = NoInit()
 
         In [6]: %pdoc obj
         No documentation found for obj
 
         In [5]: obj2 = NoDoc()
 
         In [6]: %pdoc obj2
         No documentation found for obj2
         """
 
         head = self.__head  # For convenience
         lines = []
         ds = getdoc(obj)
         if formatter:
             ds = formatter(ds).get('plain/text', ds)
         if ds:
             lines.append(head("Class docstring:"))
             lines.append(indent(ds))
         if inspect.isclass(obj) and hasattr(obj, '__init__'):
             init_ds = getdoc(obj.__init__)
             if init_ds is not None:
                 lines.append(head("Init docstring:"))
                 lines.append(indent(init_ds))
         elif hasattr(obj,'__call__'):
             call_ds = getdoc(obj.__call__)
             if call_ds:
                 lines.append(head("Call docstring:"))
                 lines.append(indent(call_ds))
 
         if not lines:
             self.noinfo('documentation',oname)
         else:
             page.page('\n'.join(lines))
 
     def psource(self, obj, oname=''):
         """Print the source code for an object."""
 
         # Flush the source cache because inspect can return out-of-date source
         linecache.checkcache()
         try:
             src = getsource(obj, oname=oname)
         except Exception:
             src = None
 
         if src is None:
             self.noinfo('source', oname)
         else:
             page.page(self.format(src))
 
     def pfile(self, obj, oname=''):
         """Show the whole file where an object was defined."""
 
         lineno = find_source_lines(obj)
         if lineno is None:
             self.noinfo('file', oname)
             return
 
         ofile = find_file(obj)
         # run contents of file through pager starting at line where the object
         # is defined, as long as the file isn't binary and is actually on the
         # filesystem.
         if ofile.endswith(('.so', '.dll', '.pyd')):
             print('File %r is binary, not printing.' % ofile)
         elif not os.path.isfile(ofile):
             print('File %r does not exist, not printing.' % ofile)
         else:
             # Print only text files, not extension binaries.  Note that
             # getsourcelines returns lineno with 1-offset and page() uses
             # 0-offset, so we must adjust.
             page.page(self.format(openpy.read_py_file(ofile, skip_encoding_cookie=False)), lineno - 1)
 
     def _format_fields(self, fields, title_width=0):
         """Formats a list of fields for display.
 
         Parameters
         ----------
         fields : list
           A list of 2-tuples: (field_title, field_content)
         title_width : int
           How many characters to pad titles to. Default to longest title.
         """
         out = []
         header = self.__head
         if title_width == 0:
             title_width = max(len(title) + 2 for title, _ in fields)
         for title, content in fields:
             if len(content.splitlines()) > 1:
                 title = header(title + ':') + '\n'
             else:
                 title = header((title + ':').ljust(title_width))
             out.append(cast_unicode(title) + cast_unicode(content))
         return "\n".join(out)
 
     def _mime_format(self, text, formatter=None):
         """Return a mime bundle representation of the input text.
 
         - if `formatter` is None, the returned mime bundle has
            a `text/plain` field, with the input text.
            a `text/html` field with a `<pre>` tag containing the input text.
 
         - if `formatter` is not None, it must be a callable transforming the
           input text into a mime bundle. Default values for `text/plain` and
           `text/html` representations are the ones described above.
 
         Note:
 
         Formatters returning strings are supported but this behavior is deprecated.
 
         """
         text = cast_unicode(text)
         defaults = {
             'text/plain': text,
             'text/html': '<pre>' + text + '</pre>'
         }
 
         if formatter is None:
             return defaults
         else:
             formatted = formatter(text)
 
             if not isinstance(formatted, dict):
                 # Handle the deprecated behavior of a formatter returning
                 # a string instead of a mime bundle.
                 return {
                     'text/plain': formatted,
                     'text/html': '<pre>' + formatted + '</pre>'
                 }
 
             else:
                 return dict(defaults, **formatted)
 
 
     def format_mime(self, bundle):
 
         text_plain = bundle['text/plain']
 
         text = ''
         heads, bodies = list(zip(*text_plain))
         _len = max(len(h) for h in heads)
 
         for head, body in zip(heads, bodies):
             body = body.strip('\n')
             delim = '\n' if '\n' in body else ' '
             text += self.__head(head+':') + (_len - len(head))*' ' +delim + body +'\n'
 
         bundle['text/plain'] = text
         return bundle
 
     def _get_info(self, obj, oname='', formatter=None, info=None, detail_level=0):
         """Retrieve an info dict and format it.
 
         Parameters
         ==========
 
         obj: any
             Object to inspect and return info from
         oname: str (default: ''):
             Name of the variable pointing to `obj`.
         formatter: callable
         info:
             already computed information
         detail_level: integer
             Granularity of detail level, if set to 1, give more information.
         """
 
         info = self._info(obj, oname=oname, info=info, detail_level=detail_level)
 
         _mime = {
             'text/plain': [],
             'text/html': '',
         }
 
         def append_field(bundle, title, key, formatter=None):
             field = info[key]
             if field is not None:
                 formatted_field = self._mime_format(field, formatter)
                 bundle['text/plain'].append((title, formatted_field['text/plain']))
                 bundle['text/html'] += '<h1>' + title + '</h1>\n' + formatted_field['text/html'] + '\n'
 
         def code_formatter(text):
             return {
                 'text/plain': self.format(text),
                 'text/html': pylight(text)
             }
 
         if info['isalias']:
             append_field(_mime, 'Repr', 'string_form')
 
         elif info['ismagic']:
             if detail_level > 0:
                 append_field(_mime, 'Source', 'source', code_formatter)
             else:
                 append_field(_mime, 'Docstring', 'docstring', formatter)
             append_field(_mime, 'File', 'file')
 
         elif info['isclass'] or is_simple_callable(obj):
             # Functions, methods, classes
             append_field(_mime, 'Signature', 'definition', code_formatter)
             append_field(_mime, 'Init signature', 'init_definition', code_formatter)
             append_field(_mime, 'Docstring', 'docstring', formatter)
             if detail_level > 0 and info['source']:
                 append_field(_mime, 'Source', 'source', code_formatter)
             else:
                 append_field(_mime, 'Init docstring', 'init_docstring', formatter)
 
             append_field(_mime, 'File', 'file')
             append_field(_mime, 'Type', 'type_name')
             append_field(_mime, 'Subclasses', 'subclasses')
 
         else:
             # General Python objects
             append_field(_mime, 'Signature', 'definition', code_formatter)
             append_field(_mime, 'Call signature', 'call_def', code_formatter)
             append_field(_mime, 'Type', 'type_name')
             append_field(_mime, 'String form', 'string_form')
 
             # Namespace
             if info['namespace'] != 'Interactive':
                 append_field(_mime, 'Namespace', 'namespace')
 
             append_field(_mime, 'Length', 'length')
             append_field(_mime, 'File', 'file')
 
             # Source or docstring, depending on detail level and whether
             # source found.
             if detail_level > 0 and info['source']:
                 append_field(_mime, 'Source', 'source', code_formatter)
             else:
                 append_field(_mime, 'Docstring', 'docstring', formatter)
 
             append_field(_mime, 'Class docstring', 'class_docstring', formatter)
             append_field(_mime, 'Init docstring', 'init_docstring', formatter)
             append_field(_mime, 'Call docstring', 'call_docstring', formatter)
 
 
         return self.format_mime(_mime)
 
     def pinfo(self, obj, oname='', formatter=None, info=None, detail_level=0, enable_html_pager=True):
         """Show detailed information about an object.
 
         Optional arguments:
 
         - oname: name of the variable pointing to the object.
 
         - formatter: callable (optional)
               A special formatter for docstrings.
 
               The formatter is a callable that takes a string as an input
               and returns either a formatted string or a mime type bundle
               in the form of a dictionary.
 
               Although the support of custom formatter returning a string
               instead of a mime type bundle is deprecated.
 
         - info: a structure with some information fields which may have been
           precomputed already.
 
         - detail_level: if set to 1, more information is given.
         """
         info = self._get_info(obj, oname, formatter, info, detail_level)
         if not enable_html_pager:
             del info['text/html']
         page.page(info)
 
     def info(self, obj, oname='', formatter=None, info=None, detail_level=0):
         """DEPRECATED. Compute a dict with detailed information about an object.
         """
         if formatter is not None:
             warnings.warn('The `formatter` keyword argument to `Inspector.info`'
                      'is deprecated as of IPython 5.0 and will have no effects.',
                       DeprecationWarning, stacklevel=2)
         return self._info(obj, oname=oname, info=info, detail_level=detail_level)
 
     def _info(self, obj, oname='', info=None, detail_level=0) -> dict:
         """Compute a dict with detailed information about an object.
 
         Parameters
         ==========
 
         obj: any
             An object to find information about
         oname: str (default: ''):
             Name of the variable pointing to `obj`.
         info: (default: None)
             A struct (dict like with attr access) with some information fields
             which may have been precomputed already.
         detail_level: int (default:0)
             If set to 1, more information is given.
 
         Returns
         =======
 
         An object info dict with known fields from `info_fields`.
         """
 
         if info is None:
             ismagic = False
             isalias = False
             ospace = ''
         else:
             ismagic = info.ismagic
             isalias = info.isalias
             ospace = info.namespace
 
         # Get docstring, special-casing aliases:
         if isalias:
             if not callable(obj):
                 try:
                     ds = "Alias to the system command:\n  %s" % obj[1]
                 except:
                     ds = "Alias: " + str(obj)
             else:
                 ds = "Alias to " + str(obj)
                 if obj.__doc__:
                     ds += "\nDocstring:\n" + obj.__doc__
         else:
             ds = getdoc(obj)
             if ds is None:
                 ds = '<no docstring>'
 
         # store output in a dict, we initialize it here and fill it as we go
         out = dict(name=oname, found=True, isalias=isalias, ismagic=ismagic, subclasses=None)
 
         string_max = 200 # max size of strings to show (snipped if longer)
         shalf = int((string_max - 5) / 2)
 
         if ismagic:
             out['type_name'] = 'Magic function'
         elif isalias:
             out['type_name'] = 'System alias'
         else:
             out['type_name'] = type(obj).__name__
 
         try:
             bclass = obj.__class__
             out['base_class'] = str(bclass)
         except:
             pass
 
         # String form, but snip if too long in ? form (full in ??)
         if detail_level >= self.str_detail_level:
             try:
                 ostr = str(obj)
                 str_head = 'string_form'
                 if not detail_level and len(ostr)>string_max:
                     ostr = ostr[:shalf] + ' <...> ' + ostr[-shalf:]
                     ostr = ("\n" + " " * len(str_head.expandtabs())).\
                             join(q.strip() for q in ostr.split("\n"))
                 out[str_head] = ostr
             except:
                 pass
 
         if ospace:
             out['namespace'] = ospace
 
         # Length (for strings and lists)
         try:
             out['length'] = str(len(obj))
         except Exception:
             pass
 
         # Filename where object was defined
         binary_file = False
         fname = find_file(obj)
         if fname is None:
             # if anything goes wrong, we don't want to show source, so it's as
             # if the file was binary
             binary_file = True
         else:
             if fname.endswith(('.so', '.dll', '.pyd')):
                 binary_file = True
             elif fname.endswith('<string>'):
                 fname = 'Dynamically generated function. No source code available.'
             out['file'] = compress_user(fname)
 
         # Original source code for a callable, class or property.
         if detail_level:
             # Flush the source cache because inspect can return out-of-date
             # source
             linecache.checkcache()
             try:
                 if isinstance(obj, property) or not binary_file:
                     src = getsource(obj, oname)
                     if src is not None:
                         src = src.rstrip()
                     out['source'] = src
 
             except Exception:
                 pass
 
         # Add docstring only if no source is to be shown (avoid repetitions).
         if ds and not self._source_contains_docstring(out.get('source'), ds):
             out['docstring'] = ds
 
         # Constructor docstring for classes
         if inspect.isclass(obj):
             out['isclass'] = True
 
             # get the init signature:
             try:
                 init_def = self._getdef(obj, oname)
             except AttributeError:
                 init_def = None
 
             # get the __init__ docstring
             try:
                 obj_init = obj.__init__
             except AttributeError:
                 init_ds = None
             else:
                 if init_def is None:
                     # Get signature from init if top-level sig failed.
                     # Can happen for built-in types (list, etc.).
                     try:
                         init_def = self._getdef(obj_init, oname)
                     except AttributeError:
                         pass
                 init_ds = getdoc(obj_init)
                 # Skip Python's auto-generated docstrings
                 if init_ds == _object_init_docstring:
                     init_ds = None
 
             if init_def:
                 out['init_definition'] = init_def
 
             if init_ds:
                 out['init_docstring'] = init_ds
 
             names = [sub.__name__ for sub in type.__subclasses__(obj)]
             if len(names) < 10:
                 all_names = ', '.join(names)
             else:
                 all_names = ', '.join(names[:10]+['...'])
             out['subclasses'] = all_names
         # and class docstring for instances:
         else:
             # reconstruct the function definition and print it:
             defln = self._getdef(obj, oname)
             if defln:
                 out['definition'] = defln
 
             # First, check whether the instance docstring is identical to the
             # class one, and print it separately if they don't coincide.  In
             # most cases they will, but it's nice to print all the info for
             # objects which use instance-customized docstrings.
             if ds:
                 try:
                     cls = getattr(obj,'__class__')
                 except:
                     class_ds = None
                 else:
                     class_ds = getdoc(cls)
                 # Skip Python's auto-generated docstrings
                 if class_ds in _builtin_type_docstrings:
                     class_ds = None
                 if class_ds and ds != class_ds:
                     out['class_docstring'] = class_ds
 
             # Next, try to show constructor docstrings
             try:
                 init_ds = getdoc(obj.__init__)
                 # Skip Python's auto-generated docstrings
                 if init_ds == _object_init_docstring:
                     init_ds = None
             except AttributeError:
                 init_ds = None
             if init_ds:
                 out['init_docstring'] = init_ds
 
             # Call form docstring for callable instances
             if safe_hasattr(obj, '__call__') and not is_simple_callable(obj):
                 call_def = self._getdef(obj.__call__, oname)
                 if call_def and (call_def != out.get('definition')):
                     # it may never be the case that call def and definition differ,
                     # but don't include the same signature twice
                     out['call_def'] = call_def
                 call_ds = getdoc(obj.__call__)
                 # Skip Python's auto-generated docstrings
                 if call_ds == _func_call_docstring:
                     call_ds = None
                 if call_ds:
                     out['call_docstring'] = call_ds
 
-        # Compute the object's argspec as a callable.  The key is to decide
-        # whether to pull it from the object itself, from its __init__ or
-        # from its __call__ method.
-
-        if inspect.isclass(obj):
-            # Old-style classes need not have an __init__
-            callable_obj = getattr(obj, "__init__", None)
-        elif callable(obj):
-            callable_obj = obj
-        else:
-            callable_obj = None
-
-        if callable_obj is not None:
-            try:
-                argspec = getargspec(callable_obj)
-            except Exception:
-                # For extensions/builtins we can't retrieve the argspec
-                pass
-            else:
-                # named tuples' _asdict() method returns an OrderedDict, but we
-                # we want a normal
-                out['argspec'] = argspec_dict = dict(argspec._asdict())
-                # We called this varkw before argspec became a named tuple.
-                # With getfullargspec it's also called varkw.
-                if 'varkw' not in argspec_dict:
-                    argspec_dict['varkw'] = argspec_dict.pop('keywords')
-
         return object_info(**out)
 
     @staticmethod
     def _source_contains_docstring(src, doc):
         """
         Check whether the source *src* contains the docstring *doc*.
 
         This is is helper function to skip displaying the docstring if the
         source already contains it, avoiding repetition of information.
         """
         try:
             def_node, = ast.parse(dedent(src)).body
             return ast.get_docstring(def_node) == doc
         except Exception:
             # The source can become invalid or even non-existent (because it
             # is re-fetched from the source file) so the above code fail in
             # arbitrary ways.
             return False
 
     def psearch(self,pattern,ns_table,ns_search=[],
                 ignore_case=False,show_all=False, *, list_types=False):
         """Search namespaces with wildcards for objects.
 
         Arguments:
 
         - pattern: string containing shell-like wildcards to use in namespace
           searches and optionally a type specification to narrow the search to
           objects of that type.
 
         - ns_table: dict of name->namespaces for search.
 
         Optional arguments:
 
           - ns_search: list of namespace names to include in search.
 
           - ignore_case(False): make the search case-insensitive.
 
           - show_all(False): show all names, including those starting with
             underscores.
             
           - list_types(False): list all available object types for object matching.
         """
         #print 'ps pattern:<%r>' % pattern # dbg
 
         # defaults
         type_pattern = 'all'
         filter = ''
 
         # list all object types
         if list_types:
             page.page('\n'.join(sorted(typestr2type)))
             return
 
         cmds = pattern.split()
         len_cmds  =  len(cmds)
         if len_cmds == 1:
             # Only filter pattern given
             filter = cmds[0]
         elif len_cmds == 2:
             # Both filter and type specified
             filter,type_pattern = cmds
         else:
             raise ValueError('invalid argument string for psearch: <%s>' %
                              pattern)
 
         # filter search namespaces
         for name in ns_search:
             if name not in ns_table:
                 raise ValueError('invalid namespace <%s>. Valid names: %s' %
                                  (name,ns_table.keys()))
 
         #print 'type_pattern:',type_pattern # dbg
         search_result, namespaces_seen = set(), set()
         for ns_name in ns_search:
             ns = ns_table[ns_name]
             # Normally, locals and globals are the same, so we just check one.
             if id(ns) in namespaces_seen:
                 continue
             namespaces_seen.add(id(ns))
             tmp_res = list_namespace(ns, type_pattern, filter,
                                     ignore_case=ignore_case, show_all=show_all)
             search_result.update(tmp_res)
 
         page.page('\n'.join(sorted(search_result)))
 
 
 def _render_signature(obj_signature, obj_name):
     """
     This was mostly taken from inspect.Signature.__str__.
     Look there for the comments.
     The only change is to add linebreaks when this gets too long.
     """
     result = []
     pos_only = False
     kw_only = True
     for param in obj_signature.parameters.values():
         if param.kind == inspect._POSITIONAL_ONLY:
             pos_only = True
         elif pos_only:
             result.append('/')
             pos_only = False
 
         if param.kind == inspect._VAR_POSITIONAL:
             kw_only = False
         elif param.kind == inspect._KEYWORD_ONLY and kw_only:
             result.append('*')
             kw_only = False
 
         result.append(str(param))
 
     if pos_only:
         result.append('/')
 
     # add up name, parameters, braces (2), and commas
     if len(obj_name) + sum(len(r) + 2 for r in result) > 75:
         # This doesn’t fit behind “Signature: ” in an inspect window.
         rendered = '{}(\n{})'.format(obj_name, ''.join(
             '    {},\n'.format(r) for r in result)
         )
     else:
         rendered = '{}({})'.format(obj_name, ', '.join(result))
 
     if obj_signature.return_annotation is not inspect._empty:
         anno = inspect.formatannotation(obj_signature.return_annotation)
         rendered += ' -> {}'.format(anno)
 
     return rendered

IPython/core/page.py

@@ -1,366 +1,372 b''
 # encoding: utf-8
 """
 Paging capabilities for IPython.core
 
 Notes
 -----
 
 For now this uses IPython hooks, so it can't be in IPython.utils.  If we can get
 rid of that dependency, we could move it there.
 -----
 """
 
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
 
 import os
+import io
 import re
 import sys
 import tempfile
+import subprocess
 
 from io import UnsupportedOperation
 
 from IPython import get_ipython
 from IPython.core.display import display
 from IPython.core.error import TryNext
 from IPython.utils.data import chop
 from IPython.utils.process import system
 from IPython.utils.terminal import get_terminal_size
 from IPython.utils import py3compat
 
 
 def display_page(strng, start=0, screen_lines=25):
     """Just display, no paging. screen_lines is ignored."""
     if isinstance(strng, dict):
         data = strng
     else:
         if start:
             strng = u'\n'.join(strng.splitlines()[start:])
         data = { 'text/plain': strng }
     display(data, raw=True)
 
 
 def as_hook(page_func):
     """Wrap a pager func to strip the `self` arg
     
     so it can be called as a hook.
     """
     return lambda self, *args, **kwargs: page_func(*args, **kwargs)
 
 
 esc_re = re.compile(r"(\x1b[^m]+m)")
 
 def page_dumb(strng, start=0, screen_lines=25):
     """Very dumb 'pager' in Python, for when nothing else works.
 
     Only moves forward, same interface as page(), except for pager_cmd and
     mode.
     """
     if isinstance(strng, dict):
         strng = strng.get('text/plain', '')
     out_ln  = strng.splitlines()[start:]
     screens = chop(out_ln,screen_lines-1)
     if len(screens) == 1:
         print(os.linesep.join(screens[0]))
     else:
         last_escape = ""
         for scr in screens[0:-1]:
             hunk = os.linesep.join(scr)
             print(last_escape + hunk)
             if not page_more():
                 return
             esc_list = esc_re.findall(hunk)
             if len(esc_list) > 0:
                 last_escape = esc_list[-1]
         print(last_escape + os.linesep.join(screens[-1]))
 
 def _detect_screen_size(screen_lines_def):
     """Attempt to work out the number of lines on the screen.
 
     This is called by page(). It can raise an error (e.g. when run in the
     test suite), so it's separated out so it can easily be called in a try block.
     """
     TERM = os.environ.get('TERM',None)
     if not((TERM=='xterm' or TERM=='xterm-color') and sys.platform != 'sunos5'):
         # curses causes problems on many terminals other than xterm, and
         # some termios calls lock up on Sun OS5.
         return screen_lines_def
     
     try:
         import termios
         import curses
     except ImportError:
         return screen_lines_def
     
     # There is a bug in curses, where *sometimes* it fails to properly
     # initialize, and then after the endwin() call is made, the
     # terminal is left in an unusable state.  Rather than trying to
     # check every time for this (by requesting and comparing termios
     # flags each time), we just save the initial terminal state and
     # unconditionally reset it every time.  It's cheaper than making
     # the checks.
     try:
         term_flags = termios.tcgetattr(sys.stdout)
     except termios.error as err:
         # can fail on Linux 2.6, pager_page will catch the TypeError
         raise TypeError('termios error: {0}'.format(err))
 
     try:
         scr = curses.initscr()
     except AttributeError:
         # Curses on Solaris may not be complete, so we can't use it there
         return screen_lines_def
     
     screen_lines_real,screen_cols = scr.getmaxyx()
     curses.endwin()
 
     # Restore terminal state in case endwin() didn't.
     termios.tcsetattr(sys.stdout,termios.TCSANOW,term_flags)
     # Now we have what we needed: the screen size in rows/columns
     return screen_lines_real
     #print '***Screen size:',screen_lines_real,'lines x',\
     #screen_cols,'columns.' # dbg
 
 def pager_page(strng, start=0, screen_lines=0, pager_cmd=None):
     """Display a string, piping through a pager after a certain length.
     
     strng can be a mime-bundle dict, supplying multiple representations,
     keyed by mime-type.
 
     The screen_lines parameter specifies the number of *usable* lines of your
     terminal screen (total lines minus lines you need to reserve to show other
     information).
 
     If you set screen_lines to a number <=0, page() will try to auto-determine
     your screen size and will only use up to (screen_size+screen_lines) for
     printing, paging after that. That is, if you want auto-detection but need
     to reserve the bottom 3 lines of the screen, use screen_lines = -3, and for
     auto-detection without any lines reserved simply use screen_lines = 0.
 
     If a string won't fit in the allowed lines, it is sent through the
     specified pager command. If none given, look for PAGER in the environment,
     and ultimately default to less.
 
     If no system pager works, the string is sent through a 'dumb pager'
     written in python, very simplistic.
     """
     
     # for compatibility with mime-bundle form:
     if isinstance(strng, dict):
         strng = strng['text/plain']
 
     # Ugly kludge, but calling curses.initscr() flat out crashes in emacs
     TERM = os.environ.get('TERM','dumb')
     if TERM in ['dumb','emacs'] and os.name != 'nt':
         print(strng)
         return
     # chop off the topmost part of the string we don't want to see
     str_lines = strng.splitlines()[start:]
     str_toprint = os.linesep.join(str_lines)
     num_newlines = len(str_lines)
     len_str = len(str_toprint)
 
     # Dumb heuristics to guesstimate number of on-screen lines the string
     # takes.  Very basic, but good enough for docstrings in reasonable
     # terminals. If someone later feels like refining it, it's not hard.
     numlines = max(num_newlines,int(len_str/80)+1)
 
     screen_lines_def = get_terminal_size()[1]
 
     # auto-determine screen size
     if screen_lines <= 0:
         try:
             screen_lines += _detect_screen_size(screen_lines_def)
         except (TypeError, UnsupportedOperation):
             print(str_toprint)
             return
 
     #print 'numlines',numlines,'screenlines',screen_lines  # dbg
     if numlines <= screen_lines :
         #print '*** normal print'  # dbg
         print(str_toprint)
     else:
         # Try to open pager and default to internal one if that fails.
         # All failure modes are tagged as 'retval=1', to match the return
         # value of a failed system command.  If any intermediate attempt
         # sets retval to 1, at the end we resort to our own page_dumb() pager.
         pager_cmd = get_pager_cmd(pager_cmd)
         pager_cmd += ' ' + get_pager_start(pager_cmd,start)
         if os.name == 'nt':
             if pager_cmd.startswith('type'):
                 # The default WinXP 'type' command is failing on complex strings.
                 retval = 1
             else:
                 fd, tmpname = tempfile.mkstemp('.txt')
                 try:
                     os.close(fd)
                     with open(tmpname, 'wt') as tmpfile:
                         tmpfile.write(strng)
                         cmd = "%s < %s" % (pager_cmd, tmpname)
                     # tmpfile needs to be closed for windows
                     if os.system(cmd):
                         retval = 1
                     else:
                         retval = None
                 finally:
                     os.remove(tmpname)
         else:
             try:
                 retval = None
-                # if I use popen4, things hang. No idea why.
-                #pager,shell_out = os.popen4(pager_cmd)
-                pager = os.popen(pager_cmd, 'w')
+                # Emulate os.popen, but redirect stderr
+                proc = subprocess.Popen(pager_cmd,
+                                shell=True,
+                                stdin=subprocess.PIPE,
+                                stderr=subprocess.DEVNULL
+                                )
+                pager = os._wrap_close(io.TextIOWrapper(proc.stdin), proc)
                 try:
                     pager_encoding = pager.encoding or sys.stdout.encoding
                     pager.write(strng)
                 finally:
                     retval = pager.close()
             except IOError as msg:  # broken pipe when user quits
                 if msg.args == (32, 'Broken pipe'):
                     retval = None
                 else:
                     retval = 1
             except OSError:
                 # Other strange problems, sometimes seen in Win2k/cygwin
                 retval = 1
         if retval is not None:
             page_dumb(strng,screen_lines=screen_lines)
 
 
 def page(data, start=0, screen_lines=0, pager_cmd=None):
     """Display content in a pager, piping through a pager after a certain length.
     
     data can be a mime-bundle dict, supplying multiple representations,
     keyed by mime-type, or text.
     
     Pager is dispatched via the `show_in_pager` IPython hook.
     If no hook is registered, `pager_page` will be used.
     """
     # Some routines may auto-compute start offsets incorrectly and pass a
     # negative value.  Offset to 0 for robustness.
     start = max(0, start)
 
     # first, try the hook
     ip = get_ipython()
     if ip:
         try:
             ip.hooks.show_in_pager(data, start=start, screen_lines=screen_lines)
             return
         except TryNext:
             pass
     
     # fallback on default pager
     return pager_page(data, start, screen_lines, pager_cmd)
 
 
 def page_file(fname, start=0, pager_cmd=None):
     """Page a file, using an optional pager command and starting line.
     """
 
     pager_cmd = get_pager_cmd(pager_cmd)
     pager_cmd += ' ' + get_pager_start(pager_cmd,start)
 
     try:
         if os.environ['TERM'] in ['emacs','dumb']:
             raise EnvironmentError
         system(pager_cmd + ' ' + fname)
     except:
         try:
             if start > 0:
                 start -= 1
             page(open(fname).read(),start)
         except:
             print('Unable to show file',repr(fname))
 
 
 def get_pager_cmd(pager_cmd=None):
     """Return a pager command.
 
     Makes some attempts at finding an OS-correct one.
     """
     if os.name == 'posix':
         default_pager_cmd = 'less -R'  # -R for color control sequences
     elif os.name in ['nt','dos']:
         default_pager_cmd = 'type'
 
     if pager_cmd is None:
         try:
             pager_cmd = os.environ['PAGER']
         except:
             pager_cmd = default_pager_cmd
     
     if pager_cmd == 'less' and '-r' not in os.environ.get('LESS', '').lower():
         pager_cmd += ' -R'
     
     return pager_cmd
 
 
 def get_pager_start(pager, start):
     """Return the string for paging files with an offset.
 
     This is the '+N' argument which less and more (under Unix) accept.
     """
 
     if pager in ['less','more']:
         if start:
             start_string = '+' + str(start)
         else:
             start_string = ''
     else:
         start_string = ''
     return start_string
 
 
 # (X)emacs on win32 doesn't like to be bypassed with msvcrt.getch()
 if os.name == 'nt' and os.environ.get('TERM','dumb') != 'emacs':
     import msvcrt
     def page_more():
         """ Smart pausing between pages
 
         @return:    True if need print more lines, False if quit
         """
         sys.stdout.write('---Return to continue, q to quit--- ')
         ans = msvcrt.getwch()
         if ans in ("q", "Q"):
             result = False
         else:
             result = True
         sys.stdout.write("\b"*37 + " "*37 + "\b"*37)
         return result
 else:
     def page_more():
         ans = py3compat.input('---Return to continue, q to quit--- ')
         if ans.lower().startswith('q'):
             return False
         else:
             return True
 
 
 def snip_print(str,width = 75,print_full = 0,header = ''):
     """Print a string snipping the midsection to fit in width.
 
     print_full: mode control:
     
       - 0: only snip long strings
       - 1: send to page() directly.
       - 2: snip long strings and ask for full length viewing with page()
     
     Return 1 if snipping was necessary, 0 otherwise."""
 
     if print_full == 1:
         page(header+str)
         return 0
 
     print(header, end=' ')
     if len(str) < width:
         print(str)
         snip = 0
     else:
         whalf = int((width -5)/2)
         print(str[:whalf] + ' <...> ' + str[-whalf:])
         snip = 1
     if snip and print_full == 2:
         if py3compat.input(header+' Snipped. View (y/n)? [N]').lower() == 'y':
             page(str)
     return snip

IPython/core/release.py

@@ -1,119 +1,119 b''
 # -*- coding: utf-8 -*-
 """Release data for the IPython project."""
 
 #-----------------------------------------------------------------------------
 #  Copyright (c) 2008, IPython Development Team.
 #  Copyright (c) 2001, Fernando Perez <[e-mail]>
 #  Copyright (c) 2001, Janko Hauser <[e-mail]>
 #  Copyright (c) 2001, Nathaniel Gray <[e-mail]>
 #
 #  Distributed under the terms of the Modified BSD License.
 #
 #  The full license is in the file COPYING.txt, distributed with this software.
 #-----------------------------------------------------------------------------
 
 # Name of the package for release purposes.  This is the name which labels
 # the tarballs and RPMs made by distutils, so it's best to lowercase it.
 name = 'ipython'
 
 # IPython version information.  An empty _version_extra corresponds to a full
 # release.  'dev' as a _version_extra string means this is a development
 # version
 _version_major = 7
-_version_minor = 10
+_version_minor = 11
 _version_patch = 0
 _version_extra = '.dev'
 # _version_extra = 'b1'
 # _version_extra = ''  # Uncomment this for full releases
 
 # Construct full version string from these.
 _ver = [_version_major, _version_minor, _version_patch]
 
 __version__ = '.'.join(map(str, _ver))
 if _version_extra:
     __version__ = __version__  + _version_extra
 
 version = __version__  # backwards compatibility name
 version_info = (_version_major, _version_minor, _version_patch, _version_extra)
 
 # Change this when incrementing the kernel protocol version
 kernel_protocol_version_info = (5, 0)
 kernel_protocol_version = "%i.%i" % kernel_protocol_version_info
 
 description = "IPython: Productive Interactive Computing"
 
 long_description = \
 """
 IPython provides a rich toolkit to help you make the most out of using Python
 interactively.  Its main components are:
 
 * A powerful interactive Python shell
 * A `Jupyter <https://jupyter.org/>`_ kernel to work with Python code in Jupyter
   notebooks and other interactive frontends.
 
 The enhanced interactive Python shells have the following main features:
 
 * Comprehensive object introspection.
 
 * Input history, persistent across sessions.
 
 * Caching of output results during a session with automatically generated
   references.
 
 * Extensible tab completion, with support by default for completion of python
   variables and keywords, filenames and function keywords.
 
 * Extensible system of 'magic' commands for controlling the environment and
   performing many tasks related either to IPython or the operating system.
 
 * A rich configuration system with easy switching between different setups
   (simpler than changing $PYTHONSTARTUP environment variables every time).
 
 * Session logging and reloading.
 
 * Extensible syntax processing for special purpose situations.
 
 * Access to the system shell with user-extensible alias system.
 
 * Easily embeddable in other Python programs and GUIs.
 
 * Integrated access to the pdb debugger and the Python profiler.
 
 The latest development version is always available from IPython's `GitHub
 site <http://github.com/ipython>`_.
 """
 
 license = 'BSD'
 
 authors = {'Fernando' : ('Fernando Perez','[e-mail]'),
            'Janko'    : ('Janko Hauser','[e-mail]'),
            'Nathan'   : ('Nathaniel Gray','[e-mail]'),
            'Ville'    : ('Ville Vainio','[e-mail]'),
            'Brian'    : ('Brian E Granger', '[e-mail]'),
            'Min'      : ('Min Ragan-Kelley', '[e-mail]'),
            'Thomas'   : ('Thomas A. Kluyver', '[e-mail]'),
            'Jorgen'   : ('Jorgen Stenarson', '[e-mail]'),
            'Matthias' : ('Matthias Bussonnier', '[e-mail]'),
            }
 
 author = 'The IPython Development Team'
 
 author_email = '[e-mail]'
 
 url = 'https://ipython.org'
 
 
 platforms = ['Linux','Mac OSX','Windows']
 
 keywords = ['Interactive','Interpreter','Shell', 'Embedding']
 
 classifiers = [
     'Framework :: IPython',
     'Intended Audience :: Developers',
     'Intended Audience :: Science/Research',
     'License :: OSI Approved :: BSD License',
     'Programming Language :: Python',
     'Programming Language :: Python :: 3',
     'Programming Language :: Python :: 3 :: Only',
     'Topic :: System :: Shells'
     ]

IPython/core/tests/test_autocall.py

@@ -1,67 +1,66 b''
 """These kinds of tests are less than ideal, but at least they run.
 
 This was an old test that was being run interactively in the top-level tests/
 directory, which we are removing.  For now putting this here ensures at least
 we do run the test, though ultimately this functionality should all be tested
 with better-isolated tests that don't rely on the global instance in iptest.
 """
 from IPython.core.splitinput import LineInfo
 from IPython.core.prefilter import AutocallChecker
-from IPython.utils import py3compat
 
 def doctest_autocall():
     """
     In [1]: def f1(a,b,c):
        ...:     return a+b+c
        ...:
 
     In [2]: def f2(a):
        ...:     return a + a
        ...:
 
     In [3]: def r(x):
        ...:     return True
        ...:
 
     In [4]: ;f2 a b c
     Out[4]: 'a b ca b c'
 
     In [5]: assert _ == "a b ca b c"
 
     In [6]: ,f1 a b c
     Out[6]: 'abc'
 
     In [7]: assert _ == 'abc'
 
     In [8]: print(_)
     abc
 
     In [9]: /f1 1,2,3
     Out[9]: 6
 
     In [10]: assert _ == 6
 
     In [11]: /f2 4
     Out[11]: 8
 
     In [12]: assert _ == 8
 
     In [12]: del f1, f2
 
     In [13]: ,r a
     Out[13]: True
 
     In [14]: assert _ == True
 
     In [15]: r'a'
     Out[15]: 'a'
 
     In [16]: assert _ == 'a'
     """
 
 
 def test_autocall_should_ignore_raw_strings():
     line_info = LineInfo("r'a'")
     pm = ip.prefilter_manager
     ac = AutocallChecker(shell=pm.shell, prefilter_manager=pm, config=pm.config)
     assert ac.check(line_info) is None

IPython/core/tests/test_handlers.py

@@ -1,95 +1,95 b''
 """Tests for input handlers.
 """
 #-----------------------------------------------------------------------------
 # Module imports
 #-----------------------------------------------------------------------------
 
 # third party
 import nose.tools as nt
 
 # our own packages
 from IPython.core import autocall
 from IPython.testing import tools as tt
 from IPython.utils import py3compat
 
 #-----------------------------------------------------------------------------
 # Globals
 #-----------------------------------------------------------------------------
 
 # Get the public instance of IPython
 
 failures = []
 num_tests = 0
 
 #-----------------------------------------------------------------------------
 # Test functions
 #-----------------------------------------------------------------------------
 
 class CallableIndexable(object):
     def __getitem__(self, idx): return True
     def __call__(self, *args, **kws): return True
 
 
 class Autocallable(autocall.IPyAutocall):
     def __call__(self):
         return "called"
 
 
 def run(tests):
     """Loop through a list of (pre, post) inputs, where pre is the string
     handed to ipython, and post is how that string looks after it's been
     transformed (i.e. ipython's notion of _i)"""
     tt.check_pairs(ip.prefilter_manager.prefilter_lines, tests)
 
 
 def test_handlers():
     call_idx = CallableIndexable()
     ip.user_ns['call_idx'] = call_idx
 
     # For many of the below, we're also checking that leading whitespace
     # turns off the esc char, which it should unless there is a continuation
     # line.
-    run([(i,py3compat.u_format(o)) for i,o in \
+    run(
         [('"no change"', '"no change"'),             # normal
          (u"lsmagic",     "get_ipython().run_line_magic('lsmagic', '')"),   # magic
          #("a = b # PYTHON-MODE", '_i'),          # emacs -- avoids _in cache
-         ]])
+         ])
 
     # Objects which are instances of IPyAutocall are *always* autocalled
     autocallable = Autocallable()
     ip.user_ns['autocallable'] = autocallable
 
     # auto
     ip.magic('autocall 0')
     # Only explicit escapes or instances of IPyAutocallable should get
     # expanded
     run([
         ('len "abc"',       'len "abc"'),
         ('autocallable',    'autocallable()'),
         # Don't add extra brackets (gh-1117)
         ('autocallable()',    'autocallable()'),
         ])
     ip.magic('autocall 1')
     run([
         ('len "abc"', 'len("abc")'),
         ('len "abc";', 'len("abc");'),  # ; is special -- moves out of parens
         # Autocall is turned off if first arg is [] and the object
         # is both callable and indexable.  Like so:
         ('len [1,2]', 'len([1,2])'),      # len doesn't support __getitem__...
         ('call_idx [1]', 'call_idx [1]'), # call_idx *does*..
         ('call_idx 1', 'call_idx(1)'),
         ('len', 'len'), # only at 2 does it auto-call on single args
         ])
     ip.magic('autocall 2')
     run([
         ('len "abc"', 'len("abc")'),
         ('len "abc";', 'len("abc");'),
         ('len [1,2]', 'len([1,2])'),
         ('call_idx [1]', 'call_idx [1]'),
         ('call_idx 1', 'call_idx(1)'),
         # This is what's different:
         ('len', 'len()'), # only at 2 does it auto-call on single args
         ])
     ip.magic('autocall 1')
 
     nt.assert_equal(failures, [])

IPython/core/tests/test_run.py

@@ -1,559 +1,588 b''
 # encoding: utf-8
 """Tests for code execution (%run and related), which is particularly tricky.
 
 Because of how %run manages namespaces, and the fact that we are trying here to
 verify subtle object deletion and reference counting issues, the %run tests
 will be kept in this separate file.  This makes it easier to aggregate in one
 place the tricks needed to handle it; most other magics are much easier to test
 and we do so in a common test_magic file.
 
 Note that any test using `run -i` should make sure to do a `reset` afterwards,
 as otherwise it may influence later tests.
 """
 
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
 
 
 import functools
 import os
 from os.path import join as pjoin
 import random
 import string
 import sys
 import textwrap
 import unittest
 from unittest.mock import patch
 
 import nose.tools as nt
 from nose import SkipTest
 
 from IPython.testing import decorators as dec
 from IPython.testing import tools as tt
 from IPython.utils.io import capture_output
 from IPython.utils.tempdir import TemporaryDirectory
 from IPython.core import debugger
 
 def doctest_refbug():
     """Very nasty problem with references held by multiple runs of a script.
     See: https://github.com/ipython/ipython/issues/141
 
     In [1]: _ip.clear_main_mod_cache()
     # random
 
     In [2]: %run refbug
 
     In [3]: call_f()
     lowercased: hello
 
     In [4]: %run refbug
 
     In [5]: call_f()
     lowercased: hello
     lowercased: hello
     """
 
 
 def doctest_run_builtins():
     r"""Check that %run doesn't damage __builtins__.
 
     In [1]: import tempfile
 
     In [2]: bid1 = id(__builtins__)
 
     In [3]: fname = tempfile.mkstemp('.py')[1]
 
     In [3]: f = open(fname,'w')
 
     In [4]: dummy= f.write('pass\n')
 
     In [5]: f.flush()
 
     In [6]: t1 = type(__builtins__)
 
     In [7]: %run $fname
 
     In [7]: f.close()
 
     In [8]: bid2 = id(__builtins__)
 
     In [9]: t2 = type(__builtins__)
 
     In [10]: t1 == t2
     Out[10]: True
 
     In [10]: bid1 == bid2
     Out[10]: True
 
     In [12]: try:
        ....:     os.unlink(fname)
        ....: except:
        ....:     pass
        ....:
     """
 
 
 def doctest_run_option_parser():
     r"""Test option parser in %run.
 
     In [1]: %run print_argv.py
     []
 
     In [2]: %run print_argv.py print*.py
     ['print_argv.py']
 
     In [3]: %run -G print_argv.py print*.py
     ['print*.py']
 
     """
 
 
 @dec.skip_win32
 def doctest_run_option_parser_for_posix():
     r"""Test option parser in %run (Linux/OSX specific).
 
     You need double quote to escape glob in POSIX systems:
 
     In [1]: %run print_argv.py print\\*.py
     ['print*.py']
 
     You can't use quote to escape glob in POSIX systems:
 
     In [2]: %run print_argv.py 'print*.py'
     ['print_argv.py']
 
     """
 
 
 @dec.skip_if_not_win32
 def doctest_run_option_parser_for_windows():
     r"""Test option parser in %run (Windows specific).
 
     In Windows, you can't escape ``*` `by backslash:
 
     In [1]: %run print_argv.py print\\*.py
     ['print\\*.py']
 
     You can use quote to escape glob:
 
     In [2]: %run print_argv.py 'print*.py'
     ['print*.py']
 
     """
 
 
 def doctest_reset_del():
     """Test that resetting doesn't cause errors in __del__ methods.
 
     In [2]: class A(object):
        ...:     def __del__(self):
        ...:         print(str("Hi"))
        ...:
 
     In [3]: a = A()
 
     In [4]: get_ipython().reset()
     Hi
 
     In [5]: 1+1
     Out[5]: 2
     """
 
 # For some tests, it will be handy to organize them in a class with a common
 # setup that makes a temp file
 
 class TestMagicRunPass(tt.TempFileMixin):
 
     def setUp(self):
         content = "a = [1,2,3]\nb = 1"
         self.mktmp(content)
         
     def run_tmpfile(self):
         _ip = get_ipython()
         # This fails on Windows if self.tmpfile.name has spaces or "~" in it.
         # See below and ticket https://bugs.launchpad.net/bugs/366353
         _ip.magic('run %s' % self.fname)
         
     def run_tmpfile_p(self):
         _ip = get_ipython()
         # This fails on Windows if self.tmpfile.name has spaces or "~" in it.
         # See below and ticket https://bugs.launchpad.net/bugs/366353
         _ip.magic('run -p %s' % self.fname)
 
     def test_builtins_id(self):
         """Check that %run doesn't damage __builtins__ """
         _ip = get_ipython()
         # Test that the id of __builtins__ is not modified by %run
         bid1 = id(_ip.user_ns['__builtins__'])
         self.run_tmpfile()
         bid2 = id(_ip.user_ns['__builtins__'])
         nt.assert_equal(bid1, bid2)
 
     def test_builtins_type(self):
         """Check that the type of __builtins__ doesn't change with %run.
 
         However, the above could pass if __builtins__ was already modified to
         be a dict (it should be a module) by a previous use of %run.  So we
         also check explicitly that it really is a module:
         """
         _ip = get_ipython()
         self.run_tmpfile()
         nt.assert_equal(type(_ip.user_ns['__builtins__']),type(sys))
         
     def test_run_profile( self ):
         """Test that the option -p, which invokes the profiler, do not
         crash by invoking execfile"""
         self.run_tmpfile_p()
 
     def test_run_debug_twice(self):
         # https://github.com/ipython/ipython/issues/10028
         _ip = get_ipython()
         with tt.fake_input(['c']):
             _ip.magic('run -d %s' % self.fname)
         with tt.fake_input(['c']):
             _ip.magic('run -d %s' % self.fname)
 
     def test_run_debug_twice_with_breakpoint(self):
         """Make a valid python temp file."""
         _ip = get_ipython()
         with tt.fake_input(['b 2', 'c', 'c']):
             _ip.magic('run -d %s' % self.fname)
 
         with tt.fake_input(['c']):
             with tt.AssertNotPrints('KeyError'):
                 _ip.magic('run -d %s' % self.fname)
 
 
 class TestMagicRunSimple(tt.TempFileMixin):
 
     def test_simpledef(self):
         """Test that simple class definitions work."""
         src = ("class foo: pass\n"
                "def f(): return foo()")
         self.mktmp(src)
         _ip.magic('run %s' % self.fname)
         _ip.run_cell('t = isinstance(f(), foo)')
         nt.assert_true(_ip.user_ns['t'])
 
     def test_obj_del(self):
         """Test that object's __del__ methods are called on exit."""
         if sys.platform == 'win32':
             try:
                 import win32api
             except ImportError:
                 raise SkipTest("Test requires pywin32")
         src = ("class A(object):\n"
                "    def __del__(self):\n"
                "        print('object A deleted')\n"
                "a = A()\n")
         self.mktmp(src)
         if dec.module_not_available('sqlite3'):
             err = 'WARNING: IPython History requires SQLite, your history will not be saved\n'
         else:
             err = None
         tt.ipexec_validate(self.fname, 'object A deleted', err)
     
     def test_aggressive_namespace_cleanup(self):
         """Test that namespace cleanup is not too aggressive GH-238
 
         Returning from another run magic deletes the namespace"""
         # see ticket https://github.com/ipython/ipython/issues/238
         
         with tt.TempFileMixin() as empty:
             empty.mktmp('')
             # On Windows, the filename will have \users in it, so we need to use the
             # repr so that the \u becomes \\u.
             src = ("ip = get_ipython()\n"
                    "for i in range(5):\n"
                    "   try:\n"
                    "       ip.magic(%r)\n"
                    "   except NameError as e:\n"
                    "       print(i)\n"
                    "       break\n" % ('run ' + empty.fname))
             self.mktmp(src)
             _ip.magic('run %s' % self.fname)
             _ip.run_cell('ip == get_ipython()')
             nt.assert_equal(_ip.user_ns['i'], 4)
     
     def test_run_second(self):
         """Test that running a second file doesn't clobber the first, gh-3547
         """
         self.mktmp("avar = 1\n"
                    "def afunc():\n"
                    "  return avar\n")
 
         with tt.TempFileMixin() as empty:
             empty.mktmp("")
             
             _ip.magic('run %s' % self.fname)
             _ip.magic('run %s' % empty.fname)
             nt.assert_equal(_ip.user_ns['afunc'](), 1)
 
     @dec.skip_win32
     def test_tclass(self):
         mydir = os.path.dirname(__file__)
         tc = os.path.join(mydir, 'tclass')
         src = ("%%run '%s' C-first\n"
                "%%run '%s' C-second\n"
                "%%run '%s' C-third\n") % (tc, tc, tc)
         self.mktmp(src, '.ipy')
         out = """\
 ARGV 1-: ['C-first']
 ARGV 1-: ['C-second']
 tclass.py: deleting object: C-first
 ARGV 1-: ['C-third']
 tclass.py: deleting object: C-second
 tclass.py: deleting object: C-third
 """
         if dec.module_not_available('sqlite3'):
             err = 'WARNING: IPython History requires SQLite, your history will not be saved\n'
         else:
             err = None
         tt.ipexec_validate(self.fname, out, err)
 
     def test_run_i_after_reset(self):
         """Check that %run -i still works after %reset (gh-693)"""
         src = "yy = zz\n"
         self.mktmp(src)
         _ip.run_cell("zz = 23")
         try:
             _ip.magic('run -i %s' % self.fname)
             nt.assert_equal(_ip.user_ns['yy'], 23)
         finally:
             _ip.magic('reset -f')
             
         _ip.run_cell("zz = 23")
         try:
             _ip.magic('run -i %s' % self.fname)
             nt.assert_equal(_ip.user_ns['yy'], 23)
         finally:
             _ip.magic('reset -f')
             
     def test_unicode(self):
         """Check that files in odd encodings are accepted."""
         mydir = os.path.dirname(__file__)
         na = os.path.join(mydir, 'nonascii.py')
         _ip.magic('run "%s"' % na)
         nt.assert_equal(_ip.user_ns['u'], u'Ўт№Ф')
 
     def test_run_py_file_attribute(self):
         """Test handling of `__file__` attribute in `%run <file>.py`."""
         src = "t = __file__\n"
         self.mktmp(src)
         _missing = object()
         file1 = _ip.user_ns.get('__file__', _missing)
         _ip.magic('run %s' % self.fname)
         file2 = _ip.user_ns.get('__file__', _missing)
 
         # Check that __file__ was equal to the filename in the script's
         # namespace.
         nt.assert_equal(_ip.user_ns['t'], self.fname)
 
         # Check that __file__ was not leaked back into user_ns.
         nt.assert_equal(file1, file2)
 
     def test_run_ipy_file_attribute(self):
         """Test handling of `__file__` attribute in `%run <file.ipy>`."""
         src = "t = __file__\n"
         self.mktmp(src, ext='.ipy')
         _missing = object()
         file1 = _ip.user_ns.get('__file__', _missing)
         _ip.magic('run %s' % self.fname)
         file2 = _ip.user_ns.get('__file__', _missing)
 
         # Check that __file__ was equal to the filename in the script's
         # namespace.
         nt.assert_equal(_ip.user_ns['t'], self.fname)
 
         # Check that __file__ was not leaked back into user_ns.
         nt.assert_equal(file1, file2)
 
     def test_run_formatting(self):
         """ Test that %run -t -N<N> does not raise a TypeError for N > 1."""
         src = "pass"
         self.mktmp(src)
         _ip.magic('run -t -N 1 %s' % self.fname)
         _ip.magic('run -t -N 10 %s' % self.fname)
     
     def test_ignore_sys_exit(self):
         """Test the -e option to ignore sys.exit()"""
         src = "import sys; sys.exit(1)"
         self.mktmp(src)
         with tt.AssertPrints('SystemExit'):
             _ip.magic('run %s' % self.fname)
         
         with tt.AssertNotPrints('SystemExit'):
             _ip.magic('run -e %s' % self.fname)
 
     def test_run_nb(self):
         """Test %run notebook.ipynb"""
         from nbformat import v4, writes
         nb = v4.new_notebook(
            cells=[
                 v4.new_markdown_cell("The Ultimate Question of Everything"),
                 v4.new_code_cell("answer=42")
             ]
         )
         src = writes(nb, version=4)
         self.mktmp(src, ext='.ipynb')
         
         _ip.magic("run %s" % self.fname)
         
         nt.assert_equal(_ip.user_ns['answer'], 42)
 
     def test_file_options(self):
         src = ('import sys\n'
                'a = " ".join(sys.argv[1:])\n')
         self.mktmp(src)
         test_opts = '-x 3 --verbose'
         _ip.run_line_magic("run", '{0} {1}'.format(self.fname, test_opts))
         nt.assert_equal(_ip.user_ns['a'], test_opts)
 
 
 class TestMagicRunWithPackage(unittest.TestCase):
 
     def writefile(self, name, content):
         path = os.path.join(self.tempdir.name, name)
         d = os.path.dirname(path)
         if not os.path.isdir(d):
             os.makedirs(d)
         with open(path, 'w') as f:
             f.write(textwrap.dedent(content))
 
     def setUp(self):
         self.package = package = 'tmp{0}'.format(''.join([random.choice(string.ascii_letters) for i in range(10)]))
         """Temporary  (probably) valid python package name."""
 
         self.value = int(random.random() * 10000)
 
         self.tempdir = TemporaryDirectory()
         self.__orig_cwd = os.getcwd()
         sys.path.insert(0, self.tempdir.name)
 
         self.writefile(os.path.join(package, '__init__.py'), '')
         self.writefile(os.path.join(package, 'sub.py'), """
         x = {0!r}
         """.format(self.value))
         self.writefile(os.path.join(package, 'relative.py'), """
         from .sub import x
         """)
         self.writefile(os.path.join(package, 'absolute.py'), """
         from {0}.sub import x
         """.format(package))
         self.writefile(os.path.join(package, 'args.py'), """
         import sys
         a = " ".join(sys.argv[1:])
         """.format(package))
 
     def tearDown(self):
         os.chdir(self.__orig_cwd)
         sys.path[:] = [p for p in sys.path if p != self.tempdir.name]
         self.tempdir.cleanup()
 
     def check_run_submodule(self, submodule, opts=''):
         _ip.user_ns.pop('x', None)
         _ip.magic('run {2} -m {0}.{1}'.format(self.package, submodule, opts))
         self.assertEqual(_ip.user_ns['x'], self.value,
                          'Variable `x` is not loaded from module `{0}`.'
                          .format(submodule))
 
     def test_run_submodule_with_absolute_import(self):
         self.check_run_submodule('absolute')
 
     def test_run_submodule_with_relative_import(self):
         """Run submodule that has a relative import statement (#2727)."""
         self.check_run_submodule('relative')
 
     def test_prun_submodule_with_absolute_import(self):
         self.check_run_submodule('absolute', '-p')
 
     def test_prun_submodule_with_relative_import(self):
         self.check_run_submodule('relative', '-p')
 
     def with_fake_debugger(func):
         @functools.wraps(func)
         def wrapper(*args, **kwds):
             with patch.object(debugger.Pdb, 'run', staticmethod(eval)):
                 return func(*args, **kwds)
         return wrapper
 
     @with_fake_debugger
     def test_debug_run_submodule_with_absolute_import(self):
         self.check_run_submodule('absolute', '-d')
 
     @with_fake_debugger
     def test_debug_run_submodule_with_relative_import(self):
         self.check_run_submodule('relative', '-d')
 
     def test_module_options(self):
         _ip.user_ns.pop('a', None)
         test_opts = '-x abc -m test'
         _ip.run_line_magic('run', '-m {0}.args {1}'.format(self.package, test_opts))
         nt.assert_equal(_ip.user_ns['a'], test_opts)
 
     def test_module_options_with_separator(self):
         _ip.user_ns.pop('a', None)
         test_opts = '-x abc -m test'
         _ip.run_line_magic('run', '-m {0}.args -- {1}'.format(self.package, test_opts))
         nt.assert_equal(_ip.user_ns['a'], test_opts)
 
 def test_run__name__():
     with TemporaryDirectory() as td:
         path = pjoin(td, 'foo.py')
         with open(path, 'w') as f:
             f.write("q = __name__")
         
         _ip.user_ns.pop('q', None)
         _ip.magic('run {}'.format(path))
         nt.assert_equal(_ip.user_ns.pop('q'), '__main__')
         
         _ip.magic('run -n {}'.format(path))
         nt.assert_equal(_ip.user_ns.pop('q'), 'foo')
 
         try:
             _ip.magic('run -i -n {}'.format(path))
             nt.assert_equal(_ip.user_ns.pop('q'), 'foo')
         finally:
             _ip.magic('reset -f')
 
 
 def test_run_tb():
     """Test traceback offset in %run"""
     with TemporaryDirectory() as td:
         path = pjoin(td, 'foo.py')
         with open(path, 'w') as f:
             f.write('\n'.join([
                 "def foo():",
                 "    return bar()",
                 "def bar():",
                 "    raise RuntimeError('hello!')",
                 "foo()",
             ]))
         with capture_output() as io:
             _ip.magic('run {}'.format(path))
         out = io.stdout
         nt.assert_not_in("execfile", out)
         nt.assert_in("RuntimeError", out)
         nt.assert_equal(out.count("---->"), 3)
         del ip.user_ns['bar']
         del ip.user_ns['foo']
 
+
+def test_multiprocessing_run():
+    """Set we can run mutiprocesgin without messing up up main namespace
+
+    Note that import `nose.tools as nt` mdify the value s
+    sys.module['__mp_main__'] so wee need to temporarily set it to None to test
+    the issue.
+    """
+    with TemporaryDirectory() as td:
+        mpm = sys.modules.get('__mp_main__')
+        assert mpm is not None
+        sys.modules['__mp_main__'] = None
+        try:
+            path = pjoin(td, 'test.py')
+            with open(path, 'w') as f:
+                f.write("import multiprocessing\nprint('hoy')")
+            with capture_output() as io:
+                _ip.run_line_magic('run', path)
+                _ip.run_cell("i_m_undefined")
+            out = io.stdout
+            nt.assert_in("hoy", out)
+            nt.assert_not_in("AttributeError", out)
+            nt.assert_in("NameError", out)
+            nt.assert_equal(out.count("---->"), 1)
+        except:
+            raise
+        finally:
+            sys.modules['__mp_main__'] = mpm
+
 @dec.knownfailureif(sys.platform == 'win32', "writes to io.stdout aren't captured on Windows")
 def test_script_tb():
     """Test traceback offset in `ipython script.py`"""
     with TemporaryDirectory() as td:
         path = pjoin(td, 'foo.py')
         with open(path, 'w') as f:
             f.write('\n'.join([
                 "def foo():",
                 "    return bar()",
                 "def bar():",
                 "    raise RuntimeError('hello!')",
                 "foo()",
             ]))
         out, err = tt.ipexec(path)
         nt.assert_not_in("execfile", out)
         nt.assert_in("RuntimeError", out)
         nt.assert_equal(out.count("---->"), 3)
 

IPython/core/tests/test_ultratb.py

@@ -1,440 +1,459 b''
 # encoding: utf-8
 """Tests for IPython.core.ultratb
 """
 import io
 import logging
 import sys
 import os.path
 from textwrap import dedent
 import traceback
 import unittest
 from unittest import mock
 
 import IPython.core.ultratb as ultratb
 from IPython.core.ultratb import ColorTB, VerboseTB, find_recursion
 
 
 from IPython.testing import tools as tt
 from IPython.testing.decorators import onlyif_unicode_paths
 from IPython.utils.syspathcontext import prepended_to_syspath
 from IPython.utils.tempdir import TemporaryDirectory
 
 file_1 = """1
 2
 3
 def f():
   1/0
 """
 
 file_2 = """def f():
   1/0
 """
 
 
 def recursionlimit(frames):
     """
     decorator to set the recursion limit temporarily
     """
 
     def inner(test_function):
         def wrapper(*args, **kwargs):
             _orig_rec_limit = ultratb._FRAME_RECURSION_LIMIT
             ultratb._FRAME_RECURSION_LIMIT = 50
 
             rl = sys.getrecursionlimit()
             sys.setrecursionlimit(frames)
             try:
                 return test_function(*args, **kwargs)
             finally:
                 sys.setrecursionlimit(rl)
                 ultratb._FRAME_RECURSION_LIMIT = _orig_rec_limit
 
         return wrapper
 
     return inner
 
 
 class ChangedPyFileTest(unittest.TestCase):
     def test_changing_py_file(self):
         """Traceback produced if the line where the error occurred is missing?
         
         https://github.com/ipython/ipython/issues/1456
         """
         with TemporaryDirectory() as td:
             fname = os.path.join(td, "foo.py")
             with open(fname, "w") as f:
                 f.write(file_1)
             
             with prepended_to_syspath(td):
                 ip.run_cell("import foo")
             
             with tt.AssertPrints("ZeroDivisionError"):
                 ip.run_cell("foo.f()")
             
             # Make the file shorter, so the line of the error is missing.
             with open(fname, "w") as f:
                 f.write(file_2)
             
             # For some reason, this was failing on the *second* call after
             # changing the file, so we call f() twice.
             with tt.AssertNotPrints("Internal Python error", channel='stderr'):
                 with tt.AssertPrints("ZeroDivisionError"):
                     ip.run_cell("foo.f()")
                 with tt.AssertPrints("ZeroDivisionError"):
                     ip.run_cell("foo.f()")
 
 iso_8859_5_file = u'''# coding: iso-8859-5
 
 def fail():
     """дбИЖ"""
     1/0     # дбИЖ
 '''
 
 class NonAsciiTest(unittest.TestCase):
     @onlyif_unicode_paths
     def test_nonascii_path(self):
         # Non-ascii directory name as well.
         with TemporaryDirectory(suffix=u'é') as td:
             fname = os.path.join(td, u"fooé.py")
             with open(fname, "w") as f:
                 f.write(file_1)
             
             with prepended_to_syspath(td):
                 ip.run_cell("import foo")
             
             with tt.AssertPrints("ZeroDivisionError"):
                 ip.run_cell("foo.f()")
     
     def test_iso8859_5(self):
         with TemporaryDirectory() as td:
             fname = os.path.join(td, 'dfghjkl.py')
 
             with io.open(fname, 'w', encoding='iso-8859-5') as f:
                 f.write(iso_8859_5_file)
             
             with prepended_to_syspath(td):
                 ip.run_cell("from dfghjkl import fail")
             
             with tt.AssertPrints("ZeroDivisionError"):
                 with tt.AssertPrints(u'дбИЖ', suppress=False):
                     ip.run_cell('fail()')
     
     def test_nonascii_msg(self):
         cell = u"raise Exception('é')"
         expected = u"Exception('é')"
         ip.run_cell("%xmode plain")
         with tt.AssertPrints(expected):
             ip.run_cell(cell)
 
         ip.run_cell("%xmode verbose")
         with tt.AssertPrints(expected):
             ip.run_cell(cell)
 
         ip.run_cell("%xmode context")
         with tt.AssertPrints(expected):
             ip.run_cell(cell)
 
         ip.run_cell("%xmode minimal")
         with tt.AssertPrints(u"Exception: é"):
             ip.run_cell(cell)
 
         # Put this back into Context mode for later tests.
         ip.run_cell("%xmode context")
 
 class NestedGenExprTestCase(unittest.TestCase):
     """
     Regression test for the following issues:
     https://github.com/ipython/ipython/issues/8293
     https://github.com/ipython/ipython/issues/8205
     """
     def test_nested_genexpr(self):
         code = dedent(
             """\
             class SpecificException(Exception):
                 pass
 
             def foo(x):
                 raise SpecificException("Success!")
 
             sum(sum(foo(x) for _ in [0]) for x in [0])
             """
         )
         with tt.AssertPrints('SpecificException: Success!', suppress=False):
             ip.run_cell(code)
 
 
 indentationerror_file = """if True:
 zoon()
 """
 
 class IndentationErrorTest(unittest.TestCase):
     def test_indentationerror_shows_line(self):
         # See issue gh-2398
         with tt.AssertPrints("IndentationError"):
             with tt.AssertPrints("zoon()", suppress=False):
                 ip.run_cell(indentationerror_file)
         
         with TemporaryDirectory() as td:
             fname = os.path.join(td, "foo.py")
             with open(fname, "w") as f:
                 f.write(indentationerror_file)
             
             with tt.AssertPrints("IndentationError"):
                 with tt.AssertPrints("zoon()", suppress=False):
                     ip.magic('run %s' % fname)
 
 se_file_1 = """1
 2
 7/
 """
 
 se_file_2 = """7/
 """
 
 class SyntaxErrorTest(unittest.TestCase):
     def test_syntaxerror_without_lineno(self):
         with tt.AssertNotPrints("TypeError"):
             with tt.AssertPrints("line unknown"):
                 ip.run_cell("raise SyntaxError()")
 
     def test_syntaxerror_no_stacktrace_at_compile_time(self):
         syntax_error_at_compile_time = """
 def foo():
     ..
 """
         with tt.AssertPrints("SyntaxError"):
             ip.run_cell(syntax_error_at_compile_time)
 
         with tt.AssertNotPrints("foo()"):
             ip.run_cell(syntax_error_at_compile_time)
 
     def test_syntaxerror_stacktrace_when_running_compiled_code(self):
         syntax_error_at_runtime = """
 def foo():
     eval("..")
 
 def bar():
     foo()
 
 bar()
 """
         with tt.AssertPrints("SyntaxError"):
             ip.run_cell(syntax_error_at_runtime)
         # Assert syntax error during runtime generate stacktrace
         with tt.AssertPrints(["foo()", "bar()"]):
             ip.run_cell(syntax_error_at_runtime)
         del ip.user_ns['bar']
         del ip.user_ns['foo']
 
     def test_changing_py_file(self):
         with TemporaryDirectory() as td:
             fname = os.path.join(td, "foo.py")
             with open(fname, 'w') as f:
                 f.write(se_file_1)
 
             with tt.AssertPrints(["7/", "SyntaxError"]):
                 ip.magic("run " + fname)
 
             # Modify the file
             with open(fname, 'w') as f:
                 f.write(se_file_2)
 
             # The SyntaxError should point to the correct line
             with tt.AssertPrints(["7/", "SyntaxError"]):
                 ip.magic("run " + fname)
 
     def test_non_syntaxerror(self):
         # SyntaxTB may be called with an error other than a SyntaxError
         # See e.g. gh-4361
         try:
             raise ValueError('QWERTY')
         except ValueError:
             with tt.AssertPrints('QWERTY'):
                 ip.showsyntaxerror()
 
 
 class Python3ChainedExceptionsTest(unittest.TestCase):
     DIRECT_CAUSE_ERROR_CODE = """
 try:
     x = 1 + 2
     print(not_defined_here)
 except Exception as e:
     x += 55
     x - 1
     y = {}
     raise KeyError('uh') from e
     """
 
     EXCEPTION_DURING_HANDLING_CODE = """
 try:
     x = 1 + 2
     print(not_defined_here)
 except Exception as e:
     x += 55
     x - 1
     y = {}
     raise KeyError('uh')
     """
 
     SUPPRESS_CHAINING_CODE = """
 try:
     1/0
 except Exception:
     raise ValueError("Yikes") from None
     """
 
     def test_direct_cause_error(self):
         with tt.AssertPrints(["KeyError", "NameError", "direct cause"]):
             ip.run_cell(self.DIRECT_CAUSE_ERROR_CODE)
 
     def test_exception_during_handling_error(self):
         with tt.AssertPrints(["KeyError", "NameError", "During handling"]):
             ip.run_cell(self.EXCEPTION_DURING_HANDLING_CODE)
 
     def test_suppress_exception_chaining(self):
         with tt.AssertNotPrints("ZeroDivisionError"), \
              tt.AssertPrints("ValueError", suppress=False):
             ip.run_cell(self.SUPPRESS_CHAINING_CODE)
 
+    def test_plain_direct_cause_error(self):
+        with tt.AssertPrints(["KeyError", "NameError", "direct cause"]):
+            ip.run_cell("%xmode Plain")
+            ip.run_cell(self.DIRECT_CAUSE_ERROR_CODE)
+            ip.run_cell("%xmode Verbose")
+
+    def test_plain_exception_during_handling_error(self):
+        with tt.AssertPrints(["KeyError", "NameError", "During handling"]):
+            ip.run_cell("%xmode Plain")
+            ip.run_cell(self.EXCEPTION_DURING_HANDLING_CODE)
+            ip.run_cell("%xmode Verbose")
+
+    def test_plain_suppress_exception_chaining(self):
+        with tt.AssertNotPrints("ZeroDivisionError"), \
+             tt.AssertPrints("ValueError", suppress=False):
+            ip.run_cell("%xmode Plain")
+            ip.run_cell(self.SUPPRESS_CHAINING_CODE)
+            ip.run_cell("%xmode Verbose")
+
 
 class RecursionTest(unittest.TestCase):
     DEFINITIONS = """
 def non_recurs():
     1/0
 
 def r1():
     r1()
 
 def r3a():
     r3b()
 
 def r3b():
     r3c()
 
 def r3c():
     r3a()
 
 def r3o1():
     r3a()
 
 def r3o2():
     r3o1()
 """
     def setUp(self):
         ip.run_cell(self.DEFINITIONS)
 
     def test_no_recursion(self):
         with tt.AssertNotPrints("frames repeated"):
             ip.run_cell("non_recurs()")
 
     @recursionlimit(150)
     def test_recursion_one_frame(self):
         with tt.AssertPrints("1 frames repeated"):
             ip.run_cell("r1()")
 
     @recursionlimit(150)
     def test_recursion_three_frames(self):
         with tt.AssertPrints("3 frames repeated"):
             ip.run_cell("r3o2()")
 
     @recursionlimit(150)
     def test_find_recursion(self):
         captured = []
         def capture_exc(*args, **kwargs):
             captured.append(sys.exc_info())
         with mock.patch.object(ip, 'showtraceback', capture_exc):
             ip.run_cell("r3o2()")
 
         self.assertEqual(len(captured), 1)
         etype, evalue, tb = captured[0]
         self.assertIn("recursion", str(evalue))
 
         records = ip.InteractiveTB.get_records(tb, 3, ip.InteractiveTB.tb_offset)
         for r in records[:10]:
             print(r[1:4])
 
         # The outermost frames should be:
         # 0: the 'cell' that was running when the exception came up
         # 1: r3o2()
         # 2: r3o1()
         # 3: r3a()
         # Then repeating r3b, r3c, r3a
         last_unique, repeat_length = find_recursion(etype, evalue, records)
         self.assertEqual(last_unique, 2)
         self.assertEqual(repeat_length, 3)
 
 
 #----------------------------------------------------------------------------
 
 # module testing (minimal)
 def test_handlers():
     def spam(c, d_e):
         (d, e) = d_e
         x = c + d
         y = c * d
         foo(x, y)
 
     def foo(a, b, bar=1):
         eggs(a, b + bar)
 
     def eggs(f, g, z=globals()):
         h = f + g
         i = f - g
         return h / i
 
     buff = io.StringIO()
 
     buff.write('')
     buff.write('*** Before ***')
     try:
         buff.write(spam(1, (2, 3)))
     except:
         traceback.print_exc(file=buff)
 
     handler = ColorTB(ostream=buff)
     buff.write('*** ColorTB ***')
     try:
         buff.write(spam(1, (2, 3)))
     except:
         handler(*sys.exc_info())
     buff.write('')
 
     handler = VerboseTB(ostream=buff)
     buff.write('*** VerboseTB ***')
     try:
         buff.write(spam(1, (2, 3)))
     except:
         handler(*sys.exc_info())
     buff.write('')
 
 from IPython.testing.decorators import skipif
 
 class TokenizeFailureTest(unittest.TestCase):
     """Tests related to https://github.com/ipython/ipython/issues/6864."""
 
     # that appear to test that we are handling an exception that can be thrown
     # by the tokenizer due to a bug that seem to have been fixed in 3.8, though
     # I'm unsure if other sequences can make it raise this error. Let's just
     # skip in 3.8 for now
     @skipif(sys.version_info > (3,8))
     def testLogging(self):
         message = "An unexpected error occurred while tokenizing input"
         cell = 'raise ValueError("""a\nb""")'
 
         stream = io.StringIO()
         handler = logging.StreamHandler(stream)
         logger = logging.getLogger()
         loglevel = logger.level
         logger.addHandler(handler)
         self.addCleanup(lambda: logger.removeHandler(handler))
         self.addCleanup(lambda: logger.setLevel(loglevel))
 
         logger.setLevel(logging.INFO)
         with tt.AssertNotPrints(message):
             ip.run_cell(cell)
         self.assertNotIn(message, stream.getvalue())
 
         logger.setLevel(logging.DEBUG)
         with tt.AssertNotPrints(message):
             ip.run_cell(cell)
         self.assertIn(message, stream.getvalue())

IPython/core/ultratb.py

@@ -1,1473 +1,1502 b''
 # -*- coding: utf-8 -*-
 """
 Verbose and colourful traceback formatting.
 
 **ColorTB**
 
 I've always found it a bit hard to visually parse tracebacks in Python.  The
 ColorTB class is a solution to that problem.  It colors the different parts of a
 traceback in a manner similar to what you would expect from a syntax-highlighting
 text editor.
 
 Installation instructions for ColorTB::
 
     import sys,ultratb
     sys.excepthook = ultratb.ColorTB()
 
 **VerboseTB**
 
 I've also included a port of Ka-Ping Yee's "cgitb.py" that produces all kinds
 of useful info when a traceback occurs.  Ping originally had it spit out HTML
 and intended it for CGI programmers, but why should they have all the fun?  I
 altered it to spit out colored text to the terminal.  It's a bit overwhelming,
 but kind of neat, and maybe useful for long-running programs that you believe
 are bug-free.  If a crash *does* occur in that type of program you want details.
 Give it a shot--you'll love it or you'll hate it.
 
 .. note::
 
   The Verbose mode prints the variables currently visible where the exception
   happened (shortening their strings if too long). This can potentially be
   very slow, if you happen to have a huge data structure whose string
   representation is complex to compute. Your computer may appear to freeze for
   a while with cpu usage at 100%. If this occurs, you can cancel the traceback
   with Ctrl-C (maybe hitting it more than once).
 
   If you encounter this kind of situation often, you may want to use the
   Verbose_novars mode instead of the regular Verbose, which avoids formatting
   variables (but otherwise includes the information and context given by
   Verbose).
 
 .. note::
 
   The verbose mode print all variables in the stack, which means it can
   potentially leak sensitive information like access keys, or unencrypted
   password.
 
 Installation instructions for VerboseTB::
 
     import sys,ultratb
     sys.excepthook = ultratb.VerboseTB()
 
 Note:  Much of the code in this module was lifted verbatim from the standard
 library module 'traceback.py' and Ka-Ping Yee's 'cgitb.py'.
 
 Color schemes
 -------------
 
 The colors are defined in the class TBTools through the use of the
 ColorSchemeTable class. Currently the following exist:
 
   - NoColor: allows all of this module to be used in any terminal (the color
     escapes are just dummy blank strings).
 
   - Linux: is meant to look good in a terminal like the Linux console (black
     or very dark background).
 
   - LightBG: similar to Linux but swaps dark/light colors to be more readable
     in light background terminals.
 
   - Neutral: a neutral color scheme that should be readable on both light and
     dark background
 
 You can implement other color schemes easily, the syntax is fairly
 self-explanatory. Please send back new schemes you develop to the author for
 possible inclusion in future releases.
 
 Inheritance diagram:
 
 .. inheritance-diagram:: IPython.core.ultratb
    :parts: 3
 """
 
 #*****************************************************************************
 # Copyright (C) 2001 Nathaniel Gray <[e-mail]>
 # Copyright (C) 2001-2004 Fernando Perez <[e-mail]>
 #
 # Distributed under the terms of the BSD License.  The full license is in
 # the file COPYING, distributed as part of this software.
 #*****************************************************************************
 
 
 import dis
 import inspect
 import keyword
 import linecache
 import os
 import pydoc
 import re
 import sys
 import time
 import tokenize
 import traceback
 
 try:  # Python 2
     generate_tokens = tokenize.generate_tokens
 except AttributeError:  # Python 3
     generate_tokens = tokenize.tokenize
 
 # For purposes of monkeypatching inspect to fix a bug in it.
 from inspect import getsourcefile, getfile, getmodule, \
     ismodule, isclass, ismethod, isfunction, istraceback, isframe, iscode
 
 # IPython's own modules
 from IPython import get_ipython
 from IPython.core import debugger
 from IPython.core.display_trap import DisplayTrap
 from IPython.core.excolors import exception_colors
 from IPython.utils import PyColorize
 from IPython.utils import path as util_path
 from IPython.utils import py3compat
 from IPython.utils.data import uniq_stable
 from IPython.utils.terminal import get_terminal_size
 
 from logging import info, error, debug
 
 from importlib.util import source_from_cache
 
 import IPython.utils.colorable as colorable
 
 # Globals
 # amount of space to put line numbers before verbose tracebacks
 INDENT_SIZE = 8
 
 # Default color scheme.  This is used, for example, by the traceback
 # formatter.  When running in an actual IPython instance, the user's rc.colors
 # value is used, but having a module global makes this functionality available
 # to users of ultratb who are NOT running inside ipython.
 DEFAULT_SCHEME = 'NoColor'
 
 
 # Number of frame above which we are likely to have a recursion and will
 # **attempt** to detect it.  Made modifiable mostly to speedup test suite
 # as detecting recursion is one of our slowest test
 _FRAME_RECURSION_LIMIT = 500
 
 # ---------------------------------------------------------------------------
 # Code begins
 
 # Utility functions
 def inspect_error():
     """Print a message about internal inspect errors.
 
     These are unfortunately quite common."""
 
     error('Internal Python error in the inspect module.\n'
           'Below is the traceback from this internal error.\n')
 
 
 # This function is a monkeypatch we apply to the Python inspect module. We have
 # now found when it's needed (see discussion on issue gh-1456), and we have a
 # test case (IPython.core.tests.test_ultratb.ChangedPyFileTest) that fails if
 # the monkeypatch is not applied. TK, Aug 2012.
 def findsource(object):
     """Return the entire source file and starting line number for an object.
 
     The argument may be a module, class, method, function, traceback, frame,
     or code object.  The source code is returned as a list of all the lines
     in the file and the line number indexes a line in that list.  An IOError
     is raised if the source code cannot be retrieved.
 
     FIXED version with which we monkeypatch the stdlib to work around a bug."""
 
     file = getsourcefile(object) or getfile(object)
     # If the object is a frame, then trying to get the globals dict from its
     # module won't work. Instead, the frame object itself has the globals
     # dictionary.
     globals_dict = None
     if inspect.isframe(object):
         # XXX: can this ever be false?
         globals_dict = object.f_globals
     else:
         module = getmodule(object, file)
         if module:
             globals_dict = module.__dict__
     lines = linecache.getlines(file, globals_dict)
     if not lines:
         raise IOError('could not get source code')
 
     if ismodule(object):
         return lines, 0
 
     if isclass(object):
         name = object.__name__
         pat = re.compile(r'^(\s*)class\s*' + name + r'\b')
         # make some effort to find the best matching class definition:
         # use the one with the least indentation, which is the one
         # that's most probably not inside a function definition.
         candidates = []
         for i, line in enumerate(lines):
             match = pat.match(line)
             if match:
                 # if it's at toplevel, it's already the best one
                 if line[0] == 'c':
                     return lines, i
                 # else add whitespace to candidate list
                 candidates.append((match.group(1), i))
         if candidates:
             # this will sort by whitespace, and by line number,
             # less whitespace first
             candidates.sort()
             return lines, candidates[0][1]
         else:
             raise IOError('could not find class definition')
 
     if ismethod(object):
         object = object.__func__
     if isfunction(object):
         object = object.__code__
     if istraceback(object):
         object = object.tb_frame
     if isframe(object):
         object = object.f_code
     if iscode(object):
         if not hasattr(object, 'co_firstlineno'):
             raise IOError('could not find function definition')
         pat = re.compile(r'^(\s*def\s)|(.*(?<!\w)lambda(:|\s))|^(\s*@)')
         pmatch = pat.match
         # fperez - fix: sometimes, co_firstlineno can give a number larger than
         # the length of lines, which causes an error.  Safeguard against that.
         lnum = min(object.co_firstlineno, len(lines)) - 1
         while lnum > 0:
             if pmatch(lines[lnum]):
                 break
             lnum -= 1
 
         return lines, lnum
     raise IOError('could not find code object')
 
 
 # This is a patched version of inspect.getargs that applies the (unmerged)
 # patch for http://bugs.python.org/issue14611 by Stefano Taschini.  This fixes
 # https://github.com/ipython/ipython/issues/8205 and
 # https://github.com/ipython/ipython/issues/8293
 def getargs(co):
     """Get information about the arguments accepted by a code object.
 
     Three things are returned: (args, varargs, varkw), where 'args' is
     a list of argument names (possibly containing nested lists), and
     'varargs' and 'varkw' are the names of the * and ** arguments or None."""
     if not iscode(co):
         raise TypeError('{!r} is not a code object'.format(co))
 
     nargs = co.co_argcount
     names = co.co_varnames
     args = list(names[:nargs])
     step = 0
 
     # The following acrobatics are for anonymous (tuple) arguments.
     for i in range(nargs):
         if args[i][:1] in ('', '.'):
             stack, remain, count = [], [], []
             while step < len(co.co_code):
                 op = ord(co.co_code[step])
                 step = step + 1
                 if op >= dis.HAVE_ARGUMENT:
                     opname = dis.opname[op]
                     value = ord(co.co_code[step]) + ord(co.co_code[step+1])*256
                     step = step + 2
                     if opname in ('UNPACK_TUPLE', 'UNPACK_SEQUENCE'):
                         remain.append(value)
                         count.append(value)
                     elif opname in ('STORE_FAST', 'STORE_DEREF'):
                         if op in dis.haslocal:
                             stack.append(co.co_varnames[value])
                         elif op in dis.hasfree:
                             stack.append((co.co_cellvars + co.co_freevars)[value])
                         # Special case for sublists of length 1: def foo((bar))
                         # doesn't generate the UNPACK_TUPLE bytecode, so if
                         # `remain` is empty here, we have such a sublist.
                         if not remain:
                             stack[0] = [stack[0]]
                             break
                         else:
                             remain[-1] = remain[-1] - 1
                             while remain[-1] == 0:
                                 remain.pop()
                                 size = count.pop()
                                 stack[-size:] = [stack[-size:]]
                                 if not remain:
                                     break
                                 remain[-1] = remain[-1] - 1
                             if not remain:
                                 break
             args[i] = stack[0]
 
     varargs = None
     if co.co_flags & inspect.CO_VARARGS:
         varargs = co.co_varnames[nargs]
         nargs = nargs + 1
     varkw = None
     if co.co_flags & inspect.CO_VARKEYWORDS:
         varkw = co.co_varnames[nargs]
     return inspect.Arguments(args, varargs, varkw)
 
 
 # Monkeypatch inspect to apply our bugfix.
 def with_patch_inspect(f):
     """
     Deprecated since IPython 6.0
     decorator for monkeypatching inspect.findsource
     """
 
     def wrapped(*args, **kwargs):
         save_findsource = inspect.findsource
         save_getargs = inspect.getargs
         inspect.findsource = findsource
         inspect.getargs = getargs
         try:
             return f(*args, **kwargs)
         finally:
             inspect.findsource = save_findsource
             inspect.getargs = save_getargs
 
     return wrapped
 
 
 def fix_frame_records_filenames(records):
     """Try to fix the filenames in each record from inspect.getinnerframes().
 
     Particularly, modules loaded from within zip files have useless filenames
     attached to their code object, and inspect.getinnerframes() just uses it.
     """
     fixed_records = []
     for frame, filename, line_no, func_name, lines, index in records:
         # Look inside the frame's globals dictionary for __file__,
         # which should be better. However, keep Cython filenames since
         # we prefer the source filenames over the compiled .so file.
         if not filename.endswith(('.pyx', '.pxd', '.pxi')):
             better_fn = frame.f_globals.get('__file__', None)
             if isinstance(better_fn, str):
                 # Check the type just in case someone did something weird with
                 # __file__. It might also be None if the error occurred during
                 # import.
                 filename = better_fn
         fixed_records.append((frame, filename, line_no, func_name, lines, index))
     return fixed_records
 
 
 @with_patch_inspect
 def _fixed_getinnerframes(etb, context=1, tb_offset=0):
     LNUM_POS, LINES_POS, INDEX_POS = 2, 4, 5
 
     records = fix_frame_records_filenames(inspect.getinnerframes(etb, context))
     # If the error is at the console, don't build any context, since it would
     # otherwise produce 5 blank lines printed out (there is no file at the
     # console)
     rec_check = records[tb_offset:]
     try:
         rname = rec_check[0][1]
         if rname == '<ipython console>' or rname.endswith('<string>'):
             return rec_check
     except IndexError:
         pass
 
     aux = traceback.extract_tb(etb)
     assert len(records) == len(aux)
     for i, (file, lnum, _, _) in enumerate(aux):
         maybeStart = lnum - 1 - context // 2
         start = max(maybeStart, 0)
         end = start + context
         lines = linecache.getlines(file)[start:end]
         buf = list(records[i])
         buf[LNUM_POS] = lnum
         buf[INDEX_POS] = lnum - 1 - start
         buf[LINES_POS] = lines
         records[i] = tuple(buf)
     return records[tb_offset:]
 
 # Helper function -- largely belongs to VerboseTB, but we need the same
 # functionality to produce a pseudo verbose TB for SyntaxErrors, so that they
 # can be recognized properly by ipython.el's py-traceback-line-re
 # (SyntaxErrors have to be treated specially because they have no traceback)
 
 
 def _format_traceback_lines(lnum, index, lines, Colors, lvals, _line_format):
     """
     Format tracebacks lines with pointing arrow, leading numbers...
 
     Parameters
     ==========
 
     lnum: int
     index: int
     lines: list[string]
     Colors:
         ColorScheme used.
     lvals: bytes
         Values of local variables, already colored, to inject just after the error line.
     _line_format: f (str) -> (str, bool)
         return (colorized version of str, failure to do so)
     """
     numbers_width = INDENT_SIZE - 1
     res = []
 
     for i,line in enumerate(lines, lnum-index):
         line = py3compat.cast_unicode(line)
 
         new_line, err = _line_format(line, 'str')
         if not err:
             line = new_line
 
         if i == lnum:
             # This is the line with the error
             pad = numbers_width - len(str(i))
             num = '%s%s' % (debugger.make_arrow(pad), str(lnum))
             line = '%s%s%s %s%s' % (Colors.linenoEm, num,
                                     Colors.line, line, Colors.Normal)
         else:
             num = '%*s' % (numbers_width, i)
             line = '%s%s%s %s' % (Colors.lineno, num,
                                   Colors.Normal, line)
 
         res.append(line)
         if lvals and i == lnum:
             res.append(lvals + '\n')
     return res
 
 def is_recursion_error(etype, value, records):
     try:
         # RecursionError is new in Python 3.5
         recursion_error_type = RecursionError
     except NameError:
         recursion_error_type = RuntimeError
 
     # The default recursion limit is 1000, but some of that will be taken up
     # by stack frames in IPython itself. >500 frames probably indicates
     # a recursion error.
     return (etype is recursion_error_type) \
            and "recursion" in str(value).lower() \
            and len(records) > _FRAME_RECURSION_LIMIT
 
 def find_recursion(etype, value, records):
     """Identify the repeating stack frames from a RecursionError traceback
 
     'records' is a list as returned by VerboseTB.get_records()
 
     Returns (last_unique, repeat_length)
     """
     # This involves a bit of guesswork - we want to show enough of the traceback
     # to indicate where the recursion is occurring. We guess that the innermost
     # quarter of the traceback (250 frames by default) is repeats, and find the
     # first frame (from in to out) that looks different.
     if not is_recursion_error(etype, value, records):
         return len(records), 0
 
     # Select filename, lineno, func_name to track frames with
     records = [r[1:4] for r in records]
     inner_frames = records[-(len(records)//4):]
     frames_repeated = set(inner_frames)
 
     last_seen_at = {}
     longest_repeat = 0
     i = len(records)
     for frame in reversed(records):
         i -= 1
         if frame not in frames_repeated:
             last_unique = i
             break
 
         if frame in last_seen_at:
             distance = last_seen_at[frame] - i
             longest_repeat = max(longest_repeat, distance)
 
         last_seen_at[frame] = i
     else:
         last_unique = 0 # The whole traceback was recursion
 
     return last_unique, longest_repeat
 
 #---------------------------------------------------------------------------
 # Module classes
 class TBTools(colorable.Colorable):
     """Basic tools used by all traceback printer classes."""
 
     # Number of frames to skip when reporting tracebacks
     tb_offset = 0
 
     def __init__(self, color_scheme='NoColor', call_pdb=False, ostream=None, parent=None, config=None):
         # Whether to call the interactive pdb debugger after printing
         # tracebacks or not
         super(TBTools, self).__init__(parent=parent, config=config)
         self.call_pdb = call_pdb
 
         # Output stream to write to.  Note that we store the original value in
         # a private attribute and then make the public ostream a property, so
         # that we can delay accessing sys.stdout until runtime.  The way
         # things are written now, the sys.stdout object is dynamically managed
         # so a reference to it should NEVER be stored statically.  This
         # property approach confines this detail to a single location, and all
         # subclasses can simply access self.ostream for writing.
         self._ostream = ostream
 
         # Create color table
         self.color_scheme_table = exception_colors()
 
         self.set_colors(color_scheme)
         self.old_scheme = color_scheme  # save initial value for toggles
 
         if call_pdb:
             self.pdb = debugger.Pdb()
         else:
             self.pdb = None
 
     def _get_ostream(self):
         """Output stream that exceptions are written to.
 
         Valid values are:
 
         - None: the default, which means that IPython will dynamically resolve
           to sys.stdout.  This ensures compatibility with most tools, including
           Windows (where plain stdout doesn't recognize ANSI escapes).
 
         - Any object with 'write' and 'flush' attributes.
         """
         return sys.stdout if self._ostream is None else self._ostream
 
     def _set_ostream(self, val):
         assert val is None or (hasattr(val, 'write') and hasattr(val, 'flush'))
         self._ostream = val
 
     ostream = property(_get_ostream, _set_ostream)
 
+    def get_parts_of_chained_exception(self, evalue):
+        def get_chained_exception(exception_value):
+            cause = getattr(exception_value, '__cause__', None)
+            if cause:
+                return cause
+            if getattr(exception_value, '__suppress_context__', False):
+                return None
+            return getattr(exception_value, '__context__', None)
+
+        chained_evalue = get_chained_exception(evalue)
+
+        if chained_evalue:
+            return chained_evalue.__class__, chained_evalue, chained_evalue.__traceback__
+
+    def prepare_chained_exception_message(self, cause):
+        direct_cause = "\nThe above exception was the direct cause of the following exception:\n"
+        exception_during_handling = "\nDuring handling of the above exception, another exception occurred:\n"
+
+        if cause:
+            message = [[direct_cause]]
+        else:
+            message = [[exception_during_handling]]
+        return message
+
     def set_colors(self, *args, **kw):
         """Shorthand access to the color table scheme selector method."""
 
         # Set own color table
         self.color_scheme_table.set_active_scheme(*args, **kw)
         # for convenience, set Colors to the active scheme
         self.Colors = self.color_scheme_table.active_colors
         # Also set colors of debugger
         if hasattr(self, 'pdb') and self.pdb is not None:
             self.pdb.set_colors(*args, **kw)
 
     def color_toggle(self):
         """Toggle between the currently active color scheme and NoColor."""
 
         if self.color_scheme_table.active_scheme_name == 'NoColor':
             self.color_scheme_table.set_active_scheme(self.old_scheme)
             self.Colors = self.color_scheme_table.active_colors
         else:
             self.old_scheme = self.color_scheme_table.active_scheme_name
             self.color_scheme_table.set_active_scheme('NoColor')
             self.Colors = self.color_scheme_table.active_colors
 
     def stb2text(self, stb):
         """Convert a structured traceback (a list) to a string."""
         return '\n'.join(stb)
 
     def text(self, etype, value, tb, tb_offset=None, context=5):
         """Return formatted traceback.
 
         Subclasses may override this if they add extra arguments.
         """
         tb_list = self.structured_traceback(etype, value, tb,
                                             tb_offset, context)
         return self.stb2text(tb_list)
 
     def structured_traceback(self, etype, evalue, tb, tb_offset=None,
                              context=5, mode=None):
         """Return a list of traceback frames.
 
         Must be implemented by each class.
         """
         raise NotImplementedError()
 
 
 #---------------------------------------------------------------------------
 class ListTB(TBTools):
     """Print traceback information from a traceback list, with optional color.
 
     Calling requires 3 arguments: (etype, evalue, elist)
     as would be obtained by::
     
       etype, evalue, tb = sys.exc_info()
       if tb:
         elist = traceback.extract_tb(tb)
       else:
         elist = None
 
     It can thus be used by programs which need to process the traceback before
     printing (such as console replacements based on the code module from the
     standard library).
 
     Because they are meant to be called without a full traceback (only a
     list), instances of this class can't call the interactive pdb debugger."""
 
     def __init__(self, color_scheme='NoColor', call_pdb=False, ostream=None, parent=None, config=None):
         TBTools.__init__(self, color_scheme=color_scheme, call_pdb=call_pdb,
                          ostream=ostream, parent=parent,config=config)
 
     def __call__(self, etype, value, elist):
         self.ostream.flush()
         self.ostream.write(self.text(etype, value, elist))
         self.ostream.write('\n')
 
-    def structured_traceback(self, etype, value, elist, tb_offset=None,
+    def structured_traceback(self, etype, evalue, etb=None, tb_offset=None,
                              context=5):
         """Return a color formatted string with the traceback info.
 
         Parameters
         ----------
         etype : exception type
           Type of the exception raised.
 
-        value : object
+        evalue : object
           Data stored in the exception
 
-        elist : list
-          List of frames, see class docstring for details.
+        etb : object
+          If list: List of frames, see class docstring for details.
+          If Traceback: Traceback of the exception.
 
         tb_offset : int, optional
           Number of frames in the traceback to skip.  If not given, the
-          instance value is used (set in constructor).
+          instance evalue is used (set in constructor).
 
         context : int, optional
           Number of lines of context information to print.
 
         Returns
         -------
         String with formatted exception.
         """
+        # This is a workaround to get chained_exc_ids in recursive calls
+        # etb should not be a tuple if structured_traceback is not recursive
+        if isinstance(etb, tuple):
+            etb, chained_exc_ids = etb
+        else:
+            chained_exc_ids = set()
+
+        if isinstance(etb, list):
+            elist = etb
+        elif etb is not None:
+            elist = self._extract_tb(etb)
+        else:
+            elist = []
         tb_offset = self.tb_offset if tb_offset is None else tb_offset
         Colors = self.Colors
         out_list = []
         if elist:
 
             if tb_offset and len(elist) > tb_offset:
                 elist = elist[tb_offset:]
 
             out_list.append('Traceback %s(most recent call last)%s:' %
                             (Colors.normalEm, Colors.Normal) + '\n')
             out_list.extend(self._format_list(elist))
         # The exception info should be a single entry in the list.
-        lines = ''.join(self._format_exception_only(etype, value))
+        lines = ''.join(self._format_exception_only(etype, evalue))
         out_list.append(lines)
 
+        exception = self.get_parts_of_chained_exception(evalue)
+
+        if exception and not id(exception[1]) in chained_exc_ids:
+            chained_exception_message = self.prepare_chained_exception_message(
+                evalue.__cause__)[0]
+            etype, evalue, etb = exception
+            # Trace exception to avoid infinite 'cause' loop
+            chained_exc_ids.add(id(exception[1]))
+            chained_exceptions_tb_offset = 0
+            out_list = (
+                self.structured_traceback(
+                    etype, evalue, (etb, chained_exc_ids),
+                    chained_exceptions_tb_offset, context)
+                + chained_exception_message
+                + out_list)
+
         return out_list
 
     def _format_list(self, extracted_list):
         """Format a list of traceback entry tuples for printing.
 
         Given a list of tuples as returned by extract_tb() or
         extract_stack(), return a list of strings ready for printing.
         Each string in the resulting list corresponds to the item with the
         same index in the argument list.  Each string ends in a newline;
         the strings may contain internal newlines as well, for those items
         whose source text line is not None.
 
         Lifted almost verbatim from traceback.py
         """
 
         Colors = self.Colors
         list = []
         for filename, lineno, name, line in extracted_list[:-1]:
             item = '  File %s"%s"%s, line %s%d%s, in %s%s%s\n' % \
                    (Colors.filename, filename, Colors.Normal,
                     Colors.lineno, lineno, Colors.Normal,
                     Colors.name, name, Colors.Normal)
             if line:
                 item += '    %s\n' % line.strip()
             list.append(item)
         # Emphasize the last entry
         filename, lineno, name, line = extracted_list[-1]
         item = '%s  File %s"%s"%s, line %s%d%s, in %s%s%s%s\n' % \
                (Colors.normalEm,
                 Colors.filenameEm, filename, Colors.normalEm,
                 Colors.linenoEm, lineno, Colors.normalEm,
                 Colors.nameEm, name, Colors.normalEm,
                 Colors.Normal)
         if line:
             item += '%s    %s%s\n' % (Colors.line, line.strip(),
                                       Colors.Normal)
         list.append(item)
         return list
 
     def _format_exception_only(self, etype, value):
         """Format the exception part of a traceback.
 
         The arguments are the exception type and value such as given by
         sys.exc_info()[:2]. The return value is a list of strings, each ending
         in a newline.  Normally, the list contains a single string; however,
         for SyntaxError exceptions, it contains several lines that (when
         printed) display detailed information about where the syntax error
         occurred.  The message indicating which exception occurred is the
         always last string in the list.
 
         Also lifted nearly verbatim from traceback.py
         """
         have_filedata = False
         Colors = self.Colors
         list = []
         stype = py3compat.cast_unicode(Colors.excName + etype.__name__ + Colors.Normal)
         if value is None:
             # Not sure if this can still happen in Python 2.6 and above
             list.append(stype + '\n')
         else:
             if issubclass(etype, SyntaxError):
                 have_filedata = True
                 if not value.filename: value.filename = "<string>"
                 if value.lineno:
                     lineno = value.lineno
                     textline = linecache.getline(value.filename, value.lineno)
                 else:
                     lineno = 'unknown'
                     textline = ''
                 list.append('%s  File %s"%s"%s, line %s%s%s\n' % \
                             (Colors.normalEm,
                              Colors.filenameEm, py3compat.cast_unicode(value.filename), Colors.normalEm,
                              Colors.linenoEm, lineno, Colors.Normal  ))
                 if textline == '':
                     textline = py3compat.cast_unicode(value.text, "utf-8")
 
                 if textline is not None:
                     i = 0
                     while i < len(textline) and textline[i].isspace():
                         i += 1
                     list.append('%s    %s%s\n' % (Colors.line,
                                                   textline.strip(),
                                                   Colors.Normal))
                     if value.offset is not None:
                         s = '    '
                         for c in textline[i:value.offset - 1]:
                             if c.isspace():
                                 s += c
                             else:
                                 s += ' '
                         list.append('%s%s^%s\n' % (Colors.caret, s,
                                                    Colors.Normal))
 
             try:
                 s = value.msg
             except Exception:
                 s = self._some_str(value)
             if s:
                 list.append('%s%s:%s %s\n' % (stype, Colors.excName,
                                               Colors.Normal, s))
             else:
                 list.append('%s\n' % stype)
 
         # sync with user hooks
         if have_filedata:
             ipinst = get_ipython()
             if ipinst is not None:
                 ipinst.hooks.synchronize_with_editor(value.filename, value.lineno, 0)
 
         return list
 
     def get_exception_only(self, etype, value):
         """Only print the exception type and message, without a traceback.
 
         Parameters
         ----------
         etype : exception type
         value : exception value
         """
-        return ListTB.structured_traceback(self, etype, value, [])
+        return ListTB.structured_traceback(self, etype, value)
 
     def show_exception_only(self, etype, evalue):
         """Only print the exception type and message, without a traceback.
 
         Parameters
         ----------
         etype : exception type
         value : exception value
         """
         # This method needs to use __call__ from *this* class, not the one from
         # a subclass whose signature or behavior may be different
         ostream = self.ostream
         ostream.flush()
         ostream.write('\n'.join(self.get_exception_only(etype, evalue)))
         ostream.flush()
 
     def _some_str(self, value):
         # Lifted from traceback.py
         try:
             return py3compat.cast_unicode(str(value))
         except:
             return u'<unprintable %s object>' % type(value).__name__
 
 
 #----------------------------------------------------------------------------
 class VerboseTB(TBTools):
     """A port of Ka-Ping Yee's cgitb.py module that outputs color text instead
     of HTML.  Requires inspect and pydoc.  Crazy, man.
 
     Modified version which optionally strips the topmost entries from the
     traceback, to be used with alternate interpreters (because their own code
     would appear in the traceback)."""
 
     def __init__(self, color_scheme='Linux', call_pdb=False, ostream=None,
                  tb_offset=0, long_header=False, include_vars=True,
                  check_cache=None, debugger_cls = None,
                  parent=None, config=None):
         """Specify traceback offset, headers and color scheme.
 
         Define how many frames to drop from the tracebacks. Calling it with
         tb_offset=1 allows use of this handler in interpreters which will have
         their own code at the top of the traceback (VerboseTB will first
         remove that frame before printing the traceback info)."""
         TBTools.__init__(self, color_scheme=color_scheme, call_pdb=call_pdb,
                          ostream=ostream, parent=parent, config=config)
         self.tb_offset = tb_offset
         self.long_header = long_header
         self.include_vars = include_vars
         # By default we use linecache.checkcache, but the user can provide a
         # different check_cache implementation.  This is used by the IPython
         # kernel to provide tracebacks for interactive code that is cached,
         # by a compiler instance that flushes the linecache but preserves its
         # own code cache.
         if check_cache is None:
             check_cache = linecache.checkcache
         self.check_cache = check_cache
 
         self.debugger_cls = debugger_cls or debugger.Pdb
 
     def format_records(self, records, last_unique, recursion_repeat):
         """Format the stack frames of the traceback"""
         frames = []
         for r in records[:last_unique+recursion_repeat+1]:
             #print '*** record:',file,lnum,func,lines,index  # dbg
             frames.append(self.format_record(*r))
 
         if recursion_repeat:
             frames.append('... last %d frames repeated, from the frame below ...\n' % recursion_repeat)
             frames.append(self.format_record(*records[last_unique+recursion_repeat+1]))
 
         return frames
 
     def format_record(self, frame, file, lnum, func, lines, index):
         """Format a single stack frame"""
         Colors = self.Colors  # just a shorthand + quicker name lookup
         ColorsNormal = Colors.Normal  # used a lot
         col_scheme = self.color_scheme_table.active_scheme_name
         indent = ' ' * INDENT_SIZE
         em_normal = '%s\n%s%s' % (Colors.valEm, indent, ColorsNormal)
         undefined = '%sundefined%s' % (Colors.em, ColorsNormal)
         tpl_link = '%s%%s%s' % (Colors.filenameEm, ColorsNormal)
         tpl_call = 'in %s%%s%s%%s%s' % (Colors.vName, Colors.valEm,
                                         ColorsNormal)
         tpl_call_fail = 'in %s%%s%s(***failed resolving arguments***)%s' % \
                         (Colors.vName, Colors.valEm, ColorsNormal)
         tpl_local_var = '%s%%s%s' % (Colors.vName, ColorsNormal)
         tpl_global_var = '%sglobal%s %s%%s%s' % (Colors.em, ColorsNormal,
                                                  Colors.vName, ColorsNormal)
         tpl_name_val = '%%s %s= %%s%s' % (Colors.valEm, ColorsNormal)
 
         if not file:
             file = '?'
         elif file.startswith(str("<")) and file.endswith(str(">")):
             # Not a real filename, no problem...
             pass
         elif not os.path.isabs(file):
             # Try to make the filename absolute by trying all
             # sys.path entries (which is also what linecache does)
             for dirname in sys.path:
                 try:
                     fullname = os.path.join(dirname, file)
                     if os.path.isfile(fullname):
                         file = os.path.abspath(fullname)
                         break
                 except Exception:
                     # Just in case that sys.path contains very
                     # strange entries...
                     pass
 
         file = py3compat.cast_unicode(file, util_path.fs_encoding)
         link = tpl_link % util_path.compress_user(file)
         args, varargs, varkw, locals_ = inspect.getargvalues(frame)
 
         if func == '?':
             call = ''
         elif func == '<module>':
             call = tpl_call % (func, '')
         else:
             # Decide whether to include variable details or not
             var_repr = eqrepr if self.include_vars else nullrepr
             try:
                 call = tpl_call % (func, inspect.formatargvalues(args,
                                                                  varargs, varkw,
                                                                  locals_, formatvalue=var_repr))
             except KeyError:
                 # This happens in situations like errors inside generator
                 # expressions, where local variables are listed in the
                 # line, but can't be extracted from the frame.  I'm not
                 # 100% sure this isn't actually a bug in inspect itself,
                 # but since there's no info for us to compute with, the
                 # best we can do is report the failure and move on.  Here
                 # we must *not* call any traceback construction again,
                 # because that would mess up use of %debug later on.  So we
                 # simply report the failure and move on.  The only
                 # limitation will be that this frame won't have locals
                 # listed in the call signature.  Quite subtle problem...
                 # I can't think of a good way to validate this in a unit
                 # test, but running a script consisting of:
                 #  dict( (k,v.strip()) for (k,v) in range(10) )
                 # will illustrate the error, if this exception catch is
                 # disabled.
                 call = tpl_call_fail % func
 
         # Don't attempt to tokenize binary files.
         if file.endswith(('.so', '.pyd', '.dll')):
             return '%s %s\n' % (link, call)
 
         elif file.endswith(('.pyc', '.pyo')):
             # Look up the corresponding source file.
             try:
                 file = source_from_cache(file)
             except ValueError:
                 # Failed to get the source file for some reason
                 # E.g. https://github.com/ipython/ipython/issues/9486
                 return '%s %s\n' % (link, call)
 
         def linereader(file=file, lnum=[lnum], getline=linecache.getline):
             line = getline(file, lnum[0])
             lnum[0] += 1
             return line
 
         # Build the list of names on this line of code where the exception
         # occurred.
         try:
             names = []
             name_cont = False
 
             for token_type, token, start, end, line in generate_tokens(linereader):
                 # build composite names
                 if token_type == tokenize.NAME and token not in keyword.kwlist:
                     if name_cont:
                         # Continuation of a dotted name
                         try:
                             names[-1].append(token)
                         except IndexError:
                             names.append([token])
                         name_cont = False
                     else:
                         # Regular new names.  We append everything, the caller
                         # will be responsible for pruning the list later.  It's
                         # very tricky to try to prune as we go, b/c composite
                         # names can fool us.  The pruning at the end is easy
                         # to do (or the caller can print a list with repeated
                         # names if so desired.
                         names.append([token])
                 elif token == '.':
                     name_cont = True
                 elif token_type == tokenize.NEWLINE:
                     break
 
         except (IndexError, UnicodeDecodeError, SyntaxError):
             # signals exit of tokenizer
             # SyntaxError can occur if the file is not actually Python
             #  - see gh-6300
             pass
         except tokenize.TokenError as msg:
             # Tokenizing may fail for various reasons, many of which are
             # harmless. (A good example is when the line in question is the
             # close of a triple-quoted string, cf gh-6864). We don't want to
             # show this to users, but want make it available for debugging
             # purposes.
             _m = ("An unexpected error occurred while tokenizing input\n"
                   "The following traceback may be corrupted or invalid\n"
                   "The error message is: %s\n" % msg)
             debug(_m)
 
         # Join composite names (e.g. "dict.fromkeys")
         names = ['.'.join(n) for n in names]
         # prune names list of duplicates, but keep the right order
         unique_names = uniq_stable(names)
 
         # Start loop over vars
         lvals = ''
         lvals_list = []
         if self.include_vars:
             for name_full in unique_names:
                 name_base = name_full.split('.', 1)[0]
                 if name_base in frame.f_code.co_varnames:
                     if name_base in locals_:
                         try:
                             value = repr(eval(name_full, locals_))
                         except:
                             value = undefined
                     else:
                         value = undefined
                     name = tpl_local_var % name_full
                 else:
                     if name_base in frame.f_globals:
                         try:
                             value = repr(eval(name_full, frame.f_globals))
                         except:
                             value = undefined
                     else:
                         value = undefined
                     name = tpl_global_var % name_full
                 lvals_list.append(tpl_name_val % (name, value))
         if lvals_list:
             lvals = '%s%s' % (indent, em_normal.join(lvals_list))
 
         level = '%s %s\n' % (link, call)
 
         if index is None:
             return level
         else:
             _line_format = PyColorize.Parser(style=col_scheme, parent=self).format2
             return '%s%s' % (level, ''.join(
                 _format_traceback_lines(lnum, index, lines, Colors, lvals,
                                          _line_format)))
 
-    def prepare_chained_exception_message(self, cause):
-        direct_cause = "\nThe above exception was the direct cause of the following exception:\n"
-        exception_during_handling = "\nDuring handling of the above exception, another exception occurred:\n"
-
-        if cause:
-            message = [[direct_cause]]
-        else:
-            message = [[exception_during_handling]]
-        return message
-
     def prepare_header(self, etype, long_version=False):
         colors = self.Colors  # just a shorthand + quicker name lookup
         colorsnormal = colors.Normal  # used a lot
         exc = '%s%s%s' % (colors.excName, etype, colorsnormal)
         width = min(75, get_terminal_size()[0])
         if long_version:
             # Header with the exception type, python version, and date
             pyver = 'Python ' + sys.version.split()[0] + ': ' + sys.executable
             date = time.ctime(time.time())
 
             head = '%s%s%s\n%s%s%s\n%s' % (colors.topline, '-' * width, colorsnormal,
                                            exc, ' ' * (width - len(str(etype)) - len(pyver)),
                                            pyver, date.rjust(width) )
             head += "\nA problem occurred executing Python code.  Here is the sequence of function" \
                     "\ncalls leading up to the error, with the most recent (innermost) call last."
         else:
             # Simplified header
             head = '%s%s' % (exc, 'Traceback (most recent call last)'. \
                              rjust(width - len(str(etype))) )
 
         return head
 
     def format_exception(self, etype, evalue):
         colors = self.Colors  # just a shorthand + quicker name lookup
         colorsnormal = colors.Normal  # used a lot
         # Get (safely) a string form of the exception info
         try:
             etype_str, evalue_str = map(str, (etype, evalue))
         except:
             # User exception is improperly defined.
             etype, evalue = str, sys.exc_info()[:2]
             etype_str, evalue_str = map(str, (etype, evalue))
         # ... and format it
         return ['%s%s%s: %s' % (colors.excName, etype_str,
                                 colorsnormal, py3compat.cast_unicode(evalue_str))]
 
     def format_exception_as_a_whole(self, etype, evalue, etb, number_of_lines_of_context, tb_offset):
         """Formats the header, traceback and exception message for a single exception.
 
         This may be called multiple times by Python 3 exception chaining
         (PEP 3134).
         """
         # some locals
         orig_etype = etype
         try:
             etype = etype.__name__
         except AttributeError:
             pass
 
         tb_offset = self.tb_offset if tb_offset is None else tb_offset
         head = self.prepare_header(etype, self.long_header)
         records = self.get_records(etb, number_of_lines_of_context, tb_offset)
 
         if records is None:
             return ""
 
         last_unique, recursion_repeat = find_recursion(orig_etype, evalue, records)
 
         frames = self.format_records(records, last_unique, recursion_repeat)
 
         formatted_exception = self.format_exception(etype, evalue)
         if records:
             filepath, lnum = records[-1][1:3]
             filepath = os.path.abspath(filepath)
             ipinst = get_ipython()
             if ipinst is not None:
                 ipinst.hooks.synchronize_with_editor(filepath, lnum, 0)
 
         return [[head] + frames + [''.join(formatted_exception[0])]]
 
     def get_records(self, etb, number_of_lines_of_context, tb_offset):
         try:
             # Try the default getinnerframes and Alex's: Alex's fixes some
             # problems, but it generates empty tracebacks for console errors
             # (5 blanks lines) where none should be returned.
             return _fixed_getinnerframes(etb, number_of_lines_of_context, tb_offset)
         except UnicodeDecodeError:
             # This can occur if a file's encoding magic comment is wrong.
             # I can't see a way to recover without duplicating a bunch of code
             # from the stdlib traceback module. --TK
             error('\nUnicodeDecodeError while processing traceback.\n')
             return None
         except:
             # FIXME: I've been getting many crash reports from python 2.3
             # users, traceable to inspect.py.  If I can find a small test-case
             # to reproduce this, I should either write a better workaround or
             # file a bug report against inspect (if that's the real problem).
             # So far, I haven't been able to find an isolated example to
             # reproduce the problem.
             inspect_error()
             traceback.print_exc(file=self.ostream)
             info('\nUnfortunately, your original traceback can not be constructed.\n')
             return None
 
-    def get_parts_of_chained_exception(self, evalue):
-        def get_chained_exception(exception_value):
-            cause = getattr(exception_value, '__cause__', None)
-            if cause:
-                return cause
-            if getattr(exception_value, '__suppress_context__', False):
-                return None
-            return getattr(exception_value, '__context__', None)
-
-        chained_evalue = get_chained_exception(evalue)
-
-        if chained_evalue:
-            return chained_evalue.__class__, chained_evalue, chained_evalue.__traceback__
-
     def structured_traceback(self, etype, evalue, etb, tb_offset=None,
                              number_of_lines_of_context=5):
         """Return a nice text document describing the traceback."""
 
         formatted_exception = self.format_exception_as_a_whole(etype, evalue, etb, number_of_lines_of_context,
                                                                tb_offset)
 
         colors = self.Colors  # just a shorthand + quicker name lookup
         colorsnormal = colors.Normal  # used a lot
         head = '%s%s%s' % (colors.topline, '-' * min(75, get_terminal_size()[0]), colorsnormal)
         structured_traceback_parts = [head]
         chained_exceptions_tb_offset = 0
         lines_of_context = 3
         formatted_exceptions = formatted_exception
         exception = self.get_parts_of_chained_exception(evalue)
         if exception:
             formatted_exceptions += self.prepare_chained_exception_message(evalue.__cause__)
             etype, evalue, etb = exception
         else:
             evalue = None
         chained_exc_ids = set()
         while evalue:
             formatted_exceptions += self.format_exception_as_a_whole(etype, evalue, etb, lines_of_context,
                                                                      chained_exceptions_tb_offset)
             exception = self.get_parts_of_chained_exception(evalue)
 
             if exception and not id(exception[1]) in chained_exc_ids:
                 chained_exc_ids.add(id(exception[1])) # trace exception to avoid infinite 'cause' loop
                 formatted_exceptions += self.prepare_chained_exception_message(evalue.__cause__)
                 etype, evalue, etb = exception
             else:
                 evalue = None
 
         # we want to see exceptions in a reversed order:
         # the first exception should be on top
         for formatted_exception in reversed(formatted_exceptions):
             structured_traceback_parts += formatted_exception
 
         return structured_traceback_parts
 
     def debugger(self, force=False):
         """Call up the pdb debugger if desired, always clean up the tb
         reference.
 
         Keywords:
 
           - force(False): by default, this routine checks the instance call_pdb
             flag and does not actually invoke the debugger if the flag is false.
             The 'force' option forces the debugger to activate even if the flag
             is false.
 
         If the call_pdb flag is set, the pdb interactive debugger is
         invoked. In all cases, the self.tb reference to the current traceback
         is deleted to prevent lingering references which hamper memory
         management.
 
         Note that each call to pdb() does an 'import readline', so if your app
         requires a special setup for the readline completers, you'll have to
         fix that by hand after invoking the exception handler."""
 
         if force or self.call_pdb:
             if self.pdb is None:
                 self.pdb = self.debugger_cls()
             # the system displayhook may have changed, restore the original
             # for pdb
             display_trap = DisplayTrap(hook=sys.__displayhook__)
             with display_trap:
                 self.pdb.reset()
                 # Find the right frame so we don't pop up inside ipython itself
                 if hasattr(self, 'tb') and self.tb is not None:
                     etb = self.tb
                 else:
                     etb = self.tb = sys.last_traceback
                 while self.tb is not None and self.tb.tb_next is not None:
                     self.tb = self.tb.tb_next
                 if etb and etb.tb_next:
                     etb = etb.tb_next
                 self.pdb.botframe = etb.tb_frame
                 self.pdb.interaction(None, etb)
 
         if hasattr(self, 'tb'):
             del self.tb
 
     def handler(self, info=None):
         (etype, evalue, etb) = info or sys.exc_info()
         self.tb = etb
         ostream = self.ostream
         ostream.flush()
         ostream.write(self.text(etype, evalue, etb))
         ostream.write('\n')
         ostream.flush()
 
     # Changed so an instance can just be called as VerboseTB_inst() and print
     # out the right info on its own.
     def __call__(self, etype=None, evalue=None, etb=None):
         """This hook can replace sys.excepthook (for Python 2.1 or higher)."""
         if etb is None:
             self.handler()
         else:
             self.handler((etype, evalue, etb))
         try:
             self.debugger()
         except KeyboardInterrupt:
             print("\nKeyboardInterrupt")
 
 
 #----------------------------------------------------------------------------
 class FormattedTB(VerboseTB, ListTB):
     """Subclass ListTB but allow calling with a traceback.
 
     It can thus be used as a sys.excepthook for Python > 2.1.
 
     Also adds 'Context' and 'Verbose' modes, not available in ListTB.
 
     Allows a tb_offset to be specified. This is useful for situations where
     one needs to remove a number of topmost frames from the traceback (such as
     occurs with python programs that themselves execute other python code,
     like Python shells).  """
 
     def __init__(self, mode='Plain', color_scheme='Linux', call_pdb=False,
                  ostream=None,
                  tb_offset=0, long_header=False, include_vars=False,
                  check_cache=None, debugger_cls=None,
                  parent=None, config=None):
 
         # NEVER change the order of this list. Put new modes at the end:
         self.valid_modes = ['Plain', 'Context', 'Verbose', 'Minimal']
         self.verbose_modes = self.valid_modes[1:3]
 
         VerboseTB.__init__(self, color_scheme=color_scheme, call_pdb=call_pdb,
                            ostream=ostream, tb_offset=tb_offset,
                            long_header=long_header, include_vars=include_vars,
                            check_cache=check_cache, debugger_cls=debugger_cls,
                            parent=parent, config=config)
 
         # Different types of tracebacks are joined with different separators to
         # form a single string.  They are taken from this dict
         self._join_chars = dict(Plain='', Context='\n', Verbose='\n',
                                 Minimal='')
         # set_mode also sets the tb_join_char attribute
         self.set_mode(mode)
 
     def _extract_tb(self, tb):
         if tb:
             return traceback.extract_tb(tb)
         else:
             return None
 
     def structured_traceback(self, etype, value, tb, tb_offset=None, number_of_lines_of_context=5):
         tb_offset = self.tb_offset if tb_offset is None else tb_offset
         mode = self.mode
         if mode in self.verbose_modes:
             # Verbose modes need a full traceback
             return VerboseTB.structured_traceback(
                 self, etype, value, tb, tb_offset, number_of_lines_of_context
             )
         elif mode == 'Minimal':
             return ListTB.get_exception_only(self, etype, value)
         else:
             # We must check the source cache because otherwise we can print
             # out-of-date source code.
             self.check_cache()
             # Now we can extract and format the exception
-            elist = self._extract_tb(tb)
             return ListTB.structured_traceback(
-                self, etype, value, elist, tb_offset, number_of_lines_of_context
+                self, etype, value, tb, tb_offset, number_of_lines_of_context
             )
 
     def stb2text(self, stb):
         """Convert a structured traceback (a list) to a string."""
         return self.tb_join_char.join(stb)
 
 
     def set_mode(self, mode=None):
         """Switch to the desired mode.
 
         If mode is not specified, cycles through the available modes."""
 
         if not mode:
             new_idx = (self.valid_modes.index(self.mode) + 1 ) % \
                       len(self.valid_modes)
             self.mode = self.valid_modes[new_idx]
         elif mode not in self.valid_modes:
             raise ValueError('Unrecognized mode in FormattedTB: <' + mode + '>\n'
                                                                             'Valid modes: ' + str(self.valid_modes))
         else:
             self.mode = mode
         # include variable details only in 'Verbose' mode
         self.include_vars = (self.mode == self.valid_modes[2])
         # Set the join character for generating text tracebacks
         self.tb_join_char = self._join_chars[self.mode]
 
     # some convenient shortcuts
     def plain(self):
         self.set_mode(self.valid_modes[0])
 
     def context(self):
         self.set_mode(self.valid_modes[1])
 
     def verbose(self):
         self.set_mode(self.valid_modes[2])
 
     def minimal(self):
         self.set_mode(self.valid_modes[3])
 
 
 #----------------------------------------------------------------------------
 class AutoFormattedTB(FormattedTB):
     """A traceback printer which can be called on the fly.
 
     It will find out about exceptions by itself.
 
     A brief example::
 
         AutoTB = AutoFormattedTB(mode = 'Verbose',color_scheme='Linux')
         try:
           ...
         except:
           AutoTB()  # or AutoTB(out=logfile) where logfile is an open file object
     """
 
     def __call__(self, etype=None, evalue=None, etb=None,
                  out=None, tb_offset=None):
         """Print out a formatted exception traceback.
 
         Optional arguments:
           - out: an open file-like object to direct output to.
 
           - tb_offset: the number of frames to skip over in the stack, on a
           per-call basis (this overrides temporarily the instance's tb_offset
           given at initialization time.  """
 
         if out is None:
             out = self.ostream
         out.flush()
         out.write(self.text(etype, evalue, etb, tb_offset))
         out.write('\n')
         out.flush()
         # FIXME: we should remove the auto pdb behavior from here and leave
         # that to the clients.
         try:
             self.debugger()
         except KeyboardInterrupt:
             print("\nKeyboardInterrupt")
 
     def structured_traceback(self, etype=None, value=None, tb=None,
                              tb_offset=None, number_of_lines_of_context=5):
         if etype is None:
             etype, value, tb = sys.exc_info()
         self.tb = tb
         return FormattedTB.structured_traceback(
             self, etype, value, tb, tb_offset, number_of_lines_of_context)
 
 
 #---------------------------------------------------------------------------
 
 # A simple class to preserve Nathan's original functionality.
 class ColorTB(FormattedTB):
     """Shorthand to initialize a FormattedTB in Linux colors mode."""
 
     def __init__(self, color_scheme='Linux', call_pdb=0, **kwargs):
         FormattedTB.__init__(self, color_scheme=color_scheme,
                              call_pdb=call_pdb, **kwargs)
 
 
 class SyntaxTB(ListTB):
     """Extension which holds some state: the last exception value"""
 
     def __init__(self, color_scheme='NoColor', parent=None, config=None):
         ListTB.__init__(self, color_scheme, parent=parent, config=config)
         self.last_syntax_error = None
 
     def __call__(self, etype, value, elist):
         self.last_syntax_error = value
 
         ListTB.__call__(self, etype, value, elist)
 
     def structured_traceback(self, etype, value, elist, tb_offset=None,
                              context=5):
         # If the source file has been edited, the line in the syntax error can
         # be wrong (retrieved from an outdated cache). This replaces it with
         # the current value.
         if isinstance(value, SyntaxError) \
                 and isinstance(value.filename, str) \
                 and isinstance(value.lineno, int):
             linecache.checkcache(value.filename)
             newtext = linecache.getline(value.filename, value.lineno)
             if newtext:
                 value.text = newtext
         self.last_syntax_error = value
         return super(SyntaxTB, self).structured_traceback(etype, value, elist,
                                                           tb_offset=tb_offset, context=context)
 
     def clear_err_state(self):
         """Return the current error state and clear it"""
         e = self.last_syntax_error
         self.last_syntax_error = None
         return e
 
     def stb2text(self, stb):
         """Convert a structured traceback (a list) to a string."""
         return ''.join(stb)
 
 
 # some internal-use functions
 def text_repr(value):
     """Hopefully pretty robust repr equivalent."""
     # this is pretty horrible but should always return *something*
     try:
         return pydoc.text.repr(value)
     except KeyboardInterrupt:
         raise
     except:
         try:
             return repr(value)
         except KeyboardInterrupt:
             raise
         except:
             try:
                 # all still in an except block so we catch
                 # getattr raising
                 name = getattr(value, '__name__', None)
                 if name:
                     # ick, recursion
                     return text_repr(name)
                 klass = getattr(value, '__class__', None)
                 if klass:
                     return '%s instance' % text_repr(klass)
             except KeyboardInterrupt:
                 raise
             except:
                 return 'UNRECOVERABLE REPR FAILURE'
 
 
 def eqrepr(value, repr=text_repr):
     return '=%s' % repr(value)
 
 
 def nullrepr(value, repr=text_repr):
     return ''

IPython/lib/clipboard.py

@@ -1,69 +1,69 b''
 """ Utilities for accessing the platform's clipboard.
 """
 
 import subprocess
 
 from IPython.core.error import TryNext
 import IPython.utils.py3compat as py3compat
 
 class ClipboardEmpty(ValueError):
     pass
 
 def win32_clipboard_get():
     """ Get the current clipboard's text on Windows.
 
     Requires Mark Hammond's pywin32 extensions.
     """
     try:
         import win32clipboard
     except ImportError:
         raise TryNext("Getting text from the clipboard requires the pywin32 "
                       "extensions: http://sourceforge.net/projects/pywin32/")
     win32clipboard.OpenClipboard()
     try:
         text = win32clipboard.GetClipboardData(win32clipboard.CF_UNICODETEXT)
     except (TypeError, win32clipboard.error):
         try:
             text = win32clipboard.GetClipboardData(win32clipboard.CF_TEXT)
             text = py3compat.cast_unicode(text, py3compat.DEFAULT_ENCODING)
         except (TypeError, win32clipboard.error):
             raise ClipboardEmpty
     finally:
         win32clipboard.CloseClipboard()
     return text
 
-def osx_clipboard_get():
+def osx_clipboard_get() -> str:
     """ Get the clipboard's text on OS X.
     """
     p = subprocess.Popen(['pbpaste', '-Prefer', 'ascii'],
         stdout=subprocess.PIPE)
-    text, stderr = p.communicate()
+    bytes_, stderr = p.communicate()
     # Text comes in with old Mac \r line endings. Change them to \n.
-    text = text.replace(b'\r', b'\n')
-    text = py3compat.cast_unicode(text, py3compat.DEFAULT_ENCODING)
+    bytes_ = bytes_.replace(b'\r', b'\n')
+    text = py3compat.decode(bytes_)
     return text
 
 def tkinter_clipboard_get():
     """ Get the clipboard's text using Tkinter.
 
     This is the default on systems that are not Windows or OS X. It may
     interfere with other UI toolkits and should be replaced with an
     implementation that uses that toolkit.
     """
     try:
         from tkinter import Tk, TclError 
     except ImportError:
         raise TryNext("Getting text from the clipboard on this platform requires tkinter.")
         
     root = Tk()
     root.withdraw()
     try:
         text = root.clipboard_get()
     except TclError:
         raise ClipboardEmpty
     finally:
         root.destroy()
     text = py3compat.cast_unicode(text, py3compat.DEFAULT_ENCODING)
     return text
 
 

IPython/lib/pretty.py

@@ -1,871 +1,863 b''
 # -*- coding: utf-8 -*-
 """
 Python advanced pretty printer.  This pretty printer is intended to
 replace the old `pprint` python module which does not allow developers
 to provide their own pretty print callbacks.
 
 This module is based on ruby's `prettyprint.rb` library by `Tanaka Akira`.
 
 
 Example Usage
 -------------
 
 To directly print the representation of an object use `pprint`::
 
     from pretty import pprint
     pprint(complex_object)
 
 To get a string of the output use `pretty`::
 
     from pretty import pretty
     string = pretty(complex_object)
 
 
 Extending
 ---------
 
 The pretty library allows developers to add pretty printing rules for their
 own objects.  This process is straightforward.  All you have to do is to
 add a `_repr_pretty_` method to your object and call the methods on the
 pretty printer passed::
 
     class MyObject(object):
 
         def _repr_pretty_(self, p, cycle):
             ...
 
 Here is an example implementation of a `_repr_pretty_` method for a list
 subclass::
 
     class MyList(list):
 
         def _repr_pretty_(self, p, cycle):
             if cycle:
                 p.text('MyList(...)')
             else:
                 with p.group(8, 'MyList([', '])'):
                     for idx, item in enumerate(self):
                         if idx:
                             p.text(',')
                             p.breakable()
                         p.pretty(item)
 
 The `cycle` parameter is `True` if pretty detected a cycle.  You *have* to
 react to that or the result is an infinite loop.  `p.text()` just adds
 non breaking text to the output, `p.breakable()` either adds a whitespace
 or breaks here.  If you pass it an argument it's used instead of the
 default space.  `p.pretty` prettyprints another object using the pretty print
 method.
 
 The first parameter to the `group` function specifies the extra indentation
 of the next line.  In this example the next item will either be on the same
 line (if the items are short enough) or aligned with the right edge of the
 opening bracket of `MyList`.
 
 If you just want to indent something you can use the group function
 without open / close parameters.  You can also use this code::
 
     with p.indent(2):
         ...
 
 Inheritance diagram:
 
 .. inheritance-diagram:: IPython.lib.pretty
    :parts: 3
 
 :copyright: 2007 by Armin Ronacher.
             Portions (c) 2009 by Robert Kern.
 :license: BSD License.
 """
 
 from contextlib import contextmanager
 import datetime
 import os
 import re
 import sys
 import types
 from collections import deque
 from inspect import signature
 from io import StringIO
 from warnings import warn
 
 from IPython.utils.decorators import undoc
 from IPython.utils.py3compat import PYPY
 
 __all__ = ['pretty', 'pprint', 'PrettyPrinter', 'RepresentationPrinter',
     'for_type', 'for_type_by_name']
 
 
 MAX_SEQ_LENGTH = 1000
-# The language spec says that dicts preserve order from 3.7, but CPython
-# does so from 3.6, so it seems likely that people will expect that.
-DICT_IS_ORDERED = True
 _re_pattern_type = type(re.compile(''))
 
 def _safe_getattr(obj, attr, default=None):
     """Safe version of getattr.
 
     Same as getattr, but will return ``default`` on any Exception,
     rather than raising.
     """
     try:
         return getattr(obj, attr, default)
     except Exception:
         return default
 
 @undoc
 class CUnicodeIO(StringIO):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         warn(("CUnicodeIO is deprecated since IPython 6.0. "
               "Please use io.StringIO instead."),
              DeprecationWarning, stacklevel=2)
 
 def _sorted_for_pprint(items):
     """
     Sort the given items for pretty printing. Since some predictable
     sorting is better than no sorting at all, we sort on the string
     representation if normal sorting fails.
     """
     items = list(items)
     try:
         return sorted(items)
     except Exception:
         try:
             return sorted(items, key=str)
         except Exception:
             return items
 
 def pretty(obj, verbose=False, max_width=79, newline='\n', max_seq_length=MAX_SEQ_LENGTH):
     """
     Pretty print the object's representation.
     """
     stream = StringIO()
     printer = RepresentationPrinter(stream, verbose, max_width, newline, max_seq_length=max_seq_length)
     printer.pretty(obj)
     printer.flush()
     return stream.getvalue()
 
 
 def pprint(obj, verbose=False, max_width=79, newline='\n', max_seq_length=MAX_SEQ_LENGTH):
     """
     Like `pretty` but print to stdout.
     """
     printer = RepresentationPrinter(sys.stdout, verbose, max_width, newline, max_seq_length=max_seq_length)
     printer.pretty(obj)
     printer.flush()
     sys.stdout.write(newline)
     sys.stdout.flush()
 
 class _PrettyPrinterBase(object):
 
     @contextmanager
     def indent(self, indent):
         """with statement support for indenting/dedenting."""
         self.indentation += indent
         try:
             yield
         finally:
             self.indentation -= indent
 
     @contextmanager
     def group(self, indent=0, open='', close=''):
         """like begin_group / end_group but for the with statement."""
         self.begin_group(indent, open)
         try:
             yield
         finally:
             self.end_group(indent, close)
 
 class PrettyPrinter(_PrettyPrinterBase):
     """
     Baseclass for the `RepresentationPrinter` prettyprinter that is used to
     generate pretty reprs of objects.  Contrary to the `RepresentationPrinter`
     this printer knows nothing about the default pprinters or the `_repr_pretty_`
     callback method.
     """
 
     def __init__(self, output, max_width=79, newline='\n', max_seq_length=MAX_SEQ_LENGTH):
         self.output = output
         self.max_width = max_width
         self.newline = newline
         self.max_seq_length = max_seq_length
         self.output_width = 0
         self.buffer_width = 0
         self.buffer = deque()
 
         root_group = Group(0)
         self.group_stack = [root_group]
         self.group_queue = GroupQueue(root_group)
         self.indentation = 0
 
     def _break_outer_groups(self):
         while self.max_width < self.output_width + self.buffer_width:
             group = self.group_queue.deq()
             if not group:
                 return
             while group.breakables:
                 x = self.buffer.popleft()
                 self.output_width = x.output(self.output, self.output_width)
                 self.buffer_width -= x.width
             while self.buffer and isinstance(self.buffer[0], Text):
                 x = self.buffer.popleft()
                 self.output_width = x.output(self.output, self.output_width)
                 self.buffer_width -= x.width
 
     def text(self, obj):
         """Add literal text to the output."""
         width = len(obj)
         if self.buffer:
             text = self.buffer[-1]
             if not isinstance(text, Text):
                 text = Text()
                 self.buffer.append(text)
             text.add(obj, width)
             self.buffer_width += width
             self._break_outer_groups()
         else:
             self.output.write(obj)
             self.output_width += width
 
     def breakable(self, sep=' '):
         """
         Add a breakable separator to the output.  This does not mean that it
         will automatically break here.  If no breaking on this position takes
         place the `sep` is inserted which default to one space.
         """
         width = len(sep)
         group = self.group_stack[-1]
         if group.want_break:
             self.flush()
             self.output.write(self.newline)
             self.output.write(' ' * self.indentation)
             self.output_width = self.indentation
             self.buffer_width = 0
         else:
             self.buffer.append(Breakable(sep, width, self))
             self.buffer_width += width
             self._break_outer_groups()
 
     def break_(self):
         """
         Explicitly insert a newline into the output, maintaining correct indentation.
         """
         self.flush()
         self.output.write(self.newline)
         self.output.write(' ' * self.indentation)
         self.output_width = self.indentation
         self.buffer_width = 0
 
 
     def begin_group(self, indent=0, open=''):
         """
         Begin a group.  If you want support for python < 2.5 which doesn't has
         the with statement this is the preferred way:
 
             p.begin_group(1, '{')
             ...
             p.end_group(1, '}')
 
         The python 2.5 expression would be this:
 
             with p.group(1, '{', '}'):
                 ...
 
         The first parameter specifies the indentation for the next line (usually
         the width of the opening text), the second the opening text.  All
         parameters are optional.
         """
         if open:
             self.text(open)
         group = Group(self.group_stack[-1].depth + 1)
         self.group_stack.append(group)
         self.group_queue.enq(group)
         self.indentation += indent
 
     def _enumerate(self, seq):
         """like enumerate, but with an upper limit on the number of items"""
         for idx, x in enumerate(seq):
             if self.max_seq_length and idx >= self.max_seq_length:
                 self.text(',')
                 self.breakable()
                 self.text('...')
                 return
             yield idx, x
 
     def end_group(self, dedent=0, close=''):
         """End a group. See `begin_group` for more details."""
         self.indentation -= dedent
         group = self.group_stack.pop()
         if not group.breakables:
             self.group_queue.remove(group)
         if close:
             self.text(close)
 
     def flush(self):
         """Flush data that is left in the buffer."""
         for data in self.buffer:
             self.output_width += data.output(self.output, self.output_width)
         self.buffer.clear()
         self.buffer_width = 0
 
 
 def _get_mro(obj_class):
     """ Get a reasonable method resolution order of a class and its superclasses
     for both old-style and new-style classes.
     """
     if not hasattr(obj_class, '__mro__'):
         # Old-style class. Mix in object to make a fake new-style class.
         try:
             obj_class = type(obj_class.__name__, (obj_class, object), {})
         except TypeError:
             # Old-style extension type that does not descend from object.
             # FIXME: try to construct a more thorough MRO.
             mro = [obj_class]
         else:
             mro = obj_class.__mro__[1:-1]
     else:
         mro = obj_class.__mro__
     return mro
 
 
 class RepresentationPrinter(PrettyPrinter):
     """
     Special pretty printer that has a `pretty` method that calls the pretty
     printer for a python object.
 
     This class stores processing data on `self` so you must *never* use
     this class in a threaded environment.  Always lock it or reinstanciate
     it.
 
     Instances also have a verbose flag callbacks can access to control their
     output.  For example the default instance repr prints all attributes and
     methods that are not prefixed by an underscore if the printer is in
     verbose mode.
     """
 
     def __init__(self, output, verbose=False, max_width=79, newline='\n',
         singleton_pprinters=None, type_pprinters=None, deferred_pprinters=None,
         max_seq_length=MAX_SEQ_LENGTH):
 
         PrettyPrinter.__init__(self, output, max_width, newline, max_seq_length=max_seq_length)
         self.verbose = verbose
         self.stack = []
         if singleton_pprinters is None:
             singleton_pprinters = _singleton_pprinters.copy()
         self.singleton_pprinters = singleton_pprinters
         if type_pprinters is None:
             type_pprinters = _type_pprinters.copy()
         self.type_pprinters = type_pprinters
         if deferred_pprinters is None:
             deferred_pprinters = _deferred_type_pprinters.copy()
         self.deferred_pprinters = deferred_pprinters
 
     def pretty(self, obj):
         """Pretty print the given object."""
         obj_id = id(obj)
         cycle = obj_id in self.stack
         self.stack.append(obj_id)
         self.begin_group()
         try:
             obj_class = _safe_getattr(obj, '__class__', None) or type(obj)
             # First try to find registered singleton printers for the type.
             try:
                 printer = self.singleton_pprinters[obj_id]
             except (TypeError, KeyError):
                 pass
             else:
                 return printer(obj, self, cycle)
             # Next walk the mro and check for either:
             #   1) a registered printer
             #   2) a _repr_pretty_ method
             for cls in _get_mro(obj_class):
                 if cls in self.type_pprinters:
                     # printer registered in self.type_pprinters
                     return self.type_pprinters[cls](obj, self, cycle)
                 else:
                     # deferred printer
                     printer = self._in_deferred_types(cls)
                     if printer is not None:
                         return printer(obj, self, cycle)
                     else:
                         # Finally look for special method names.
                         # Some objects automatically create any requested
                         # attribute. Try to ignore most of them by checking for
                         # callability.
                         if '_repr_pretty_' in cls.__dict__:
                             meth = cls._repr_pretty_
                             if callable(meth):
                                 return meth(obj, self, cycle)
                         if cls is not object \
                                 and callable(cls.__dict__.get('__repr__')):
                             return _repr_pprint(obj, self, cycle)
 
             return _default_pprint(obj, self, cycle)
         finally:
             self.end_group()
             self.stack.pop()
 
     def _in_deferred_types(self, cls):
         """
         Check if the given class is specified in the deferred type registry.
 
         Returns the printer from the registry if it exists, and None if the
         class is not in the registry. Successful matches will be moved to the
         regular type registry for future use.
         """
         mod = _safe_getattr(cls, '__module__', None)
         name = _safe_getattr(cls, '__name__', None)
         key = (mod, name)
         printer = None
         if key in self.deferred_pprinters:
             # Move the printer over to the regular registry.
             printer = self.deferred_pprinters.pop(key)
             self.type_pprinters[cls] = printer
         return printer
 
 
 class Printable(object):
 
     def output(self, stream, output_width):
         return output_width
 
 
 class Text(Printable):
 
     def __init__(self):
         self.objs = []
         self.width = 0
 
     def output(self, stream, output_width):
         for obj in self.objs:
             stream.write(obj)
         return output_width + self.width
 
     def add(self, obj, width):
         self.objs.append(obj)
         self.width += width
 
 
 class Breakable(Printable):
 
     def __init__(self, seq, width, pretty):
         self.obj = seq
         self.width = width
         self.pretty = pretty
         self.indentation = pretty.indentation
         self.group = pretty.group_stack[-1]
         self.group.breakables.append(self)
 
     def output(self, stream, output_width):
         self.group.breakables.popleft()
         if self.group.want_break:
             stream.write(self.pretty.newline)
             stream.write(' ' * self.indentation)
             return self.indentation
         if not self.group.breakables:
             self.pretty.group_queue.remove(self.group)
         stream.write(self.obj)
         return output_width + self.width
 
 
 class Group(Printable):
 
     def __init__(self, depth):
         self.depth = depth
         self.breakables = deque()
         self.want_break = False
 
 
 class GroupQueue(object):
 
     def __init__(self, *groups):
         self.queue = []
         for group in groups:
             self.enq(group)
 
     def enq(self, group):
         depth = group.depth
         while depth > len(self.queue) - 1:
             self.queue.append([])
         self.queue[depth].append(group)
 
     def deq(self):
         for stack in self.queue:
             for idx, group in enumerate(reversed(stack)):
                 if group.breakables:
                     del stack[idx]
                     group.want_break = True
                     return group
             for group in stack:
                 group.want_break = True
             del stack[:]
 
     def remove(self, group):
         try:
             self.queue[group.depth].remove(group)
         except ValueError:
             pass
 
 
 def _default_pprint(obj, p, cycle):
     """
     The default print function.  Used if an object does not provide one and
     it's none of the builtin objects.
     """
     klass = _safe_getattr(obj, '__class__', None) or type(obj)
     if _safe_getattr(klass, '__repr__', None) is not object.__repr__:
         # A user-provided repr. Find newlines and replace them with p.break_()
         _repr_pprint(obj, p, cycle)
         return
     p.begin_group(1, '<')
     p.pretty(klass)
     p.text(' at 0x%x' % id(obj))
     if cycle:
         p.text(' ...')
     elif p.verbose:
         first = True
         for key in dir(obj):
             if not key.startswith('_'):
                 try:
                     value = getattr(obj, key)
                 except AttributeError:
                     continue
                 if isinstance(value, types.MethodType):
                     continue
                 if not first:
                     p.text(',')
                 p.breakable()
                 p.text(key)
                 p.text('=')
                 step = len(key) + 1
                 p.indentation += step
                 p.pretty(value)
                 p.indentation -= step
                 first = False
     p.end_group(1, '>')
 
 
 def _seq_pprinter_factory(start, end):
     """
     Factory that returns a pprint function useful for sequences.  Used by
     the default pprint for tuples, dicts, and lists.
     """
     def inner(obj, p, cycle):
         if cycle:
             return p.text(start + '...' + end)
         step = len(start)
         p.begin_group(step, start)
         for idx, x in p._enumerate(obj):
             if idx:
                 p.text(',')
                 p.breakable()
             p.pretty(x)
         if len(obj) == 1 and type(obj) is tuple:
             # Special case for 1-item tuples.
             p.text(',')
         p.end_group(step, end)
     return inner
 
 
 def _set_pprinter_factory(start, end):
     """
     Factory that returns a pprint function useful for sets and frozensets.
     """
     def inner(obj, p, cycle):
         if cycle:
             return p.text(start + '...' + end)
         if len(obj) == 0:
             # Special case.
             p.text(type(obj).__name__ + '()')
         else:
             step = len(start)
             p.begin_group(step, start)
             # Like dictionary keys, we will try to sort the items if there aren't too many
             if not (p.max_seq_length and len(obj) >= p.max_seq_length):
                 items = _sorted_for_pprint(obj)
             else:
                 items = obj
             for idx, x in p._enumerate(items):
                 if idx:
                     p.text(',')
                     p.breakable()
                 p.pretty(x)
             p.end_group(step, end)
     return inner
 
 
 def _dict_pprinter_factory(start, end):
     """
     Factory that returns a pprint function used by the default pprint of
     dicts and dict proxies.
     """
     def inner(obj, p, cycle):
         if cycle:
             return p.text('{...}')
         step = len(start)
         p.begin_group(step, start)
         keys = obj.keys()
-        # if dict isn't large enough to be truncated, sort keys before displaying
-        # From Python 3.7, dicts preserve order by definition, so we don't sort.
-        if not DICT_IS_ORDERED \
-                and not (p.max_seq_length and len(obj) >= p.max_seq_length):
-            keys = _sorted_for_pprint(keys)
         for idx, key in p._enumerate(keys):
             if idx:
                 p.text(',')
                 p.breakable()
             p.pretty(key)
             p.text(': ')
             p.pretty(obj[key])
         p.end_group(step, end)
     return inner
 
 
 def _super_pprint(obj, p, cycle):
     """The pprint for the super type."""
     p.begin_group(8, '<super: ')
     p.pretty(obj.__thisclass__)
     p.text(',')
     p.breakable()
     if PYPY: # In PyPy, super() objects don't have __self__ attributes
         dself = obj.__repr__.__self__
         p.pretty(None if dself is obj else dself)
     else:
         p.pretty(obj.__self__)
     p.end_group(8, '>')
 
 
 def _re_pattern_pprint(obj, p, cycle):
     """The pprint function for regular expression patterns."""
     p.text('re.compile(')
     pattern = repr(obj.pattern)
     if pattern[:1] in 'uU':
         pattern = pattern[1:]
         prefix = 'ur'
     else:
         prefix = 'r'
     pattern = prefix + pattern.replace('\\\\', '\\')
     p.text(pattern)
     if obj.flags:
         p.text(',')
         p.breakable()
         done_one = False
         for flag in ('TEMPLATE', 'IGNORECASE', 'LOCALE', 'MULTILINE', 'DOTALL',
             'UNICODE', 'VERBOSE', 'DEBUG'):
             if obj.flags & getattr(re, flag):
                 if done_one:
                     p.text('|')
                 p.text('re.' + flag)
                 done_one = True
     p.text(')')
 
 
 def _type_pprint(obj, p, cycle):
     """The pprint for classes and types."""
     # Heap allocated types might not have the module attribute,
     # and others may set it to None.
 
     # Checks for a __repr__ override in the metaclass. Can't compare the
     # type(obj).__repr__ directly because in PyPy the representation function
     # inherited from type isn't the same type.__repr__
     if [m for m in _get_mro(type(obj)) if "__repr__" in vars(m)][:1] != [type]:
         _repr_pprint(obj, p, cycle)
         return
 
     mod = _safe_getattr(obj, '__module__', None)
     try:
         name = obj.__qualname__
         if not isinstance(name, str):
             # This can happen if the type implements __qualname__ as a property
             # or other descriptor in Python 2.
             raise Exception("Try __name__")
     except Exception:
         name = obj.__name__
         if not isinstance(name, str):
             name = '<unknown type>'
 
     if mod in (None, '__builtin__', 'builtins', 'exceptions'):
         p.text(name)
     else:
         p.text(mod + '.' + name)
 
 
 def _repr_pprint(obj, p, cycle):
     """A pprint that just redirects to the normal repr function."""
     # Find newlines and replace them with p.break_()
     output = repr(obj)
     for idx,output_line in enumerate(output.splitlines()):
         if idx:
             p.break_()
         p.text(output_line)
 
 
 def _function_pprint(obj, p, cycle):
     """Base pprint for all functions and builtin functions."""
     name = _safe_getattr(obj, '__qualname__', obj.__name__)
     mod = obj.__module__
     if mod and mod not in ('__builtin__', 'builtins', 'exceptions'):
         name = mod + '.' + name
     try:
        func_def = name + str(signature(obj))
     except ValueError:
        func_def = name
     p.text('<function %s>' % func_def)
 
 
 def _exception_pprint(obj, p, cycle):
     """Base pprint for all exceptions."""
     name = getattr(obj.__class__, '__qualname__', obj.__class__.__name__)
     if obj.__class__.__module__ not in ('exceptions', 'builtins'):
         name = '%s.%s' % (obj.__class__.__module__, name)
     step = len(name) + 1
     p.begin_group(step, name + '(')
     for idx, arg in enumerate(getattr(obj, 'args', ())):
         if idx:
             p.text(',')
             p.breakable()
         p.pretty(arg)
     p.end_group(step, ')')
 
 
 #: the exception base
 try:
     _exception_base = BaseException
 except NameError:
     _exception_base = Exception
 
 
 #: printers for builtin types
 _type_pprinters = {
     int:                        _repr_pprint,
     float:                      _repr_pprint,
     str:                        _repr_pprint,
     tuple:                      _seq_pprinter_factory('(', ')'),
     list:                       _seq_pprinter_factory('[', ']'),
     dict:                       _dict_pprinter_factory('{', '}'),
     set:                        _set_pprinter_factory('{', '}'),
     frozenset:                  _set_pprinter_factory('frozenset({', '})'),
     super:                      _super_pprint,
     _re_pattern_type:           _re_pattern_pprint,
     type:                       _type_pprint,
     types.FunctionType:         _function_pprint,
     types.BuiltinFunctionType:  _function_pprint,
     types.MethodType:           _repr_pprint,
     datetime.datetime:          _repr_pprint,
     datetime.timedelta:         _repr_pprint,
     _exception_base:            _exception_pprint
 }
 
 # render os.environ like a dict
 _env_type = type(os.environ)
 # future-proof in case os.environ becomes a plain dict?
 if _env_type is not dict:
     _type_pprinters[_env_type] = _dict_pprinter_factory('environ{', '}')
 
 try:
     # In PyPy, types.DictProxyType is dict, setting the dictproxy printer
     # using dict.setdefault avoids overwriting the dict printer
     _type_pprinters.setdefault(types.DictProxyType,
                                _dict_pprinter_factory('dict_proxy({', '})'))
     _type_pprinters[types.ClassType] = _type_pprint
     _type_pprinters[types.SliceType] = _repr_pprint
 except AttributeError: # Python 3
     _type_pprinters[types.MappingProxyType] = \
         _dict_pprinter_factory('mappingproxy({', '})')
     _type_pprinters[slice] = _repr_pprint
 
 try:
     _type_pprinters[long] = _repr_pprint
     _type_pprinters[unicode] = _repr_pprint
 except NameError:
     _type_pprinters[range] = _repr_pprint
     _type_pprinters[bytes] = _repr_pprint
 
 #: printers for types specified by name
 _deferred_type_pprinters = {
 }
 
 def for_type(typ, func):
     """
     Add a pretty printer for a given type.
     """
     oldfunc = _type_pprinters.get(typ, None)
     if func is not None:
         # To support easy restoration of old pprinters, we need to ignore Nones.
         _type_pprinters[typ] = func
     return oldfunc
 
 def for_type_by_name(type_module, type_name, func):
     """
     Add a pretty printer for a type specified by the module and name of a type
     rather than the type object itself.
     """
     key = (type_module, type_name)
     oldfunc = _deferred_type_pprinters.get(key, None)
     if func is not None:
         # To support easy restoration of old pprinters, we need to ignore Nones.
         _deferred_type_pprinters[key] = func
     return oldfunc
 
 
 #: printers for the default singletons
 _singleton_pprinters = dict.fromkeys(map(id, [None, True, False, Ellipsis,
                                       NotImplemented]), _repr_pprint)
 
 
 def _defaultdict_pprint(obj, p, cycle):
     name = obj.__class__.__name__
     with p.group(len(name) + 1, name + '(', ')'):
         if cycle:
             p.text('...')
         else:
             p.pretty(obj.default_factory)
             p.text(',')
             p.breakable()
             p.pretty(dict(obj))
 
 def _ordereddict_pprint(obj, p, cycle):
     name = obj.__class__.__name__
     with p.group(len(name) + 1, name + '(', ')'):
         if cycle:
             p.text('...')
         elif len(obj):
             p.pretty(list(obj.items()))
 
 def _deque_pprint(obj, p, cycle):
     name = obj.__class__.__name__
     with p.group(len(name) + 1, name + '(', ')'):
         if cycle:
             p.text('...')
         else:
             p.pretty(list(obj))
 
 
 def _counter_pprint(obj, p, cycle):
     name = obj.__class__.__name__
     with p.group(len(name) + 1, name + '(', ')'):
         if cycle:
             p.text('...')
         elif len(obj):
             p.pretty(dict(obj))
 
 for_type_by_name('collections', 'defaultdict', _defaultdict_pprint)
 for_type_by_name('collections', 'OrderedDict', _ordereddict_pprint)
 for_type_by_name('collections', 'deque', _deque_pprint)
 for_type_by_name('collections', 'Counter', _counter_pprint)
 
 if __name__ == '__main__':
     from random import randrange
     class Foo(object):
         def __init__(self):
             self.foo = 1
             self.bar = re.compile(r'\s+')
             self.blub = dict.fromkeys(range(30), randrange(1, 40))
             self.hehe = 23424.234234
             self.list = ["blub", "blah", self]
 
         def get_foo(self):
             print("foo")
 
     pprint(Foo(), verbose=True)

IPython/paths.py

@@ -1,119 +1,119 b''
 """Find files and directories which IPython uses.
 """
 import os.path
 import shutil
 import tempfile
 from warnings import warn
 
 import IPython
 from IPython.utils.importstring import import_item
 from IPython.utils.path import (
     get_home_dir, get_xdg_dir, get_xdg_cache_dir, compress_user, _writable_dir,
     ensure_dir_exists, fs_encoding)
 from IPython.utils import py3compat
 
-def get_ipython_dir():
+def get_ipython_dir() -> str:
     """Get the IPython directory for this platform and user.
 
     This uses the logic in `get_home_dir` to find the home directory
     and then adds .ipython to the end of the path.
     """
 
     env = os.environ
     pjoin = os.path.join
 
 
     ipdir_def = '.ipython'
 
     home_dir = get_home_dir()
     xdg_dir = get_xdg_dir()
 
-    # import pdb; pdb.set_trace()  # dbg
     if 'IPYTHON_DIR' in env:
-        warn('The environment variable IPYTHON_DIR is deprecated. '
-                'Please use IPYTHONDIR instead.')
+        warn('The environment variable IPYTHON_DIR is deprecated since IPython 3.0. '
+                'Please use IPYTHONDIR instead.', DeprecationWarning)
     ipdir = env.get('IPYTHONDIR', env.get('IPYTHON_DIR', None))
     if ipdir is None:
         # not set explicitly, use ~/.ipython
         ipdir = pjoin(home_dir, ipdir_def)
         if xdg_dir:
             # Several IPython versions (up to 1.x) defaulted to .config/ipython
             # on Linux. We have decided to go back to using .ipython everywhere
             xdg_ipdir = pjoin(xdg_dir, 'ipython')
 
             if _writable_dir(xdg_ipdir):
                 cu = compress_user
                 if os.path.exists(ipdir):
                     warn(('Ignoring {0} in favour of {1}. Remove {0} to '
                         'get rid of this message').format(cu(xdg_ipdir), cu(ipdir)))
                 elif os.path.islink(xdg_ipdir):
                     warn(('{0} is deprecated. Move link to {1} to '
                         'get rid of this message').format(cu(xdg_ipdir), cu(ipdir)))
                 else:
                     warn('Moving {0} to {1}'.format(cu(xdg_ipdir), cu(ipdir)))
                     shutil.move(xdg_ipdir, ipdir)
 
     ipdir = os.path.normpath(os.path.expanduser(ipdir))
 
     if os.path.exists(ipdir) and not _writable_dir(ipdir):
         # ipdir exists, but is not writable
         warn("IPython dir '{0}' is not a writable location,"
                 " using a temp directory.".format(ipdir))
         ipdir = tempfile.mkdtemp()
     elif not os.path.exists(ipdir):
         parent = os.path.dirname(ipdir)
         if not _writable_dir(parent):
             # ipdir does not exist and parent isn't writable
             warn("IPython parent '{0}' is not a writable location,"
                     " using a temp directory.".format(parent))
             ipdir = tempfile.mkdtemp()
+    assert isinstance(ipdir, str), "all path manipulation should be str(unicode), but are not."
+    return ipdir
 
-    return py3compat.cast_unicode(ipdir, fs_encoding)
 
-
-def get_ipython_cache_dir():
+def get_ipython_cache_dir() -> str:
     """Get the cache directory it is created if it does not exist."""
     xdgdir = get_xdg_cache_dir()
     if xdgdir is None:
         return get_ipython_dir()
     ipdir = os.path.join(xdgdir, "ipython")
     if not os.path.exists(ipdir) and _writable_dir(xdgdir):
         ensure_dir_exists(ipdir)
     elif not _writable_dir(xdgdir):
         return get_ipython_dir()
 
-    return py3compat.cast_unicode(ipdir, fs_encoding)
+    return ipdir
 
 
-def get_ipython_package_dir():
+def get_ipython_package_dir() -> str:
     """Get the base directory where IPython itself is installed."""
     ipdir = os.path.dirname(IPython.__file__)
-    return py3compat.cast_unicode(ipdir, fs_encoding)
+    assert isinstance(ipdir, str)
+    return ipdir
 
 
 def get_ipython_module_path(module_str):
     """Find the path to an IPython module in this version of IPython.
 
     This will always find the version of the module that is in this importable
     IPython package. This will always return the path to the ``.py``
     version of the module.
     """
     if module_str == 'IPython':
         return os.path.join(get_ipython_package_dir(), '__init__.py')
     mod = import_item(module_str)
     the_path = mod.__file__.replace('.pyc', '.py')
     the_path = the_path.replace('.pyo', '.py')
     return py3compat.cast_unicode(the_path, fs_encoding)
 
 def locate_profile(profile='default'):
     """Find the path to the folder associated with a given profile.
 
     I.e. find $IPYTHONDIR/profile_whatever.
     """
     from IPython.core.profiledir import ProfileDir, ProfileDirError
     try:
         pd = ProfileDir.find_profile_dir_by_name(get_ipython_dir(), profile)
     except ProfileDirError:
         # IOError makes more sense when people are expecting a path
         raise IOError("Couldn't find profile %r" % profile)
     return pd.location

IPython/terminal/debugger.py

@@ -1,120 +1,126 b''
 import signal
 import sys
 
 from IPython.core.debugger import Pdb
 
 from IPython.core.completer import IPCompleter
 from .ptutils import IPythonPTCompleter
 from .shortcuts import suspend_to_bg, cursor_in_leading_ws
 
 from prompt_toolkit.enums import DEFAULT_BUFFER
 from prompt_toolkit.filters import (Condition, has_focus, has_selection,
     vi_insert_mode, emacs_insert_mode)
 from prompt_toolkit.key_binding import KeyBindings
 from prompt_toolkit.key_binding.bindings.completion import display_completions_like_readline
 from pygments.token import Token
 from prompt_toolkit.shortcuts.prompt import PromptSession
 from prompt_toolkit.enums import EditingMode
 from prompt_toolkit.formatted_text import PygmentsTokens
 
+from prompt_toolkit import __version__ as ptk_version
+PTK3 = ptk_version.startswith('3.')
+
 
 class TerminalPdb(Pdb):
     """Standalone IPython debugger."""
 
     def __init__(self, *args, **kwargs):
         Pdb.__init__(self, *args, **kwargs)
         self._ptcomp = None
         self.pt_init()
 
     def pt_init(self):
         def get_prompt_tokens():
             return [(Token.Prompt, self.prompt)]
 
         if self._ptcomp is None:
             compl = IPCompleter(shell=self.shell,
                                         namespace={},
                                         global_namespace={},
                                         parent=self.shell,
                                        )
             self._ptcomp = IPythonPTCompleter(compl)
 
         kb = KeyBindings()
         supports_suspend = Condition(lambda: hasattr(signal, 'SIGTSTP'))
         kb.add('c-z', filter=supports_suspend)(suspend_to_bg)
 
         if self.shell.display_completions == 'readlinelike':
             kb.add('tab', filter=(has_focus(DEFAULT_BUFFER)
                                   & ~has_selection
                                   & vi_insert_mode | emacs_insert_mode
                                   & ~cursor_in_leading_ws
                               ))(display_completions_like_readline)
 
-        self.pt_app = PromptSession(
+        options = dict(
                             message=(lambda: PygmentsTokens(get_prompt_tokens())),
                             editing_mode=getattr(EditingMode, self.shell.editing_mode.upper()),
                             key_bindings=kb,
                             history=self.shell.debugger_history,
                             completer=self._ptcomp,
                             enable_history_search=True,
                             mouse_support=self.shell.mouse_support,
                             complete_style=self.shell.pt_complete_style,
                             style=self.shell.style,
-                            inputhook=self.shell.inputhook,
                             color_depth=self.shell.color_depth,
         )
 
+        if not PTK3:
+            options['inputhook'] = self.shell.inputhook
+        self.pt_app = PromptSession(**options)
+
     def cmdloop(self, intro=None):
         """Repeatedly issue a prompt, accept input, parse an initial prefix
         off the received input, and dispatch to action methods, passing them
         the remainder of the line as argument.
 
         override the same methods from cmd.Cmd to provide prompt toolkit replacement.
         """
         if not self.use_rawinput:
             raise ValueError('Sorry ipdb does not support use_rawinput=False')
 
         self.preloop()
 
         try:
             if intro is not None:
                 self.intro = intro
             if self.intro:
                 self.stdout.write(str(self.intro)+"\n")
             stop = None
             while not stop:
                 if self.cmdqueue:
                     line = self.cmdqueue.pop(0)
                 else:
                     self._ptcomp.ipy_completer.namespace = self.curframe_locals
                     self._ptcomp.ipy_completer.global_namespace = self.curframe.f_globals
                     try:
                         line = self.pt_app.prompt() # reset_current_buffer=True)
                     except EOFError:
                         line = 'EOF'
                 line = self.precmd(line)
                 stop = self.onecmd(line)
                 stop = self.postcmd(stop, line)
             self.postloop()
         except Exception:
             raise
 
 
 def set_trace(frame=None):
     """
     Start debugging from `frame`.
 
     If frame is not specified, debugging starts from caller's frame.
     """
     TerminalPdb().set_trace(frame or sys._getframe().f_back)
 
 
 if __name__ == '__main__':
     import pdb
     # IPython.core.debugger.Pdb.trace_dispatch shall not catch
     # bdb.BdbQuit. When started through __main__ and an exception
     # happened after hitting "c", this is needed in order to
     # be able to quit the debugging session (see #9950).
     old_trace_dispatch = pdb.Pdb.trace_dispatch
     pdb.Pdb = TerminalPdb
     pdb.Pdb.trace_dispatch = old_trace_dispatch
     pdb.main()

IPython/terminal/interactiveshell.py

@@ -1,570 +1,640 b''
 """IPython terminal interface using prompt_toolkit"""
 
+import asyncio
 import os
 import sys
 import warnings
 from warnings import warn
 
 from IPython.core.interactiveshell import InteractiveShell, InteractiveShellABC
 from IPython.utils import io
 from IPython.utils.py3compat import input
 from IPython.utils.terminal import toggle_set_term_title, set_term_title, restore_term_title
 from IPython.utils.process import abbrev_cwd
 from traitlets import (
     Bool, Unicode, Dict, Integer, observe, Instance, Type, default, Enum, Union,
     Any, validate
 )
 
 from prompt_toolkit.enums import DEFAULT_BUFFER, EditingMode
 from prompt_toolkit.filters import (HasFocus, Condition, IsDone)
 from prompt_toolkit.formatted_text import PygmentsTokens
 from prompt_toolkit.history import InMemoryHistory
 from prompt_toolkit.layout.processors import ConditionalProcessor, HighlightMatchingBracketProcessor
 from prompt_toolkit.output import ColorDepth
 from prompt_toolkit.patch_stdout import patch_stdout
 from prompt_toolkit.shortcuts import PromptSession, CompleteStyle, print_formatted_text
 from prompt_toolkit.styles import DynamicStyle, merge_styles
 from prompt_toolkit.styles.pygments import style_from_pygments_cls, style_from_pygments_dict
+from prompt_toolkit import __version__ as ptk_version
 
 from pygments.styles import get_style_by_name
 from pygments.style import Style
 from pygments.token import Token
 
 from .debugger import TerminalPdb, Pdb
 from .magics import TerminalMagics
 from .pt_inputhooks import get_inputhook_name_and_func
 from .prompts import Prompts, ClassicPrompts, RichPromptDisplayHook
 from .ptutils import IPythonPTCompleter, IPythonPTLexer
 from .shortcuts import create_ipython_shortcuts
 
 DISPLAY_BANNER_DEPRECATED = object()
+PTK3 = ptk_version.startswith('3.')
 
 
 class _NoStyle(Style): pass
 
 
 
 _style_overrides_light_bg = {
             Token.Prompt: '#0000ff',
             Token.PromptNum: '#0000ee bold',
             Token.OutPrompt: '#cc0000',
             Token.OutPromptNum: '#bb0000 bold',
 }
 
 _style_overrides_linux = {
             Token.Prompt: '#00cc00',
             Token.PromptNum: '#00bb00 bold',
             Token.OutPrompt: '#cc0000',
             Token.OutPromptNum: '#bb0000 bold',
 }
 
 def get_default_editor():
     try:
         return os.environ['EDITOR']
     except KeyError:
         pass
     except UnicodeError:
         warn("$EDITOR environment variable is not pure ASCII. Using platform "
              "default editor.")
 
     if os.name == 'posix':
         return 'vi'  # the only one guaranteed to be there!
     else:
         return 'notepad' # same in Windows!
 
 # conservatively check for tty
 # overridden streams can result in things like:
 # - sys.stdin = None
 # - no isatty method
 for _name in ('stdin', 'stdout', 'stderr'):
     _stream = getattr(sys, _name)
     if not _stream or not hasattr(_stream, 'isatty') or not _stream.isatty():
         _is_tty = False
         break
 else:
     _is_tty = True
 
 
 _use_simple_prompt = ('IPY_TEST_SIMPLE_PROMPT' in os.environ) or (not _is_tty)
 
+def black_reformat_handler(text_before_cursor):
+    import black
+    formatted_text = black.format_str(text_before_cursor, mode=black.FileMode())
+    if not text_before_cursor.endswith('\n') and formatted_text.endswith('\n'):
+       formatted_text = formatted_text[:-1]
+    return formatted_text
+
+
 class TerminalInteractiveShell(InteractiveShell):
+    mime_renderers = Dict().tag(config=True)
+
     space_for_menu = Integer(6, help='Number of line at the bottom of the screen '
                                      'to reserve for the completion menu'
                             ).tag(config=True)
 
     pt_app = None
     debugger_history = None
 
     simple_prompt = Bool(_use_simple_prompt,
         help="""Use `raw_input` for the REPL, without completion and prompt colors.
 
             Useful when controlling IPython as a subprocess, and piping STDIN/OUT/ERR. Known usage are:
             IPython own testing machinery, and emacs inferior-shell integration through elpy.
 
             This mode default to `True` if the `IPY_TEST_SIMPLE_PROMPT`
             environment variable is set, or the current terminal is not a tty."""
             ).tag(config=True)
 
     @property
     def debugger_cls(self):
         return Pdb if self.simple_prompt else TerminalPdb
 
     confirm_exit = Bool(True,
         help="""
         Set to confirm when you try to exit IPython with an EOF (Control-D
         in Unix, Control-Z/Enter in Windows). By typing 'exit' or 'quit',
         you can force a direct exit without any confirmation.""",
     ).tag(config=True)
 
     editing_mode = Unicode('emacs',
         help="Shortcut style to use at the prompt. 'vi' or 'emacs'.",
     ).tag(config=True)
 
+    autoformatter = Unicode(None,
+        help="Autoformatter to reformat Terminal code. Can be `'black'` or `None`",
+        allow_none=True
+    ).tag(config=True)
+
     mouse_support = Bool(False,
         help="Enable mouse support in the prompt\n(Note: prevents selecting text with the mouse)"
     ).tag(config=True)
 
     # We don't load the list of styles for the help string, because loading
     # Pygments plugins takes time and can cause unexpected errors.
     highlighting_style = Union([Unicode('legacy'), Type(klass=Style)],
         help="""The name or class of a Pygments style to use for syntax
         highlighting. To see available styles, run `pygmentize -L styles`."""
     ).tag(config=True)
 
     @validate('editing_mode')
     def _validate_editing_mode(self, proposal):
         if proposal['value'].lower() == 'vim':
             proposal['value']= 'vi'
         elif proposal['value'].lower() == 'default':
             proposal['value']= 'emacs'
 
         if hasattr(EditingMode, proposal['value'].upper()):
             return proposal['value'].lower()
 
         return self.editing_mode
 
 
     @observe('editing_mode')
     def _editing_mode(self, change):
         u_mode = change.new.upper()
         if self.pt_app:
             self.pt_app.editing_mode = u_mode
 
+    @observe('autoformatter')
+    def _autoformatter_changed(self, change):
+        formatter = change.new
+        if formatter is None:
+            self.reformat_handler = lambda x:x
+        elif formatter == 'black':
+            self.reformat_handler = black_reformat_handler
+        else:
+            raise ValueError
+
     @observe('highlighting_style')
     @observe('colors')
     def _highlighting_style_changed(self, change):
         self.refresh_style()
 
     def refresh_style(self):
         self._style = self._make_style_from_name_or_cls(self.highlighting_style)
 
 
     highlighting_style_overrides = Dict(
         help="Override highlighting format for specific tokens"
     ).tag(config=True)
 
     true_color = Bool(False,
         help=("Use 24bit colors instead of 256 colors in prompt highlighting. "
               "If your terminal supports true color, the following command "
               "should print 'TRUECOLOR' in orange: "
               "printf \"\\x1b[38;2;255;100;0mTRUECOLOR\\x1b[0m\\n\"")
     ).tag(config=True)
 
     editor = Unicode(get_default_editor(),
         help="Set the editor used by IPython (default to $EDITOR/vi/notepad)."
     ).tag(config=True)
 
     prompts_class = Type(Prompts, help='Class used to generate Prompt token for prompt_toolkit').tag(config=True)
 
     prompts = Instance(Prompts)
 
     @default('prompts')
     def _prompts_default(self):
         return self.prompts_class(self)
 
 #    @observe('prompts')
 #    def _(self, change):
 #        self._update_layout()
 
     @default('displayhook_class')
     def _displayhook_class_default(self):
         return RichPromptDisplayHook
 
     term_title = Bool(True,
         help="Automatically set the terminal title"
     ).tag(config=True)
 
     term_title_format = Unicode("IPython: {cwd}",
         help="Customize the terminal title format.  This is a python format string. " +
              "Available substitutions are: {cwd}."
     ).tag(config=True)
 
     display_completions = Enum(('column', 'multicolumn','readlinelike'),
         help= ( "Options for displaying tab completions, 'column', 'multicolumn', and "
                 "'readlinelike'. These options are for `prompt_toolkit`, see "
                 "`prompt_toolkit` documentation for more information."
                 ),
         default_value='multicolumn').tag(config=True)
 
     highlight_matching_brackets = Bool(True,
         help="Highlight matching brackets.",
     ).tag(config=True)
 
     extra_open_editor_shortcuts = Bool(False,
         help="Enable vi (v) or Emacs (C-X C-E) shortcuts to open an external editor. "
              "This is in addition to the F2 binding, which is always enabled."
     ).tag(config=True)
 
     handle_return = Any(None,
         help="Provide an alternative handler to be called when the user presses "
              "Return. This is an advanced option intended for debugging, which "
              "may be changed or removed in later releases."
     ).tag(config=True)
 
     enable_history_search = Bool(True,
         help="Allows to enable/disable the prompt toolkit history search"
     ).tag(config=True)
 
     prompt_includes_vi_mode = Bool(True,
         help="Display the current vi mode (when using vi editing mode)."
     ).tag(config=True)
 
     @observe('term_title')
     def init_term_title(self, change=None):
         # Enable or disable the terminal title.
         if self.term_title:
             toggle_set_term_title(True)
             set_term_title(self.term_title_format.format(cwd=abbrev_cwd()))
         else:
             toggle_set_term_title(False)
 
     def restore_term_title(self):
         if self.term_title:
             restore_term_title()
 
     def init_display_formatter(self):
         super(TerminalInteractiveShell, self).init_display_formatter()
         # terminal only supports plain text
         self.display_formatter.active_types = ['text/plain']
         # disable `_ipython_display_`
         self.display_formatter.ipython_display_formatter.enabled = False
 
     def init_prompt_toolkit_cli(self):
         if self.simple_prompt:
             # Fall back to plain non-interactive output for tests.
             # This is very limited.
             def prompt():
                 prompt_text = "".join(x[1] for x in self.prompts.in_prompt_tokens())
                 lines = [input(prompt_text)]
                 prompt_continuation = "".join(x[1] for x in self.prompts.continuation_prompt_tokens())
                 while self.check_complete('\n'.join(lines))[0] == 'incomplete':
                     lines.append( input(prompt_continuation) )
                 return '\n'.join(lines)
             self.prompt_for_code = prompt
             return
 
         # Set up keyboard shortcuts
         key_bindings = create_ipython_shortcuts(self)
 
         # Pre-populate history from IPython's history database
         history = InMemoryHistory()
         last_cell = u""
         for __, ___, cell in self.history_manager.get_tail(self.history_load_length,
                                                         include_latest=True):
             # Ignore blank lines and consecutive duplicates
             cell = cell.rstrip()
             if cell and (cell != last_cell):
                 history.append_string(cell)
                 last_cell = cell
 
         self._style = self._make_style_from_name_or_cls(self.highlighting_style)
         self.style = DynamicStyle(lambda: self._style)
 
         editing_mode = getattr(EditingMode, self.editing_mode.upper())
 
+        self.pt_loop = asyncio.new_event_loop()
         self.pt_app = PromptSession(
                             editing_mode=editing_mode,
                             key_bindings=key_bindings,
                             history=history,
                             completer=IPythonPTCompleter(shell=self),
                             enable_history_search = self.enable_history_search,
                             style=self.style,
                             include_default_pygments_style=False,
                             mouse_support=self.mouse_support,
                             enable_open_in_editor=self.extra_open_editor_shortcuts,
                             color_depth=self.color_depth,
                             **self._extra_prompt_options())
 
     def _make_style_from_name_or_cls(self, name_or_cls):
         """
         Small wrapper that make an IPython compatible style from a style name
 
         We need that to add style for prompt ... etc.
         """
         style_overrides = {}
         if name_or_cls == 'legacy':
             legacy = self.colors.lower()
             if legacy == 'linux':
                 style_cls = get_style_by_name('monokai')
                 style_overrides = _style_overrides_linux
             elif legacy == 'lightbg':
                 style_overrides = _style_overrides_light_bg
                 style_cls = get_style_by_name('pastie')
             elif legacy == 'neutral':
                 # The default theme needs to be visible on both a dark background
                 # and a light background, because we can't tell what the terminal
                 # looks like. These tweaks to the default theme help with that.
                 style_cls = get_style_by_name('default')
                 style_overrides.update({
                     Token.Number: '#007700',
                     Token.Operator: 'noinherit',
                     Token.String: '#BB6622',
                     Token.Name.Function: '#2080D0',
                     Token.Name.Class: 'bold #2080D0',
                     Token.Name.Namespace: 'bold #2080D0',
                     Token.Prompt: '#009900',
                     Token.PromptNum: '#ansibrightgreen bold',
                     Token.OutPrompt: '#990000',
                     Token.OutPromptNum: '#ansibrightred bold',
                 })
 
                 # Hack: Due to limited color support on the Windows console
                 # the prompt colors will be wrong without this
                 if os.name == 'nt':
                     style_overrides.update({
                         Token.Prompt: '#ansidarkgreen',
                         Token.PromptNum: '#ansigreen bold',
                         Token.OutPrompt: '#ansidarkred',
                         Token.OutPromptNum: '#ansired bold',
                     })
             elif legacy =='nocolor':
                 style_cls=_NoStyle
                 style_overrides = {}
             else :
                 raise ValueError('Got unknown colors: ', legacy)
         else :
             if isinstance(name_or_cls, str):
                 style_cls = get_style_by_name(name_or_cls)
             else:
                 style_cls = name_or_cls
             style_overrides = {
                 Token.Prompt: '#009900',
                 Token.PromptNum: '#ansibrightgreen bold',
                 Token.OutPrompt: '#990000',
                 Token.OutPromptNum: '#ansibrightred bold',
             }
         style_overrides.update(self.highlighting_style_overrides)
         style = merge_styles([
             style_from_pygments_cls(style_cls),
             style_from_pygments_dict(style_overrides),
         ])
 
         return style
 
     @property
     def pt_complete_style(self):
         return {
             'multicolumn': CompleteStyle.MULTI_COLUMN,
             'column': CompleteStyle.COLUMN,
             'readlinelike': CompleteStyle.READLINE_LIKE,
         }[self.display_completions]
 
     @property
     def color_depth(self):
         return (ColorDepth.TRUE_COLOR if self.true_color else None)
 
     def _extra_prompt_options(self):
         """
         Return the current layout option for the current Terminal InteractiveShell
         """
         def get_message():
             return PygmentsTokens(self.prompts.in_prompt_tokens())
 
         if self.editing_mode == 'emacs':
             # with emacs mode the prompt is (usually) static, so we call only
             # the function once. With VI mode it can toggle between [ins] and
             # [nor] so we can't precompute.
             # here I'm going to favor the default keybinding which almost
             # everybody uses to decrease CPU usage.
             # if we have issues with users with custom Prompts we can see how to
             # work around this.
             get_message = get_message()
 
-        return {
+        options = {
                 'complete_in_thread': False,
                 'lexer':IPythonPTLexer(),
                 'reserve_space_for_menu':self.space_for_menu,
                 'message': get_message,
                 'prompt_continuation': (
                     lambda width, lineno, is_soft_wrap:
                         PygmentsTokens(self.prompts.continuation_prompt_tokens(width))),
                 'multiline': True,
                 'complete_style': self.pt_complete_style,
 
                 # Highlight matching brackets, but only when this setting is
                 # enabled, and only when the DEFAULT_BUFFER has the focus.
                 'input_processors': [ConditionalProcessor(
                         processor=HighlightMatchingBracketProcessor(chars='[](){}'),
                         filter=HasFocus(DEFAULT_BUFFER) & ~IsDone() &
                             Condition(lambda: self.highlight_matching_brackets))],
-                'inputhook': self.inputhook,
                 }
+        if not PTK3:
+            options['inputhook'] = self.inputhook
+
+        return options
 
     def prompt_for_code(self):
         if self.rl_next_input:
             default = self.rl_next_input
             self.rl_next_input = None
         else:
             default = ''
 
+        # In order to make sure that asyncio code written in the
+        # interactive shell doesn't interfere with the prompt, we run the
+        # prompt in a different event loop.
+        # If we don't do this, people could spawn coroutine with a
+        # while/true inside which will freeze the prompt.
+
+        try:
+            old_loop = asyncio.get_event_loop()
+        except RuntimeError:
+            # This happens when the user used `asyncio.run()`.
+            old_loop = None
+
+        asyncio.set_event_loop(self.pt_loop)
+        try:
         with patch_stdout(raw=True):
             text = self.pt_app.prompt(
                 default=default,
-#                pre_run=self.pre_prompt,# reset_current_buffer=True,
                 **self._extra_prompt_options())
+        finally:
+            # Restore the original event loop.
+            asyncio.set_event_loop(old_loop)
+
         return text
 
     def enable_win_unicode_console(self):
         # Since IPython 7.10 doesn't support python < 3.6 and PEP 528, Python uses the unicode APIs for the Windows
         # console by default, so WUC shouldn't be needed.
         from warnings import warn
         warn("`enable_win_unicode_console` is deprecated since IPython 7.10, does not do anything and will be removed in the future",
              DeprecationWarning,
              stacklevel=2)
 
     def init_io(self):
         if sys.platform not in {'win32', 'cli'}:
             return
 
         import colorama
         colorama.init()
 
         # For some reason we make these wrappers around stdout/stderr.
         # For now, we need to reset them so all output gets coloured.
         # https://github.com/ipython/ipython/issues/8669
         # io.std* are deprecated, but don't show our own deprecation warnings
         # during initialization of the deprecated API.
         with warnings.catch_warnings():
             warnings.simplefilter('ignore', DeprecationWarning)
             io.stdout = io.IOStream(sys.stdout)
             io.stderr = io.IOStream(sys.stderr)
 
     def init_magics(self):
         super(TerminalInteractiveShell, self).init_magics()
         self.register_magics(TerminalMagics)
 
     def init_alias(self):
         # The parent class defines aliases that can be safely used with any
         # frontend.
         super(TerminalInteractiveShell, self).init_alias()
 
         # Now define aliases that only make sense on the terminal, because they
         # need direct access to the console in a way that we can't emulate in
         # GUI or web frontend
         if os.name == 'posix':
             for cmd in ('clear', 'more', 'less', 'man'):
                 self.alias_manager.soft_define_alias(cmd, cmd)
 
 
     def __init__(self, *args, **kwargs):
         super(TerminalInteractiveShell, self).__init__(*args, **kwargs)
         self.init_prompt_toolkit_cli()
         self.init_term_title()
         self.keep_running = True
 
         self.debugger_history = InMemoryHistory()
 
     def ask_exit(self):
         self.keep_running = False
 
     rl_next_input = None
 
     def interact(self, display_banner=DISPLAY_BANNER_DEPRECATED):
 
         if display_banner is not DISPLAY_BANNER_DEPRECATED:
             warn('interact `display_banner` argument is deprecated since IPython 5.0. Call `show_banner()` if needed.', DeprecationWarning, stacklevel=2)
 
         self.keep_running = True
         while self.keep_running:
             print(self.separate_in, end='')
 
             try:
                 code = self.prompt_for_code()
             except EOFError:
                 if (not self.confirm_exit) \
                         or self.ask_yes_no('Do you really want to exit ([y]/n)?','y','n'):
                     self.ask_exit()
 
             else:
                 if code:
                     self.run_cell(code, store_history=True)
 
     def mainloop(self, display_banner=DISPLAY_BANNER_DEPRECATED):
         # An extra layer of protection in case someone mashing Ctrl-C breaks
         # out of our internal code.
         if display_banner is not DISPLAY_BANNER_DEPRECATED:
             warn('mainloop `display_banner` argument is deprecated since IPython 5.0. Call `show_banner()` if needed.', DeprecationWarning, stacklevel=2)
         while True:
             try:
                 self.interact()
                 break
             except KeyboardInterrupt as e:
                 print("\n%s escaped interact()\n" % type(e).__name__)
             finally:
                 # An interrupt during the eventloop will mess up the
                 # internal state of the prompt_toolkit library.
                 # Stopping the eventloop fixes this, see
                 # https://github.com/ipython/ipython/pull/9867
                 if hasattr(self, '_eventloop'):
                     self._eventloop.stop()
 
                 self.restore_term_title()
 
 
     _inputhook = None
     def inputhook(self, context):
         if self._inputhook is not None:
             self._inputhook(context)
 
     active_eventloop = None
     def enable_gui(self, gui=None):
-        if gui:
+        if gui and (gui != 'inline') :
             self.active_eventloop, self._inputhook =\
                 get_inputhook_name_and_func(gui)
         else:
             self.active_eventloop = self._inputhook = None
 
+        # For prompt_toolkit 3.0. We have to create an asyncio event loop with
+        # this inputhook.
+        if PTK3:
+            import asyncio
+            from prompt_toolkit.eventloop import new_eventloop_with_inputhook
+
+            if gui == 'asyncio':
+                # When we integrate the asyncio event loop, run the UI in the
+                # same event loop as the rest of the code. don't use an actual
+                # input hook. (Asyncio is not made for nesting event loops.)
+                self.pt_loop = asyncio.get_event_loop()
+
+            elif self._inputhook:
+                # If an inputhook was set, create a new asyncio event loop with
+                # this inputhook for the prompt.
+                self.pt_loop = new_eventloop_with_inputhook(self._inputhook)
+            else:
+                # When there's no inputhook, run the prompt in a separate
+                # asyncio event loop.
+                self.pt_loop = asyncio.new_event_loop()
+
     # Run !system commands directly, not through pipes, so terminal programs
     # work correctly.
     system = InteractiveShell.system_raw
 
     def auto_rewrite_input(self, cmd):
         """Overridden from the parent class to use fancy rewriting prompt"""
         if not self.show_rewritten_input:
             return
 
         tokens = self.prompts.rewrite_prompt_tokens()
         if self.pt_app:
             print_formatted_text(PygmentsTokens(tokens), end='',
                                  style=self.pt_app.app.style)
             print(cmd)
         else:
             prompt = ''.join(s for t, s in tokens)
             print(prompt, cmd, sep='')
 
     _prompts_before = None
     def switch_doctest_mode(self, mode):
         """Switch prompts to classic for %doctest_mode"""
         if mode:
             self._prompts_before = self.prompts
             self.prompts = ClassicPrompts(self)
         elif self._prompts_before:
             self.prompts = self._prompts_before
             self._prompts_before = None
 #        self._update_layout()
 
 
 InteractiveShellABC.register(TerminalInteractiveShell)
 
 if __name__ == '__main__':
     TerminalInteractiveShell.instance().interact()

IPython/terminal/prompts.py

@@ -1,91 +1,102 b''
 """Terminal input and output prompts."""
 
 from pygments.token import Token
 import sys
 
 from IPython.core.displayhook import DisplayHook
 
 from prompt_toolkit.formatted_text import fragment_list_width, PygmentsTokens
 from prompt_toolkit.shortcuts import print_formatted_text
 
 
 class Prompts(object):
     def __init__(self, shell):
         self.shell = shell
 
     def vi_mode(self):
         if (getattr(self.shell.pt_app, 'editing_mode', None) == 'VI'
                 and self.shell.prompt_includes_vi_mode):
             return '['+str(self.shell.pt_app.app.vi_state.input_mode)[3:6]+'] '
         return ''
 
 
     def in_prompt_tokens(self):
         return [
             (Token.Prompt, self.vi_mode() ),
             (Token.Prompt, 'In ['),
             (Token.PromptNum, str(self.shell.execution_count)),
             (Token.Prompt, ']: '),
         ]
 
     def _width(self):
         return fragment_list_width(self.in_prompt_tokens())
 
     def continuation_prompt_tokens(self, width=None):
         if width is None:
             width = self._width()
         return [
             (Token.Prompt, (' ' * (width - 5)) + '...: '),
         ]
 
     def rewrite_prompt_tokens(self):
         width = self._width()
         return [
             (Token.Prompt, ('-' * (width - 2)) + '> '),
         ]
 
     def out_prompt_tokens(self):
         return [
             (Token.OutPrompt, 'Out['),
             (Token.OutPromptNum, str(self.shell.execution_count)),
             (Token.OutPrompt, ']: '),
         ]
 
 class ClassicPrompts(Prompts):
     def in_prompt_tokens(self):
         return [
             (Token.Prompt, '>>> '),
         ]
 
     def continuation_prompt_tokens(self, width=None):
         return [
             (Token.Prompt, '... ')
         ]
 
     def rewrite_prompt_tokens(self):
         return []
 
     def out_prompt_tokens(self):
         return []
 
 class RichPromptDisplayHook(DisplayHook):
     """Subclass of base display hook using coloured prompt"""
     def write_output_prompt(self):
         sys.stdout.write(self.shell.separate_out)
         # If we're not displaying a prompt, it effectively ends with a newline,
         # because the output will be left-aligned.
         self.prompt_end_newline = True
 
         if self.do_full_cache:
             tokens = self.shell.prompts.out_prompt_tokens()
             prompt_txt = ''.join(s for t, s in tokens)
             if prompt_txt and not prompt_txt.endswith('\n'):
                 # Ask for a newline before multiline output
                 self.prompt_end_newline = False
 
             if self.shell.pt_app:
                 print_formatted_text(PygmentsTokens(tokens),
                     style=self.shell.pt_app.app.style, end='',
                 )
             else:
                 sys.stdout.write(prompt_txt)
+
+    def write_format_data(self, format_dict, md_dict=None) -> None:
+        if self.shell.mime_renderers:
+
+            for mime, handler in self.shell.mime_renderers.items():
+                if mime in format_dict:
+                    handler(format_dict[mime], None)
+                    return
+                
+        super().write_format_data(format_dict, md_dict)
+

IPython/terminal/pt_inputhooks/__init__.py

@@ -1,49 +1,50 b''
 import importlib
 import os
 
 aliases = {
     'qt4': 'qt',
     'gtk2': 'gtk',
 }
 
 backends = [
     'qt', 'qt4', 'qt5',
     'gtk', 'gtk2', 'gtk3',
     'tk',
     'wx',
     'pyglet', 'glut',
     'osx',
+    'asyncio'
 ]
 
 registered = {}
 
 def register(name, inputhook):
     """Register the function *inputhook* as an event loop integration."""
     registered[name] = inputhook
 
 class UnknownBackend(KeyError):
     def __init__(self, name):
         self.name = name
 
     def __str__(self):
         return ("No event loop integration for {!r}. "
                 "Supported event loops are: {}").format(self.name,
                                     ', '.join(backends + sorted(registered)))
 
 def get_inputhook_name_and_func(gui):
     if gui in registered:
         return gui, registered[gui]
 
     if gui not in backends:
         raise UnknownBackend(gui)
 
     if gui in aliases:
         return get_inputhook_name_and_func(aliases[gui])
 
     gui_mod = gui
     if gui == 'qt5':
         os.environ['QT_API'] = 'pyqt5'
         gui_mod = 'qt'
 
     mod = importlib.import_module('IPython.terminal.pt_inputhooks.'+gui_mod)
     return gui, mod.inputhook

IPython/terminal/shortcuts.py

@@ -1,249 +1,274 b''
 """
 Module to define and register Terminal IPython shortcuts with
 :mod:`prompt_toolkit`
 """
 
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
 import warnings
 import signal
 import sys
 from typing import Callable
 
 
 from prompt_toolkit.application.current import get_app
 from prompt_toolkit.enums import DEFAULT_BUFFER, SEARCH_BUFFER
 from prompt_toolkit.filters import (has_focus, has_selection, Condition,
     vi_insert_mode, emacs_insert_mode, has_completions, vi_mode)
 from prompt_toolkit.key_binding.bindings.completion import display_completions_like_readline
 from prompt_toolkit.key_binding import KeyBindings
 
 from IPython.utils.decorators import undoc
 
 @undoc
 @Condition
 def cursor_in_leading_ws():
     before = get_app().current_buffer.document.current_line_before_cursor
     return (not before) or before.isspace()
 
 
 def create_ipython_shortcuts(shell):
     """Set up the prompt_toolkit keyboard shortcuts for IPython"""
 
     kb = KeyBindings()
     insert_mode = vi_insert_mode | emacs_insert_mode
 
     if getattr(shell, 'handle_return', None):
         return_handler = shell.handle_return(shell)
     else:
         return_handler = newline_or_execute_outer(shell)
 
     kb.add('enter', filter=(has_focus(DEFAULT_BUFFER)
                             & ~has_selection
                             & insert_mode
                         ))(return_handler)
 
+    def reformat_and_execute(event):
+        reformat_text_before_cursor(event.current_buffer, event.current_buffer.document, shell)
+        event.current_buffer.validate_and_handle()
+
+    kb.add('escape', 'enter', filter=(has_focus(DEFAULT_BUFFER)
+                            & ~has_selection
+                            & insert_mode
+                                      ))(reformat_and_execute)
+
     kb.add('c-\\')(force_exit)
 
     kb.add('c-p', filter=(vi_insert_mode & has_focus(DEFAULT_BUFFER))
                 )(previous_history_or_previous_completion)
 
     kb.add('c-n', filter=(vi_insert_mode & has_focus(DEFAULT_BUFFER))
                 )(next_history_or_next_completion)
 
     kb.add('c-g', filter=(has_focus(DEFAULT_BUFFER) & has_completions)
                 )(dismiss_completion)
 
     kb.add('c-c', filter=has_focus(DEFAULT_BUFFER))(reset_buffer)
 
     kb.add('c-c', filter=has_focus(SEARCH_BUFFER))(reset_search_buffer)
 
     supports_suspend = Condition(lambda: hasattr(signal, 'SIGTSTP'))
     kb.add('c-z', filter=supports_suspend)(suspend_to_bg)
 
     # Ctrl+I == Tab
     kb.add('tab', filter=(has_focus(DEFAULT_BUFFER)
                           & ~has_selection
                           & insert_mode
                           & cursor_in_leading_ws
                         ))(indent_buffer)
     kb.add('c-o', filter=(has_focus(DEFAULT_BUFFER) & emacs_insert_mode)
            )(newline_autoindent_outer(shell.input_transformer_manager))
 
     kb.add('f2', filter=has_focus(DEFAULT_BUFFER))(open_input_in_editor)
 
     if shell.display_completions == 'readlinelike':
         kb.add('c-i', filter=(has_focus(DEFAULT_BUFFER)
                               & ~has_selection
                               & insert_mode
                               & ~cursor_in_leading_ws
                         ))(display_completions_like_readline)
 
     if sys.platform == 'win32':
         kb.add('c-v', filter=(has_focus(DEFAULT_BUFFER) & ~vi_mode))(win_paste)
 
     return kb
 
 
+def reformat_text_before_cursor(buffer, document, shell):
+    text = buffer.delete_before_cursor(len(document.text[:document.cursor_position]))
+    try:
+        formatted_text = shell.reformat_handler(text)
+        buffer.insert_text(formatted_text)
+    except Exception as e:
+        buffer.insert_text(text)
+
+
 def newline_or_execute_outer(shell):
+
     def newline_or_execute(event):
         """When the user presses return, insert a newline or execute the code."""
         b = event.current_buffer
         d = b.document
 
         if b.complete_state:
             cc = b.complete_state.current_completion
             if cc:
                 b.apply_completion(cc)
             else:
                 b.cancel_completion()
             return
 
         # If there's only one line, treat it as if the cursor is at the end.
         # See https://github.com/ipython/ipython/issues/10425
         if d.line_count == 1:
             check_text = d.text
         else:
             check_text = d.text[:d.cursor_position]
         status, indent = shell.check_complete(check_text)
 
+        # if all we have after the cursor is whitespace: reformat current text
+        # before cursor
+        after_cursor = d.text[d.cursor_position:]
+        if not after_cursor.strip():
+            reformat_text_before_cursor(b, d, shell)
         if not (d.on_last_line or
                 d.cursor_position_row >= d.line_count - d.empty_line_count_at_the_end()
                 ):
             if shell.autoindent:
                 b.insert_text('\n' + indent)
             else:
                 b.insert_text('\n')
             return
 
         if (status != 'incomplete') and b.accept_handler:
+            reformat_text_before_cursor(b, d, shell)
             b.validate_and_handle()
         else:
             if shell.autoindent:
                 b.insert_text('\n' + indent)
             else:
                 b.insert_text('\n')
     return newline_or_execute
 
 
 def previous_history_or_previous_completion(event):
     """
     Control-P in vi edit mode on readline is history next, unlike default prompt toolkit.
 
     If completer is open this still select previous completion.
     """
     event.current_buffer.auto_up()
 
 
 def next_history_or_next_completion(event):
     """
     Control-N in vi edit mode on readline is history previous, unlike default prompt toolkit.
 
     If completer is open this still select next completion.
     """
     event.current_buffer.auto_down()
 
 
 def dismiss_completion(event):
     b = event.current_buffer
     if b.complete_state:
         b.cancel_completion()
 
 
 def reset_buffer(event):
     b = event.current_buffer
     if b.complete_state:
         b.cancel_completion()
     else:
         b.reset()
 
 
 def reset_search_buffer(event):
     if event.current_buffer.document.text:
         event.current_buffer.reset()
     else:
         event.app.layout.focus(DEFAULT_BUFFER)
 
 def suspend_to_bg(event):
     event.app.suspend_to_background()
 
 def force_exit(event):
     """
     Force exit (with a non-zero return value)
     """
     sys.exit("Quit")
 
 def indent_buffer(event):
     event.current_buffer.insert_text(' ' * 4)
 
 @undoc
 def newline_with_copy_margin(event):
     """
     DEPRECATED since IPython 6.0
 
     See :any:`newline_autoindent_outer` for a replacement.
 
     Preserve margin and cursor position when using
     Control-O to insert a newline in EMACS mode
     """
     warnings.warn("`newline_with_copy_margin(event)` is deprecated since IPython 6.0. "
       "see `newline_autoindent_outer(shell)(event)` for a replacement.",
                   DeprecationWarning, stacklevel=2)
 
     b = event.current_buffer
     cursor_start_pos = b.document.cursor_position_col
     b.newline(copy_margin=True)
     b.cursor_up(count=1)
     cursor_end_pos = b.document.cursor_position_col
     if cursor_start_pos != cursor_end_pos:
         pos_diff = cursor_start_pos - cursor_end_pos
         b.cursor_right(count=pos_diff)
 
 def newline_autoindent_outer(inputsplitter) -> Callable[..., None]:
     """
     Return a function suitable for inserting a indented newline after the cursor.
 
     Fancier version of deprecated ``newline_with_copy_margin`` which should
     compute the correct indentation of the inserted line. That is to say, indent
     by 4 extra space after a function definition, class definition, context
     manager... And dedent by 4 space after ``pass``, ``return``, ``raise ...``.
     """
 
     def newline_autoindent(event):
         """insert a newline after the cursor indented appropriately."""
         b = event.current_buffer
         d = b.document
 
         if b.complete_state:
             b.cancel_completion()
         text = d.text[:d.cursor_position] + '\n'
         _, indent = inputsplitter.check_complete(text)
         b.insert_text('\n' + (' ' * (indent or 0)), move_cursor=False)
 
     return newline_autoindent
 
 
 def open_input_in_editor(event):
     event.app.current_buffer.tempfile_suffix = ".py"
     event.app.current_buffer.open_in_editor()
 
 
 if sys.platform == 'win32':
     from IPython.core.error import TryNext
     from IPython.lib.clipboard import (ClipboardEmpty,
                                        win32_clipboard_get,
                                        tkinter_clipboard_get)
 
     @undoc
     def win_paste(event):
         try:
             text = win32_clipboard_get()
         except TryNext:
             try:
                 text = tkinter_clipboard_get()
             except (TryNext, ClipboardEmpty):
                 return
         except ClipboardEmpty:
             return
         event.current_buffer.insert_text(text.replace('\t', ' ' * 4))

IPython/utils/path.py

@@ -1,439 +1,436 b''
 # encoding: utf-8
 """
 Utilities for path handling.
 """
 
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
 import os
 import sys
 import errno
 import shutil
 import random
 import glob
 from warnings import warn
 
 from IPython.utils.process import system
 from IPython.utils import py3compat
 from IPython.utils.decorators import undoc
 
 #-----------------------------------------------------------------------------
 # Code
 #-----------------------------------------------------------------------------
 
 fs_encoding = sys.getfilesystemencoding()
 
 def _writable_dir(path):
     """Whether `path` is a directory, to which the user has write access."""
     return os.path.isdir(path) and os.access(path, os.W_OK)
 
 if sys.platform == 'win32':
     def _get_long_path_name(path):
         """Get a long path name (expand ~) on Windows using ctypes.
 
         Examples
         --------
 
         >>> get_long_path_name('c:\\docume~1')
         'c:\\\\Documents and Settings'
 
         """
         try:
             import ctypes
         except ImportError:
             raise ImportError('you need to have ctypes installed for this to work')
         _GetLongPathName = ctypes.windll.kernel32.GetLongPathNameW
         _GetLongPathName.argtypes = [ctypes.c_wchar_p, ctypes.c_wchar_p,
             ctypes.c_uint ]
 
         buf = ctypes.create_unicode_buffer(260)
         rv = _GetLongPathName(path, buf, 260)
         if rv == 0 or rv > 260:
             return path
         else:
             return buf.value
 else:
     def _get_long_path_name(path):
         """Dummy no-op."""
         return path
 
 
 
 def get_long_path_name(path):
     """Expand a path into its long form.
 
     On Windows this expands any ~ in the paths. On other platforms, it is
     a null operation.
     """
     return _get_long_path_name(path)
 
 
 def unquote_filename(name, win32=(sys.platform=='win32')):
     """ On Windows, remove leading and trailing quotes from filenames.
 
     This function has been deprecated and should not be used any more:
     unquoting is now taken care of by :func:`IPython.utils.process.arg_split`.
     """
     warn("'unquote_filename' is deprecated since IPython 5.0 and should not "
          "be used anymore", DeprecationWarning, stacklevel=2)
     if win32:
         if name.startswith(("'", '"')) and name.endswith(("'", '"')):
             name = name[1:-1]
     return name
 
 
 def compress_user(path):
     """Reverse of :func:`os.path.expanduser`
     """
     home = os.path.expanduser('~')
     if path.startswith(home):
         path =  "~" + path[len(home):]
     return path
 
 def get_py_filename(name, force_win32=None):
     """Return a valid python filename in the current directory.
 
     If the given name is not a file, it adds '.py' and searches again.
     Raises IOError with an informative message if the file isn't found.
     """
 
     name = os.path.expanduser(name)
     if force_win32 is not None:
         warn("The 'force_win32' argument to 'get_py_filename' is deprecated "
              "since IPython 5.0 and should not be used anymore",
             DeprecationWarning, stacklevel=2)
     if not os.path.isfile(name) and not name.endswith('.py'):
         name += '.py'
     if os.path.isfile(name):
         return name
     else:
         raise IOError('File `%r` not found.' % name)
 
 
 def filefind(filename, path_dirs=None):
     """Find a file by looking through a sequence of paths.
 
     This iterates through a sequence of paths looking for a file and returns
     the full, absolute path of the first occurrence of the file.  If no set of
     path dirs is given, the filename is tested as is, after running through
     :func:`expandvars` and :func:`expanduser`.  Thus a simple call::
 
         filefind('myfile.txt')
 
     will find the file in the current working dir, but::
 
         filefind('~/myfile.txt')
 
     Will find the file in the users home directory.  This function does not
     automatically try any paths, such as the cwd or the user's home directory.
 
     Parameters
     ----------
     filename : str
         The filename to look for.
     path_dirs : str, None or sequence of str
         The sequence of paths to look for the file in.  If None, the filename
         need to be absolute or be in the cwd.  If a string, the string is
         put into a sequence and the searched.  If a sequence, walk through
         each element and join with ``filename``, calling :func:`expandvars`
         and :func:`expanduser` before testing for existence.
 
     Returns
     -------
     Raises :exc:`IOError` or returns absolute path to file.
     """
 
     # If paths are quoted, abspath gets confused, strip them...
     filename = filename.strip('"').strip("'")
     # If the input is an absolute path, just check it exists
     if os.path.isabs(filename) and os.path.isfile(filename):
         return filename
 
     if path_dirs is None:
         path_dirs = ("",)
     elif isinstance(path_dirs, str):
         path_dirs = (path_dirs,)
 
     for path in path_dirs:
         if path == '.': path = os.getcwd()
         testname = expand_path(os.path.join(path, filename))
         if os.path.isfile(testname):
             return os.path.abspath(testname)
 
     raise IOError("File %r does not exist in any of the search paths: %r" %
                   (filename, path_dirs) )
 
 
 class HomeDirError(Exception):
     pass
 
 
-def get_home_dir(require_writable=False):
+def get_home_dir(require_writable=False) -> str:
     """Return the 'home' directory, as a unicode string.
 
     Uses os.path.expanduser('~'), and checks for writability.
 
     See stdlib docs for how this is determined.
     For Python <3.8, $HOME is first priority on *ALL* platforms.
     For Python >=3.8 on Windows, %HOME% is no longer considered.
 
     Parameters
     ----------
 
     require_writable : bool [default: False]
         if True:
             guarantees the return value is a writable directory, otherwise
             raises HomeDirError
         if False:
             The path is resolved, but it is not guaranteed to exist or be writable.
     """
 
     homedir = os.path.expanduser('~')
     # Next line will make things work even when /home/ is a symlink to
     # /usr/home as it is on FreeBSD, for example
     homedir = os.path.realpath(homedir)
 
     if not _writable_dir(homedir) and os.name == 'nt':
         # expanduser failed, use the registry to get the 'My Documents' folder.
         try:
-            try:
-                import winreg as wreg  # Py 3
-            except ImportError:
-                import _winreg as wreg  # Py 2
-            key = wreg.OpenKey(
+            import winreg as wreg
+            with wreg.OpenKey(
                 wreg.HKEY_CURRENT_USER,
                 r"Software\Microsoft\Windows\CurrentVersion\Explorer\Shell Folders"
-            )
+            ) as key:
             homedir = wreg.QueryValueEx(key,'Personal')[0]
-            key.Close()
         except:
             pass
 
     if (not require_writable) or _writable_dir(homedir):
-        return py3compat.cast_unicode(homedir, fs_encoding)
+        assert isinstance(homedir, str), "Homedir shoudl be unicode not bytes"
+        return homedir
     else:
         raise HomeDirError('%s is not a writable dir, '
                 'set $HOME environment variable to override' % homedir)
 
 def get_xdg_dir():
     """Return the XDG_CONFIG_HOME, if it is defined and exists, else None.
 
     This is only for non-OS X posix (Linux,Unix,etc.) systems.
     """
 
     env = os.environ
 
     if os.name == 'posix' and sys.platform != 'darwin':
         # Linux, Unix, AIX, etc.
         # use ~/.config if empty OR not set
         xdg = env.get("XDG_CONFIG_HOME", None) or os.path.join(get_home_dir(), '.config')
         if xdg and _writable_dir(xdg):
             return py3compat.cast_unicode(xdg, fs_encoding)
 
     return None
 
 
 def get_xdg_cache_dir():
     """Return the XDG_CACHE_HOME, if it is defined and exists, else None.
 
     This is only for non-OS X posix (Linux,Unix,etc.) systems.
     """
 
     env = os.environ
 
     if os.name == 'posix' and sys.platform != 'darwin':
         # Linux, Unix, AIX, etc.
         # use ~/.cache if empty OR not set
         xdg = env.get("XDG_CACHE_HOME", None) or os.path.join(get_home_dir(), '.cache')
         if xdg and _writable_dir(xdg):
             return py3compat.cast_unicode(xdg, fs_encoding)
 
     return None
 
 
 @undoc
 def get_ipython_dir():
-    warn("get_ipython_dir has moved to the IPython.paths module since IPython 4.0.", stacklevel=2)
+    warn("get_ipython_dir has moved to the IPython.paths module since IPython 4.0.", DeprecationWarning, stacklevel=2)
     from IPython.paths import get_ipython_dir
     return get_ipython_dir()
 
 @undoc
 def get_ipython_cache_dir():
-    warn("get_ipython_cache_dir has moved to the IPython.paths module since IPython 4.0.", stacklevel=2)
+    warn("get_ipython_cache_dir has moved to the IPython.paths module since IPython 4.0.", DeprecationWarning, stacklevel=2)
     from IPython.paths import get_ipython_cache_dir
     return get_ipython_cache_dir()
 
 @undoc
 def get_ipython_package_dir():
-    warn("get_ipython_package_dir has moved to the IPython.paths module since IPython 4.0.", stacklevel=2)
+    warn("get_ipython_package_dir has moved to the IPython.paths module since IPython 4.0.", DeprecationWarning, stacklevel=2)
     from IPython.paths import get_ipython_package_dir
     return get_ipython_package_dir()
 
 @undoc
 def get_ipython_module_path(module_str):
-    warn("get_ipython_module_path has moved to the IPython.paths module since IPython 4.0.", stacklevel=2)
+    warn("get_ipython_module_path has moved to the IPython.paths module since IPython 4.0.", DeprecationWarning, stacklevel=2)
     from IPython.paths import get_ipython_module_path
     return get_ipython_module_path(module_str)
 
 @undoc
 def locate_profile(profile='default'):
-    warn("locate_profile has moved to the IPython.paths module since IPython 4.0.", stacklevel=2)
+    warn("locate_profile has moved to the IPython.paths module since IPython 4.0.", DeprecationWarning, stacklevel=2)
     from IPython.paths import locate_profile
     return locate_profile(profile=profile)
 
 def expand_path(s):
     """Expand $VARS and ~names in a string, like a shell
 
     :Examples:
 
        In [2]: os.environ['FOO']='test'
 
        In [3]: expand_path('variable FOO is $FOO')
        Out[3]: 'variable FOO is test'
     """
     # This is a pretty subtle hack. When expand user is given a UNC path
     # on Windows (\\server\share$\%username%), os.path.expandvars, removes
     # the $ to get (\\server\share\%username%). I think it considered $
     # alone an empty var. But, we need the $ to remains there (it indicates
     # a hidden share).
     if os.name=='nt':
         s = s.replace('$\\', 'IPYTHON_TEMP')
     s = os.path.expandvars(os.path.expanduser(s))
     if os.name=='nt':
         s = s.replace('IPYTHON_TEMP', '$\\')
     return s
 
 
 def unescape_glob(string):
     """Unescape glob pattern in `string`."""
     def unescape(s):
         for pattern in '*[]!?':
             s = s.replace(r'\{0}'.format(pattern), pattern)
         return s
     return '\\'.join(map(unescape, string.split('\\\\')))
 
 
 def shellglob(args):
     """
     Do glob expansion for each element in `args` and return a flattened list.
 
     Unmatched glob pattern will remain as-is in the returned list.
 
     """
     expanded = []
     # Do not unescape backslash in Windows as it is interpreted as
     # path separator:
     unescape = unescape_glob if sys.platform != 'win32' else lambda x: x
     for a in args:
         expanded.extend(glob.glob(a) or [unescape(a)])
     return expanded
 
 
 def target_outdated(target,deps):
     """Determine whether a target is out of date.
 
     target_outdated(target,deps) -> 1/0
 
     deps: list of filenames which MUST exist.
     target: single filename which may or may not exist.
 
     If target doesn't exist or is older than any file listed in deps, return
     true, otherwise return false.
     """
     try:
         target_time = os.path.getmtime(target)
     except os.error:
         return 1
     for dep in deps:
         dep_time = os.path.getmtime(dep)
         if dep_time > target_time:
             #print "For target",target,"Dep failed:",dep # dbg
             #print "times (dep,tar):",dep_time,target_time # dbg
             return 1
     return 0
 
 
 def target_update(target,deps,cmd):
     """Update a target with a given command given a list of dependencies.
 
     target_update(target,deps,cmd) -> runs cmd if target is outdated.
 
     This is just a wrapper around target_outdated() which calls the given
     command if target is outdated."""
 
     if target_outdated(target,deps):
         system(cmd)
 
 
 ENOLINK = 1998
 
 def link(src, dst):
     """Hard links ``src`` to ``dst``, returning 0 or errno.
 
     Note that the special errno ``ENOLINK`` will be returned if ``os.link`` isn't
     supported by the operating system.
     """
 
     if not hasattr(os, "link"):
         return ENOLINK
     link_errno = 0
     try:
         os.link(src, dst)
     except OSError as e:
         link_errno = e.errno
     return link_errno
 
 
 def link_or_copy(src, dst):
     """Attempts to hardlink ``src`` to ``dst``, copying if the link fails.
 
     Attempts to maintain the semantics of ``shutil.copy``.
 
     Because ``os.link`` does not overwrite files, a unique temporary file
     will be used if the target already exists, then that file will be moved
     into place.
     """
 
     if os.path.isdir(dst):
         dst = os.path.join(dst, os.path.basename(src))
 
     link_errno = link(src, dst)
     if link_errno == errno.EEXIST:
         if os.stat(src).st_ino == os.stat(dst).st_ino:
             # dst is already a hard link to the correct file, so we don't need
             # to do anything else. If we try to link and rename the file
             # anyway, we get duplicate files - see http://bugs.python.org/issue21876
             return
 
         new_dst = dst + "-temp-%04X" %(random.randint(1, 16**4), )
         try:
             link_or_copy(src, new_dst)
         except:
             try:
                 os.remove(new_dst)
             except OSError:
                 pass
             raise
         os.rename(new_dst, dst)
     elif link_errno != 0:
         # Either link isn't supported, or the filesystem doesn't support
         # linking, or 'src' and 'dst' are on different filesystems.
         shutil.copy(src, dst)
 
 def ensure_dir_exists(path, mode=0o755):
     """ensure that a directory exists
 
     If it doesn't exist, try to create it and protect against a race condition
     if another process is doing the same.
 
     The default permissions are 755, which differ from os.makedirs default of 777.
     """
     if not os.path.exists(path):
         try:
             os.makedirs(path, mode=mode)
         except OSError as e:
             if e.errno != errno.EEXIST:
                 raise
     elif not os.path.isdir(path):
         raise IOError("%r exists but is not a directory" % path)