Remove PY3 variable

IPython/core/magics/execution.py

@@ -1,1380 +1,1374 b''
 # -*- coding: utf-8 -*-
 """Implementation of execution-related magic functions."""
 
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
 
 import ast
 import bdb
 import gc
 import itertools
 import os
 import sys
 import time
 import timeit
 import math
 from pdb import Restart
 
 # cProfile was added in Python2.5
 try:
     import cProfile as profile
     import pstats
 except ImportError:
     # profile isn't bundled by default in Debian for license reasons
     try:
         import profile, pstats
     except ImportError:
         profile = pstats = None
 
 from IPython.core import oinspect
 from IPython.core import magic_arguments
 from IPython.core import page
 from IPython.core.error import UsageError
 from IPython.core.macro import Macro
 from IPython.core.magic import (Magics, magics_class, line_magic, cell_magic,
                                 line_cell_magic, on_off, needs_local_scope)
 from IPython.testing.skipdoctest import skip_doctest
 from IPython.utils import py3compat
-from IPython.utils.py3compat import builtin_mod, PY3
+from IPython.utils.py3compat import builtin_mod
 from IPython.utils.contexts import preserve_keys
 from IPython.utils.capture import capture_output
 from IPython.utils.ipstruct import Struct
 from IPython.utils.module_paths import find_mod
 from IPython.utils.path import get_py_filename, shellglob
 from IPython.utils.timing import clock, clock2
 from warnings import warn
 from logging import error
+from io import StringIO
 
-if PY3:
-    from io import StringIO
-else:
-    from StringIO import StringIO
 
 #-----------------------------------------------------------------------------
 # Magic implementation classes
 #-----------------------------------------------------------------------------
 
 
 class TimeitResult(object):
     """
     Object returned by the timeit magic with info about the run.
 
     Contains the following attributes :
 
     loops: (int) number of loops done per measurement
     repeat: (int) number of times the measurement has been repeated
     best: (float) best execution time / number
     all_runs: (list of float) execution time of each run (in s)
     compile_time: (float) time of statement compilation (s)
 
     """
     def __init__(self, loops, repeat, best, worst, all_runs, compile_time, precision):
         self.loops = loops
         self.repeat = repeat
         self.best = best
         self.worst = worst
         self.all_runs = all_runs
         self.compile_time = compile_time
         self._precision = precision
         self.timings = [ dt / self.loops for dt in all_runs]
 
     @property
     def average(self):
         return math.fsum(self.timings) / len(self.timings)
 
     @property
     def stdev(self):
         mean = self.average
         return (math.fsum([(x - mean) ** 2 for x in self.timings]) / len(self.timings)) ** 0.5
 
     def __str__(self):
         return (u"%s loop%s, average of %d: %s +- %s per loop (using standard deviation)"
                    % (self.loops,"" if self.loops == 1 else "s", self.repeat,
                       _format_time(self.average, self._precision),
                       _format_time(self.stdev, self._precision)))
 
     def _repr_pretty_(self, p , cycle):
        unic = self.__str__()
        p.text(u'<TimeitResult : '+unic+u'>')
 
 
 
 class TimeitTemplateFiller(ast.NodeTransformer):
     """Fill in the AST template for timing execution.
 
     This is quite closely tied to the template definition, which is in
     :meth:`ExecutionMagics.timeit`.
     """
     def __init__(self, ast_setup, ast_stmt):
         self.ast_setup = ast_setup
         self.ast_stmt = ast_stmt
 
     def visit_FunctionDef(self, node):
         "Fill in the setup statement"
         self.generic_visit(node)
         if node.name == "inner":
             node.body[:1] = self.ast_setup.body
 
         return node
 
     def visit_For(self, node):
         "Fill in the statement to be timed"
         if getattr(getattr(node.body[0], 'value', None), 'id', None) == 'stmt':
             node.body = self.ast_stmt.body
         return node
 
 
 class Timer(timeit.Timer):
     """Timer class that explicitly uses self.inner
     
     which is an undocumented implementation detail of CPython,
     not shared by PyPy.
     """
     # Timer.timeit copied from CPython 3.4.2
     def timeit(self, number=timeit.default_number):
         """Time 'number' executions of the main statement.
 
         To be precise, this executes the setup statement once, and
         then returns the time it takes to execute the main statement
         a number of times, as a float measured in seconds.  The
         argument is the number of times through the loop, defaulting
         to one million.  The main statement, the setup statement and
         the timer function to be used are passed to the constructor.
         """
         it = itertools.repeat(None, number)
         gcold = gc.isenabled()
         gc.disable()
         try:
             timing = self.inner(it, self.timer)
         finally:
             if gcold:
                 gc.enable()
         return timing
 
 
 @magics_class
 class ExecutionMagics(Magics):
     """Magics related to code execution, debugging, profiling, etc.
 
     """
 
     def __init__(self, shell):
         super(ExecutionMagics, self).__init__(shell)
         if profile is None:
             self.prun = self.profile_missing_notice
         # Default execution function used to actually run user code.
         self.default_runner = None
 
     def profile_missing_notice(self, *args, **kwargs):
         error("""\
 The profile module could not be found. It has been removed from the standard
 python packages because of its non-free license. To use profiling, install the
 python-profiler package from non-free.""")
 
     @skip_doctest
     @line_cell_magic
     def prun(self, parameter_s='', cell=None):
 
         """Run a statement through the python code profiler.
 
         Usage, in line mode:
           %prun [options] statement
 
         Usage, in cell mode:
           %%prun [options] [statement]
           code...
           code...
 
         In cell mode, the additional code lines are appended to the (possibly
         empty) statement in the first line.  Cell mode allows you to easily
         profile multiline blocks without having to put them in a separate
         function.
 
         The given statement (which doesn't require quote marks) is run via the
         python profiler in a manner similar to the profile.run() function.
         Namespaces are internally managed to work correctly; profile.run
         cannot be used in IPython because it makes certain assumptions about
         namespaces which do not hold under IPython.
 
         Options:
 
         -l <limit>
           you can place restrictions on what or how much of the
           profile gets printed. The limit value can be:
 
              * A string: only information for function names containing this string
                is printed.
 
              * An integer: only these many lines are printed.
 
              * A float (between 0 and 1): this fraction of the report is printed
                (for example, use a limit of 0.4 to see the topmost 40% only).
 
           You can combine several limits with repeated use of the option. For
           example, ``-l __init__ -l 5`` will print only the topmost 5 lines of
           information about class constructors.
 
         -r
           return the pstats.Stats object generated by the profiling. This
           object has all the information about the profile in it, and you can
           later use it for further analysis or in other functions.
 
         -s <key>
           sort profile by given key. You can provide more than one key
           by using the option several times: '-s key1 -s key2 -s key3...'. The
           default sorting key is 'time'.
 
           The following is copied verbatim from the profile documentation
           referenced below:
 
           When more than one key is provided, additional keys are used as
           secondary criteria when the there is equality in all keys selected
           before them.
 
           Abbreviations can be used for any key names, as long as the
           abbreviation is unambiguous.  The following are the keys currently
           defined:
 
           ============  =====================
           Valid Arg     Meaning
           ============  =====================
           "calls"       call count
           "cumulative"  cumulative time
           "file"        file name
           "module"      file name
           "pcalls"      primitive call count
           "line"        line number
           "name"        function name
           "nfl"         name/file/line
           "stdname"     standard name
           "time"        internal time
           ============  =====================
 
           Note that all sorts on statistics are in descending order (placing
           most time consuming items first), where as name, file, and line number
           searches are in ascending order (i.e., alphabetical). The subtle
           distinction between "nfl" and "stdname" is that the standard name is a
           sort of the name as printed, which means that the embedded line
           numbers get compared in an odd way.  For example, lines 3, 20, and 40
           would (if the file names were the same) appear in the string order
           "20" "3" and "40".  In contrast, "nfl" does a numeric compare of the
           line numbers.  In fact, sort_stats("nfl") is the same as
           sort_stats("name", "file", "line").
 
         -T <filename>
           save profile results as shown on screen to a text
           file. The profile is still shown on screen.
 
         -D <filename>
           save (via dump_stats) profile statistics to given
           filename. This data is in a format understood by the pstats module, and
           is generated by a call to the dump_stats() method of profile
           objects. The profile is still shown on screen.
 
         -q
           suppress output to the pager.  Best used with -T and/or -D above.
 
         If you want to run complete programs under the profiler's control, use
         ``%run -p [prof_opts] filename.py [args to program]`` where prof_opts
         contains profiler specific options as described here.
 
         You can read the complete documentation for the profile module with::
 
           In [1]: import profile; profile.help()
         """
         opts, arg_str = self.parse_options(parameter_s, 'D:l:rs:T:q',
                                            list_all=True, posix=False)
         if cell is not None:
             arg_str += '\n' + cell
         arg_str = self.shell.input_splitter.transform_cell(arg_str)
         return self._run_with_profiler(arg_str, opts, self.shell.user_ns)
 
     def _run_with_profiler(self, code, opts, namespace):
         """
         Run `code` with profiler.  Used by ``%prun`` and ``%run -p``.
 
         Parameters
         ----------
         code : str
             Code to be executed.
         opts : Struct
             Options parsed by `self.parse_options`.
         namespace : dict
             A dictionary for Python namespace (e.g., `self.shell.user_ns`).
 
         """
 
         # Fill default values for unspecified options:
         opts.merge(Struct(D=[''], l=[], s=['time'], T=['']))
 
         prof = profile.Profile()
         try:
             prof = prof.runctx(code, namespace, namespace)
             sys_exit = ''
         except SystemExit:
             sys_exit = """*** SystemExit exception caught in code being profiled."""
 
         stats = pstats.Stats(prof).strip_dirs().sort_stats(*opts.s)
 
         lims = opts.l
         if lims:
             lims = []  # rebuild lims with ints/floats/strings
             for lim in opts.l:
                 try:
                     lims.append(int(lim))
                 except ValueError:
                     try:
                         lims.append(float(lim))
                     except ValueError:
                         lims.append(lim)
 
         # Trap output.
         stdout_trap = StringIO()
         stats_stream = stats.stream
         try:
             stats.stream = stdout_trap
             stats.print_stats(*lims)
         finally:
             stats.stream = stats_stream
 
         output = stdout_trap.getvalue()
         output = output.rstrip()
 
         if 'q' not in opts:
             page.page(output)
         print(sys_exit, end=' ')
 
         dump_file = opts.D[0]
         text_file = opts.T[0]
         if dump_file:
             prof.dump_stats(dump_file)
             print('\n*** Profile stats marshalled to file',\
                   repr(dump_file)+'.',sys_exit)
         if text_file:
             pfile = open(text_file,'w')
             pfile.write(output)
             pfile.close()
             print('\n*** Profile printout saved to text file',\
                   repr(text_file)+'.',sys_exit)
 
         if 'r' in opts:
             return stats
         else:
             return None
 
     @line_magic
     def pdb(self, parameter_s=''):
         """Control the automatic calling of the pdb interactive debugger.
 
         Call as '%pdb on', '%pdb 1', '%pdb off' or '%pdb 0'. If called without
         argument it works as a toggle.
 
         When an exception is triggered, IPython can optionally call the
         interactive pdb debugger after the traceback printout. %pdb toggles
         this feature on and off.
 
         The initial state of this feature is set in your configuration
         file (the option is ``InteractiveShell.pdb``).
 
         If you want to just activate the debugger AFTER an exception has fired,
         without having to type '%pdb on' and rerunning your code, you can use
         the %debug magic."""
 
         par = parameter_s.strip().lower()
 
         if par:
             try:
                 new_pdb = {'off':0,'0':0,'on':1,'1':1}[par]
             except KeyError:
                 print ('Incorrect argument. Use on/1, off/0, '
                        'or nothing for a toggle.')
                 return
         else:
             # toggle
             new_pdb = not self.shell.call_pdb
 
         # set on the shell
         self.shell.call_pdb = new_pdb
         print('Automatic pdb calling has been turned',on_off(new_pdb))
 
     @skip_doctest
     @magic_arguments.magic_arguments()
     @magic_arguments.argument('--breakpoint', '-b', metavar='FILE:LINE',
         help="""
         Set break point at LINE in FILE.
         """
     )
     @magic_arguments.argument('statement', nargs='*',
         help="""
         Code to run in debugger.
         You can omit this in cell magic mode.
         """
     )
     @line_cell_magic
     def debug(self, line='', cell=None):
         """Activate the interactive debugger.
 
         This magic command support two ways of activating debugger.
         One is to activate debugger before executing code.  This way, you
         can set a break point, to step through the code from the point.
         You can use this mode by giving statements to execute and optionally
         a breakpoint.
 
         The other one is to activate debugger in post-mortem mode.  You can
         activate this mode simply running %debug without any argument.
         If an exception has just occurred, this lets you inspect its stack
         frames interactively.  Note that this will always work only on the last
         traceback that occurred, so you must call this quickly after an
         exception that you wish to inspect has fired, because if another one
         occurs, it clobbers the previous one.
 
         If you want IPython to automatically do this on every exception, see
         the %pdb magic for more details.
         """
         args = magic_arguments.parse_argstring(self.debug, line)
 
         if not (args.breakpoint or args.statement or cell):
             self._debug_post_mortem()
         else:
             code = "\n".join(args.statement)
             if cell:
                 code += "\n" + cell
             self._debug_exec(code, args.breakpoint)
 
     def _debug_post_mortem(self):
         self.shell.debugger(force=True)
 
     def _debug_exec(self, code, breakpoint):
         if breakpoint:
             (filename, bp_line) = breakpoint.rsplit(':', 1)
             bp_line = int(bp_line)
         else:
             (filename, bp_line) = (None, None)
         self._run_with_debugger(code, self.shell.user_ns, filename, bp_line)
 
     @line_magic
     def tb(self, s):
         """Print the last traceback with the currently active exception mode.
 
         See %xmode for changing exception reporting modes."""
         self.shell.showtraceback()
 
     @skip_doctest
     @line_magic
     def run(self, parameter_s='', runner=None,
                   file_finder=get_py_filename):
         """Run the named file inside IPython as a program.
 
         Usage::
         
           %run [-n -i -e -G]
                [( -t [-N<N>] | -d [-b<N>] | -p [profile options] )]
                ( -m mod | file ) [args]
 
         Parameters after the filename are passed as command-line arguments to
         the program (put in sys.argv). Then, control returns to IPython's
         prompt.
 
         This is similar to running at a system prompt ``python file args``,
         but with the advantage of giving you IPython's tracebacks, and of
         loading all variables into your interactive namespace for further use
         (unless -p is used, see below).
 
         The file is executed in a namespace initially consisting only of
         ``__name__=='__main__'`` and sys.argv constructed as indicated. It thus
         sees its environment as if it were being run as a stand-alone program
         (except for sharing global objects such as previously imported
         modules). But after execution, the IPython interactive namespace gets
         updated with all variables defined in the program (except for __name__
         and sys.argv). This allows for very convenient loading of code for
         interactive work, while giving each program a 'clean sheet' to run in.
 
         Arguments are expanded using shell-like glob match.  Patterns
         '*', '?', '[seq]' and '[!seq]' can be used.  Additionally,
         tilde '~' will be expanded into user's home directory.  Unlike
         real shells, quotation does not suppress expansions.  Use
         *two* back slashes (e.g. ``\\\\*``) to suppress expansions.
         To completely disable these expansions, you can use -G flag.
 
         Options:
 
         -n
           __name__ is NOT set to '__main__', but to the running file's name
           without extension (as python does under import).  This allows running
           scripts and reloading the definitions in them without calling code
           protected by an ``if __name__ == "__main__"`` clause.
 
         -i
           run the file in IPython's namespace instead of an empty one. This
           is useful if you are experimenting with code written in a text editor
           which depends on variables defined interactively.
 
         -e
           ignore sys.exit() calls or SystemExit exceptions in the script
           being run.  This is particularly useful if IPython is being used to
           run unittests, which always exit with a sys.exit() call.  In such
           cases you are interested in the output of the test results, not in
           seeing a traceback of the unittest module.
 
         -t
           print timing information at the end of the run.  IPython will give
           you an estimated CPU time consumption for your script, which under
           Unix uses the resource module to avoid the wraparound problems of
           time.clock().  Under Unix, an estimate of time spent on system tasks
           is also given (for Windows platforms this is reported as 0.0).
 
         If -t is given, an additional ``-N<N>`` option can be given, where <N>
         must be an integer indicating how many times you want the script to
         run.  The final timing report will include total and per run results.
 
         For example (testing the script uniq_stable.py)::
 
             In [1]: run -t uniq_stable
 
             IPython CPU timings (estimated):
               User  :    0.19597 s.
               System:        0.0 s.
 
             In [2]: run -t -N5 uniq_stable
 
             IPython CPU timings (estimated):
             Total runs performed: 5
               Times :      Total       Per run
               User  :   0.910862 s,  0.1821724 s.
               System:        0.0 s,        0.0 s.
 
         -d
           run your program under the control of pdb, the Python debugger.
           This allows you to execute your program step by step, watch variables,
           etc.  Internally, what IPython does is similar to calling::
 
               pdb.run('execfile("YOURFILENAME")')
 
           with a breakpoint set on line 1 of your file.  You can change the line
           number for this automatic breakpoint to be <N> by using the -bN option
           (where N must be an integer). For example::
 
               %run -d -b40 myscript
 
           will set the first breakpoint at line 40 in myscript.py.  Note that
           the first breakpoint must be set on a line which actually does
           something (not a comment or docstring) for it to stop execution.
 
           Or you can specify a breakpoint in a different file::
 
               %run -d -b myotherfile.py:20 myscript
 
           When the pdb debugger starts, you will see a (Pdb) prompt.  You must
           first enter 'c' (without quotes) to start execution up to the first
           breakpoint.
 
           Entering 'help' gives information about the use of the debugger.  You
           can easily see pdb's full documentation with "import pdb;pdb.help()"
           at a prompt.
 
         -p
           run program under the control of the Python profiler module (which
           prints a detailed report of execution times, function calls, etc).
 
           You can pass other options after -p which affect the behavior of the
           profiler itself. See the docs for %prun for details.
 
           In this mode, the program's variables do NOT propagate back to the
           IPython interactive namespace (because they remain in the namespace
           where the profiler executes them).
 
           Internally this triggers a call to %prun, see its documentation for
           details on the options available specifically for profiling.
 
         There is one special usage for which the text above doesn't apply:
         if the filename ends with .ipy[nb], the file is run as ipython script,
         just as if the commands were written on IPython prompt.
 
         -m
           specify module name to load instead of script path. Similar to
           the -m option for the python interpreter. Use this option last if you
           want to combine with other %run options. Unlike the python interpreter
           only source modules are allowed no .pyc or .pyo files.
           For example::
 
               %run -m example
 
           will run the example module.
 
         -G
           disable shell-like glob expansion of arguments.
 
         """
 
         # get arguments and set sys.argv for program to be run.
         opts, arg_lst = self.parse_options(parameter_s,
                                            'nidtN:b:pD:l:rs:T:em:G',
                                            mode='list', list_all=1)
         if "m" in opts:
             modulename = opts["m"][0]
             modpath = find_mod(modulename)
             if modpath is None:
                 warn('%r is not a valid modulename on sys.path'%modulename)
                 return
             arg_lst = [modpath] + arg_lst
         try:
             filename = file_finder(arg_lst[0])
         except IndexError:
             warn('you must provide at least a filename.')
             print('\n%run:\n', oinspect.getdoc(self.run))
             return
         except IOError as e:
             try:
                 msg = str(e)
             except UnicodeError:
                 msg = e.message
             error(msg)
             return
 
         if filename.lower().endswith(('.ipy', '.ipynb')):
             with preserve_keys(self.shell.user_ns, '__file__'):
                 self.shell.user_ns['__file__'] = filename
                 self.shell.safe_execfile_ipy(filename)
             return
 
         # Control the response to exit() calls made by the script being run
         exit_ignore = 'e' in opts
 
         # Make sure that the running script gets a proper sys.argv as if it
         # were run from a system shell.
         save_argv = sys.argv # save it for later restoring
 
         if 'G' in opts:
             args = arg_lst[1:]
         else:
             # tilde and glob expansion
             args = shellglob(map(os.path.expanduser,  arg_lst[1:]))
 
         sys.argv = [filename] + args  # put in the proper filename
-        # protect sys.argv from potential unicode strings on Python 2:
-        if not py3compat.PY3:
-            sys.argv = [ py3compat.cast_bytes(a) for a in sys.argv ]
 
         if 'i' in opts:
             # Run in user's interactive namespace
             prog_ns = self.shell.user_ns
             __name__save = self.shell.user_ns['__name__']
             prog_ns['__name__'] = '__main__'
             main_mod = self.shell.user_module
             
             # Since '%run foo' emulates 'python foo.py' at the cmd line, we must
             # set the __file__ global in the script's namespace
             # TK: Is this necessary in interactive mode?
             prog_ns['__file__'] = filename
         else:
             # Run in a fresh, empty namespace
             if 'n' in opts:
                 name = os.path.splitext(os.path.basename(filename))[0]
             else:
                 name = '__main__'
 
             # The shell MUST hold a reference to prog_ns so after %run
             # exits, the python deletion mechanism doesn't zero it out
             # (leaving dangling references). See interactiveshell for details
             main_mod = self.shell.new_main_mod(filename, name)
             prog_ns = main_mod.__dict__
 
         # pickle fix.  See interactiveshell for an explanation.  But we need to
         # make sure that, if we overwrite __main__, we replace it at the end
         main_mod_name = prog_ns['__name__']
 
         if main_mod_name == '__main__':
             restore_main = sys.modules['__main__']
         else:
             restore_main = False
 
         # This needs to be undone at the end to prevent holding references to
         # every single object ever created.
         sys.modules[main_mod_name] = main_mod
 
         if 'p' in opts or 'd' in opts:
             if 'm' in opts:
                 code = 'run_module(modulename, prog_ns)'
                 code_ns = {
                     'run_module': self.shell.safe_run_module,
                     'prog_ns': prog_ns,
                     'modulename': modulename,
                 }
             else:
                 if 'd' in opts:
                     # allow exceptions to raise in debug mode
                     code = 'execfile(filename, prog_ns, raise_exceptions=True)'
                 else:
                     code = 'execfile(filename, prog_ns)'
                 code_ns = {
                     'execfile': self.shell.safe_execfile,
                     'prog_ns': prog_ns,
                     'filename': get_py_filename(filename),
                 }
 
         try:
             stats = None
             if 'p' in opts:
                 stats = self._run_with_profiler(code, opts, code_ns)
             else:
                 if 'd' in opts:
                     bp_file, bp_line = parse_breakpoint(
                         opts.get('b', ['1'])[0], filename)
                     self._run_with_debugger(
                         code, code_ns, filename, bp_line, bp_file)
                 else:
                     if 'm' in opts:
                         def run():
                             self.shell.safe_run_module(modulename, prog_ns)
                     else:
                         if runner is None:
                             runner = self.default_runner
                         if runner is None:
                             runner = self.shell.safe_execfile
 
                         def run():
                             runner(filename, prog_ns, prog_ns,
                                     exit_ignore=exit_ignore)
 
                     if 't' in opts:
                         # timed execution
                         try:
                             nruns = int(opts['N'][0])
                             if nruns < 1:
                                 error('Number of runs must be >=1')
                                 return
                         except (KeyError):
                             nruns = 1
                         self._run_with_timing(run, nruns)
                     else:
                         # regular execution
                         run()
 
             if 'i' in opts:
                 self.shell.user_ns['__name__'] = __name__save
             else:
                 # update IPython interactive namespace
 
                 # Some forms of read errors on the file may mean the
                 # __name__ key was never set; using pop we don't have to
                 # worry about a possible KeyError.
                 prog_ns.pop('__name__', None)
 
                 with preserve_keys(self.shell.user_ns, '__file__'):
                     self.shell.user_ns.update(prog_ns)
         finally:
             # It's a bit of a mystery why, but __builtins__ can change from
             # being a module to becoming a dict missing some key data after
             # %run.  As best I can see, this is NOT something IPython is doing
             # at all, and similar problems have been reported before:
             # http://coding.derkeiler.com/Archive/Python/comp.lang.python/2004-10/0188.html
             # Since this seems to be done by the interpreter itself, the best
             # we can do is to at least restore __builtins__ for the user on
             # exit.
             self.shell.user_ns['__builtins__'] = builtin_mod
 
             # Ensure key global structures are restored
             sys.argv = save_argv
             if restore_main:
                 sys.modules['__main__'] = restore_main
             else:
                 # Remove from sys.modules the reference to main_mod we'd
                 # added.  Otherwise it will trap references to objects
                 # contained therein.
                 del sys.modules[main_mod_name]
 
         return stats
 
     def _run_with_debugger(self, code, code_ns, filename=None,
                            bp_line=None, bp_file=None):
         """
         Run `code` in debugger with a break point.
 
         Parameters
         ----------
         code : str
             Code to execute.
         code_ns : dict
             A namespace in which `code` is executed.
         filename : str
             `code` is ran as if it is in `filename`.
         bp_line : int, optional
             Line number of the break point.
         bp_file : str, optional
             Path to the file in which break point is specified.
             `filename` is used if not given.
 
         Raises
         ------
         UsageError
             If the break point given by `bp_line` is not valid.
 
         """
         deb = self.shell.InteractiveTB.pdb
         if not deb:
             self.shell.InteractiveTB.pdb = self.shell.InteractiveTB.debugger_cls()
             deb = self.shell.InteractiveTB.pdb
 
         # deb.checkline() fails if deb.curframe exists but is None; it can
         # handle it not existing. https://github.com/ipython/ipython/issues/10028
         if hasattr(deb, 'curframe'):
             del deb.curframe
 
         # reset Breakpoint state, which is moronically kept
         # in a class
         bdb.Breakpoint.next = 1
         bdb.Breakpoint.bplist = {}
         bdb.Breakpoint.bpbynumber = [None]
         if bp_line is not None:
             # Set an initial breakpoint to stop execution
             maxtries = 10
             bp_file = bp_file or filename
             checkline = deb.checkline(bp_file, bp_line)
             if not checkline:
                 for bp in range(bp_line + 1, bp_line + maxtries + 1):
                     if deb.checkline(bp_file, bp):
                         break
                 else:
                     msg = ("\nI failed to find a valid line to set "
                            "a breakpoint\n"
                            "after trying up to line: %s.\n"
                            "Please set a valid breakpoint manually "
                            "with the -b option." % bp)
                     raise UsageError(msg)
             # if we find a good linenumber, set the breakpoint
             deb.do_break('%s:%s' % (bp_file, bp_line))
 
         if filename:
             # Mimic Pdb._runscript(...)
             deb._wait_for_mainpyfile = True
             deb.mainpyfile = deb.canonic(filename)
 
         # Start file run
         print("NOTE: Enter 'c' at the %s prompt to continue execution." % deb.prompt)
         try:
             if filename:
                 # save filename so it can be used by methods on the deb object
                 deb._exec_filename = filename
             while True:
                 try:
                     deb.run(code, code_ns)
                 except Restart:
                     print("Restarting")
                     if filename:
                         deb._wait_for_mainpyfile = True
                         deb.mainpyfile = deb.canonic(filename)
                     continue
                 else:
                     break
             
 
         except:
             etype, value, tb = sys.exc_info()
             # Skip three frames in the traceback: the %run one,
             # one inside bdb.py, and the command-line typed by the
             # user (run by exec in pdb itself).
             self.shell.InteractiveTB(etype, value, tb, tb_offset=3)
 
     @staticmethod
     def _run_with_timing(run, nruns):
         """
         Run function `run` and print timing information.
 
         Parameters
         ----------
         run : callable
             Any callable object which takes no argument.
         nruns : int
             Number of times to execute `run`.
 
         """
         twall0 = time.time()
         if nruns == 1:
             t0 = clock2()
             run()
             t1 = clock2()
             t_usr = t1[0] - t0[0]
             t_sys = t1[1] - t0[1]
             print("\nIPython CPU timings (estimated):")
             print("  User   : %10.2f s." % t_usr)
             print("  System : %10.2f s." % t_sys)
         else:
             runs = range(nruns)
             t0 = clock2()
             for nr in runs:
                 run()
             t1 = clock2()
             t_usr = t1[0] - t0[0]
             t_sys = t1[1] - t0[1]
             print("\nIPython CPU timings (estimated):")
             print("Total runs performed:", nruns)
             print("  Times  : %10s   %10s" % ('Total', 'Per run'))
             print("  User   : %10.2f s, %10.2f s." % (t_usr, t_usr / nruns))
             print("  System : %10.2f s, %10.2f s." % (t_sys, t_sys / nruns))
         twall1 = time.time()
         print("Wall time: %10.2f s." % (twall1 - twall0))
 
     @skip_doctest
     @line_cell_magic
     def timeit(self, line='', cell=None):
         """Time execution of a Python statement or expression
 
         Usage, in line mode:
           %timeit [-n<N> -r<R> [-t|-c] -q -p<P> -o] statement
         or in cell mode:
           %%timeit [-n<N> -r<R> [-t|-c] -q -p<P> -o] setup_code
           code
           code...
 
         Time execution of a Python statement or expression using the timeit
         module.  This function can be used both as a line and cell magic:
 
         - In line mode you can time a single-line statement (though multiple
           ones can be chained with using semicolons).
 
         - In cell mode, the statement in the first line is used as setup code
           (executed but not timed) and the body of the cell is timed.  The cell
           body has access to any variables created in the setup code.
 
         Options:
         -n<N>: execute the given statement <N> times in a loop. If this value
         is not given, a fitting value is chosen.
 
         -r<R>: repeat the loop iteration <R> times and take the best result.
         Default: 3
 
         -t: use time.time to measure the time, which is the default on Unix.
         This function measures wall time.
 
         -c: use time.clock to measure the time, which is the default on
         Windows and measures wall time. On Unix, resource.getrusage is used
         instead and returns the CPU user time.
 
         -p<P>: use a precision of <P> digits to display the timing result.
         Default: 3
 
         -q: Quiet, do not print result.
 
         -o: return a TimeitResult that can be stored in a variable to inspect
             the result in more details.
 
 
         Examples
         --------
         ::
 
           In [1]: %timeit pass
           100000000 loops, average of 7: 5.48 ns +- 0.354 ns per loop (using standard deviation)
 
           In [2]: u = None
 
           In [3]: %timeit u is None
           10000000 loops, average of 7: 22.7 ns +- 2.33 ns per loop (using standard deviation)
 
           In [4]: %timeit -r 4 u == None
           10000000 loops, average of 4: 27.5 ns +- 2.91 ns per loop (using standard deviation)
 
           In [5]: import time
 
           In [6]: %timeit -n1 time.sleep(2)
           1 loop, average of 7: 2 s +- 4.71 µs per loop (using standard deviation)
 
 
         The times reported by %timeit will be slightly higher than those
         reported by the timeit.py script when variables are accessed. This is
         due to the fact that %timeit executes the statement in the namespace
         of the shell, compared with timeit.py, which uses a single setup
         statement to import function or create variables. Generally, the bias
         does not matter as long as results from timeit.py are not mixed with
         those from %timeit."""
 
         opts, stmt = self.parse_options(line,'n:r:tcp:qo',
                                         posix=False, strict=False)
         if stmt == "" and cell is None:
             return
         
         timefunc = timeit.default_timer
         number = int(getattr(opts, "n", 0))
         default_repeat = 7 if timeit.default_repeat < 7 else timeit.default_repeat
         repeat = int(getattr(opts, "r", default_repeat))
         precision = int(getattr(opts, "p", 3))
         quiet = 'q' in opts
         return_result = 'o' in opts
         if hasattr(opts, "t"):
             timefunc = time.time
         if hasattr(opts, "c"):
             timefunc = clock
 
         timer = Timer(timer=timefunc)
         # this code has tight coupling to the inner workings of timeit.Timer,
         # but is there a better way to achieve that the code stmt has access
         # to the shell namespace?
         transform  = self.shell.input_splitter.transform_cell
 
         if cell is None:
             # called as line magic
             ast_setup = self.shell.compile.ast_parse("pass")
             ast_stmt = self.shell.compile.ast_parse(transform(stmt))
         else:
             ast_setup = self.shell.compile.ast_parse(transform(stmt))
             ast_stmt = self.shell.compile.ast_parse(transform(cell))
 
         ast_setup = self.shell.transform_ast(ast_setup)
         ast_stmt = self.shell.transform_ast(ast_stmt)
 
         # This codestring is taken from timeit.template - we fill it in as an
         # AST, so that we can apply our AST transformations to the user code
         # without affecting the timing code.
         timeit_ast_template = ast.parse('def inner(_it, _timer):\n'
                                         '    setup\n'
                                         '    _t0 = _timer()\n'
                                         '    for _i in _it:\n'
                                         '        stmt\n'
                                         '    _t1 = _timer()\n'
                                         '    return _t1 - _t0\n')
 
         timeit_ast = TimeitTemplateFiller(ast_setup, ast_stmt).visit(timeit_ast_template)
         timeit_ast = ast.fix_missing_locations(timeit_ast)
 
         # Track compilation time so it can be reported if too long
         # Minimum time above which compilation time will be reported
         tc_min = 0.1
 
         t0 = clock()
         code = self.shell.compile(timeit_ast, "<magic-timeit>", "exec")
         tc = clock()-t0
 
         ns = {}
         exec(code, self.shell.user_ns, ns)
         timer.inner = ns["inner"]
 
         # This is used to check if there is a huge difference between the
         # best and worst timings.
         # Issue: https://github.com/ipython/ipython/issues/6471
         if number == 0:
             # determine number so that 0.2 <= total time < 2.0
             for index in range(0, 10):
                 number = 10 ** index
                 time_number = timer.timeit(number)
                 if time_number >= 0.2:
                     break
 
         all_runs = timer.repeat(repeat, number)
         best = min(all_runs) / number
         worst = max(all_runs) / number
         timeit_result = TimeitResult(number, repeat, best, worst, all_runs, tc, precision)
 
         if not quiet :
             # Check best timing is greater than zero to avoid a
             # ZeroDivisionError.
             # In cases where the slowest timing is lesser than a micosecond
             # we assume that it does not really matter if the fastest
             # timing is 4 times faster than the slowest timing or not.
             if worst > 4 * best and best > 0 and worst > 1e-6:
                 print("The slowest run took %0.2f times longer than the "
                       "fastest. This could mean that an intermediate result "
                       "is being cached." % (worst / best))
            
             print( timeit_result )
 
             if tc > tc_min:
                 print("Compiler time: %.2f s" % tc)
         if return_result:
             return timeit_result
 
     @skip_doctest
     @needs_local_scope
     @line_cell_magic
     def time(self,line='', cell=None, local_ns=None):
         """Time execution of a Python statement or expression.
 
         The CPU and wall clock times are printed, and the value of the
         expression (if any) is returned.  Note that under Win32, system time
         is always reported as 0, since it can not be measured.
         
         This function can be used both as a line and cell magic:
 
         - In line mode you can time a single-line statement (though multiple
           ones can be chained with using semicolons).
 
         - In cell mode, you can time the cell body (a directly 
           following statement raises an error).
 
         This function provides very basic timing functionality.  Use the timeit 
         magic for more control over the measurement.
 
         Examples
         --------
         ::
 
           In [1]: %time 2**128
           CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s
           Wall time: 0.00
           Out[1]: 340282366920938463463374607431768211456L
 
           In [2]: n = 1000000
 
           In [3]: %time sum(range(n))
           CPU times: user 1.20 s, sys: 0.05 s, total: 1.25 s
           Wall time: 1.37
           Out[3]: 499999500000L
 
           In [4]: %time print 'hello world'
           hello world
           CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s
           Wall time: 0.00
 
           Note that the time needed by Python to compile the given expression
           will be reported if it is more than 0.1s.  In this example, the
           actual exponentiation is done by Python at compilation time, so while
           the expression can take a noticeable amount of time to compute, that
           time is purely due to the compilation:
 
           In [5]: %time 3**9999;
           CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s
           Wall time: 0.00 s
 
           In [6]: %time 3**999999;
           CPU times: user 0.00 s, sys: 0.00 s, total: 0.00 s
           Wall time: 0.00 s
           Compiler : 0.78 s
           """
 
         # fail immediately if the given expression can't be compiled
         
         if line and cell:
             raise UsageError("Can't use statement directly after '%%time'!")
         
         if cell:
             expr = self.shell.input_transformer_manager.transform_cell(cell)
         else:
             expr = self.shell.input_transformer_manager.transform_cell(line)
 
         # Minimum time above which parse time will be reported
         tp_min = 0.1
 
         t0 = clock()
         expr_ast = self.shell.compile.ast_parse(expr)
         tp = clock()-t0
 
         # Apply AST transformations
         expr_ast = self.shell.transform_ast(expr_ast)
 
         # Minimum time above which compilation time will be reported
         tc_min = 0.1
 
         if len(expr_ast.body)==1 and isinstance(expr_ast.body[0], ast.Expr):
             mode = 'eval'
             source = '<timed eval>'
             expr_ast = ast.Expression(expr_ast.body[0].value)
         else:
             mode = 'exec'
             source = '<timed exec>'
         t0 = clock()
         code = self.shell.compile(expr_ast, source, mode)
         tc = clock()-t0
 
         # skew measurement as little as possible
         glob = self.shell.user_ns
         wtime = time.time
         # time execution
         wall_st = wtime()
         if mode=='eval':
             st = clock2()
             try:
                 out = eval(code, glob, local_ns)
             except:
                 self.shell.showtraceback()
                 return
             end = clock2()
         else:
             st = clock2()
             try:
                 exec(code, glob, local_ns)
             except:
                 self.shell.showtraceback()
                 return
             end = clock2()
             out = None
         wall_end = wtime()
         # Compute actual times and report
         wall_time = wall_end-wall_st
         cpu_user = end[0]-st[0]
         cpu_sys = end[1]-st[1]
         cpu_tot = cpu_user+cpu_sys
         # On windows cpu_sys is always zero, so no new information to the next print 
         if sys.platform != 'win32':
             print("CPU times: user %s, sys: %s, total: %s" % \
                 (_format_time(cpu_user),_format_time(cpu_sys),_format_time(cpu_tot)))
         print("Wall time: %s" % _format_time(wall_time))
         if tc > tc_min:
             print("Compiler : %s" % _format_time(tc))
         if tp > tp_min:
             print("Parser   : %s" % _format_time(tp))
         return out
 
     @skip_doctest
     @line_magic
     def macro(self, parameter_s=''):
         """Define a macro for future re-execution. It accepts ranges of history,
         filenames or string objects.
 
         Usage:\\
           %macro [options] name n1-n2 n3-n4 ... n5 .. n6 ...
 
         Options:
 
           -r: use 'raw' input.  By default, the 'processed' history is used,
           so that magics are loaded in their transformed version to valid
           Python.  If this option is given, the raw input as typed at the
           command line is used instead.
           
           -q: quiet macro definition.  By default, a tag line is printed 
           to indicate the macro has been created, and then the contents of 
           the macro are printed.  If this option is given, then no printout
           is produced once the macro is created.
 
         This will define a global variable called `name` which is a string
         made of joining the slices and lines you specify (n1,n2,... numbers
         above) from your input history into a single string. This variable
         acts like an automatic function which re-executes those lines as if
         you had typed them. You just type 'name' at the prompt and the code
         executes.
 
         The syntax for indicating input ranges is described in %history.
 
         Note: as a 'hidden' feature, you can also use traditional python slice
         notation, where N:M means numbers N through M-1.
 
         For example, if your history contains (print using %hist -n )::
 
           44: x=1
           45: y=3
           46: z=x+y
           47: print x
           48: a=5
           49: print 'x',x,'y',y
 
         you can create a macro with lines 44 through 47 (included) and line 49
         called my_macro with::
 
           In [55]: %macro my_macro 44-47 49
 
         Now, typing `my_macro` (without quotes) will re-execute all this code
         in one pass.
 
         You don't need to give the line-numbers in order, and any given line
         number can appear multiple times. You can assemble macros with any
         lines from your input history in any order.
 
         The macro is a simple object which holds its value in an attribute,
         but IPython's display system checks for macros and executes them as
         code instead of printing them when you type their name.
 
         You can view a macro's contents by explicitly printing it with::
 
           print macro_name
 
         """
         opts,args = self.parse_options(parameter_s,'rq',mode='list')
         if not args:   # List existing macros
             return sorted(k for k,v in self.shell.user_ns.items() if isinstance(v, Macro))
         if len(args) == 1:
             raise UsageError(
                 "%macro insufficient args; usage '%macro name n1-n2 n3-4...")
         name, codefrom = args[0], " ".join(args[1:])
 
         #print 'rng',ranges  # dbg
         try:
             lines = self.shell.find_user_code(codefrom, 'r' in opts)
         except (ValueError, TypeError) as e:
             print(e.args[0])
             return
         macro = Macro(lines)
         self.shell.define_macro(name, macro)
         if not ( 'q' in opts) : 
             print('Macro `%s` created. To execute, type its name (without quotes).' % name)
             print('=== Macro contents: ===')
             print(macro, end=' ')
 
     @magic_arguments.magic_arguments()
     @magic_arguments.argument('output', type=str, default='', nargs='?',
         help="""The name of the variable in which to store output.
         This is a utils.io.CapturedIO object with stdout/err attributes
         for the text of the captured output.
 
         CapturedOutput also has a show() method for displaying the output,
         and __call__ as well, so you can use that to quickly display the
         output.
 
         If unspecified, captured output is discarded.
         """
     )
     @magic_arguments.argument('--no-stderr', action="store_true",
         help="""Don't capture stderr."""
     )
     @magic_arguments.argument('--no-stdout', action="store_true",
         help="""Don't capture stdout."""
     )
     @magic_arguments.argument('--no-display', action="store_true",
         help="""Don't capture IPython's rich display."""
     )
     @cell_magic
     def capture(self, line, cell):
         """run the cell, capturing stdout, stderr, and IPython's rich display() calls."""
         args = magic_arguments.parse_argstring(self.capture, line)
         out = not args.no_stdout
         err = not args.no_stderr
         disp = not args.no_display
         with capture_output(out, err, disp) as io:
             self.shell.run_cell(cell)
         if args.output:
             self.shell.user_ns[args.output] = io
 
 def parse_breakpoint(text, current_file):
     '''Returns (file, line) for file:line and (current_file, line) for line'''
     colon = text.find(':')
     if colon == -1:
         return current_file, int(text)
     else:
         return text[:colon], int(text[colon+1:])
     
 def _format_time(timespan, precision=3):
     """Formats the timespan in a human readable form"""
 
     if timespan >= 60.0:
         # we have more than a minute, format that in a human readable form
         # Idea from http://snipplr.com/view/5713/
         parts = [("d", 60*60*24),("h", 60*60),("min", 60), ("s", 1)]
         time = []
         leftover = timespan
         for suffix, length in parts:
             value = int(leftover / length)
             if value > 0:
                 leftover = leftover % length
                 time.append(u'%s%s' % (str(value), suffix))
             if leftover < 1:
                 break
         return " ".join(time)
 
     
     # Unfortunately the unicode 'micro' symbol can cause problems in
     # certain terminals.  
     # See bug: https://bugs.launchpad.net/ipython/+bug/348466
     # Try to prevent crashes by being more secure than it needs to
     # E.g. eclipse is able to print a µ, but has no sys.stdout.encoding set.
     units = [u"s", u"ms",u'us',"ns"] # the save value   
     if hasattr(sys.stdout, 'encoding') and sys.stdout.encoding:
         try:
             u'\xb5'.encode(sys.stdout.encoding)
             units = [u"s", u"ms",u'\xb5s',"ns"]
         except:
             pass
     scaling = [1, 1e3, 1e6, 1e9]
         
     if timespan > 0.0:
         order = min(-int(math.floor(math.log10(timespan)) // 3), 3)
     else:
         order = 3
     return u"%.*g %s" % (precision, timespan * scaling[order], units[order])

IPython/core/tests/test_splitinput.py

@@ -1,42 +1,39 b''
 # coding: utf-8
 import nose.tools as nt
 
 from IPython.core.splitinput import split_user_input, LineInfo
 from IPython.testing import tools as tt
 from IPython.utils import py3compat
 
 tests = [
     ('x=1', ('', '', 'x', '=1')),
     ('?', ('', '?', '', '')),
     ('??', ('', '??', '', '')),
     (' ?', (' ', '?', '', '')),
     (' ??', (' ', '??', '', '')),
     ('??x', ('', '??', 'x', '')),
     ('?x=1', ('', '?', 'x', '=1')),
     ('!ls', ('', '!', 'ls', '')),
     ('  !ls', ('  ', '!', 'ls', '')),
     ('!!ls', ('', '!!', 'ls', '')),
     ('  !!ls', ('  ', '!!', 'ls', '')),
     (',ls', ('', ',', 'ls', '')),
     (';ls', ('', ';', 'ls', '')),
     ('  ;ls', ('  ', ';', 'ls', '')),
     ('f.g(x)',  ('', '', 'f.g', '(x)')),
     ('f.g (x)', ('', '', 'f.g', '(x)')),
     ('?%hist1', ('', '?', '%hist1', '')),
     ('?%%hist2', ('', '?', '%%hist2', '')),
     ('??%hist3', ('', '??', '%hist3', '')),
     ('??%%hist4', ('', '??', '%%hist4', '')),
     ('?x*', ('', '?', 'x*', '')),
     ]
-if py3compat.PY3:
-    tests.append((u"Pérez Fernando", (u'', u'', u'Pérez', u'Fernando')))
-else:
-    tests.append((u"Pérez Fernando", (u'', u'', u'P', u'érez Fernando')))
+tests.append((u"Pérez Fernando", (u'', u'', u'Pérez', u'Fernando')))
 
 def test_split_user_input():
     return tt.check_pairs(split_user_input, tests)
 
 def test_LineInfo():
     """Simple test for LineInfo construction and str()"""
     linfo = LineInfo('  %cd /home')
     nt.assert_equal(str(linfo), 'LineInfo [  |%|cd|/home]')

IPython/core/ultratb.py

@@ -1,1469 +1,1462 b''
 # -*- coding: utf-8 -*-
 """
 Verbose and colourful traceback formatting.
 
 **ColorTB**
 
 I've always found it a bit hard to visually parse tracebacks in Python.  The
 ColorTB class is a solution to that problem.  It colors the different parts of a
 traceback in a manner similar to what you would expect from a syntax-highlighting
 text editor.
 
 Installation instructions for ColorTB::
 
     import sys,ultratb
     sys.excepthook = ultratb.ColorTB()
 
 **VerboseTB**
 
 I've also included a port of Ka-Ping Yee's "cgitb.py" that produces all kinds
 of useful info when a traceback occurs.  Ping originally had it spit out HTML
 and intended it for CGI programmers, but why should they have all the fun?  I
 altered it to spit out colored text to the terminal.  It's a bit overwhelming,
 but kind of neat, and maybe useful for long-running programs that you believe
 are bug-free.  If a crash *does* occur in that type of program you want details.
 Give it a shot--you'll love it or you'll hate it.
 
 .. note::
 
   The Verbose mode prints the variables currently visible where the exception
   happened (shortening their strings if too long). This can potentially be
   very slow, if you happen to have a huge data structure whose string
   representation is complex to compute. Your computer may appear to freeze for
   a while with cpu usage at 100%. If this occurs, you can cancel the traceback
   with Ctrl-C (maybe hitting it more than once).
 
   If you encounter this kind of situation often, you may want to use the
   Verbose_novars mode instead of the regular Verbose, which avoids formatting
   variables (but otherwise includes the information and context given by
   Verbose).
 
 .. note::
 
   The verbose mode print all variables in the stack, which means it can
   potentially leak sensitive information like access keys, or unencryted
   password.
 
 Installation instructions for VerboseTB::
 
     import sys,ultratb
     sys.excepthook = ultratb.VerboseTB()
 
 Note:  Much of the code in this module was lifted verbatim from the standard
 library module 'traceback.py' and Ka-Ping Yee's 'cgitb.py'.
 
 Color schemes
 -------------
 
 The colors are defined in the class TBTools through the use of the
 ColorSchemeTable class. Currently the following exist:
 
   - NoColor: allows all of this module to be used in any terminal (the color
     escapes are just dummy blank strings).
 
   - Linux: is meant to look good in a terminal like the Linux console (black
     or very dark background).
 
   - LightBG: similar to Linux but swaps dark/light colors to be more readable
     in light background terminals.
 
   - Neutral: a neutral color scheme that should be readable on both light and
     dark background
 
 You can implement other color schemes easily, the syntax is fairly
 self-explanatory. Please send back new schemes you develop to the author for
 possible inclusion in future releases.
 
 Inheritance diagram:
 
 .. inheritance-diagram:: IPython.core.ultratb
    :parts: 3
 """
 
 #*****************************************************************************
 # Copyright (C) 2001 Nathaniel Gray <[e-mail]>
 # Copyright (C) 2001-2004 Fernando Perez <[e-mail]>
 #
 # Distributed under the terms of the BSD License.  The full license is in
 # the file COPYING, distributed as part of this software.
 #*****************************************************************************
 
 
 import dis
 import inspect
 import keyword
 import linecache
 import os
 import pydoc
 import re
 import sys
 import time
 import tokenize
 import traceback
 import types
 
 try:  # Python 2
     generate_tokens = tokenize.generate_tokens
 except AttributeError:  # Python 3
     generate_tokens = tokenize.tokenize
 
 # For purposes of monkeypatching inspect to fix a bug in it.
 from inspect import getsourcefile, getfile, getmodule, \
     ismodule, isclass, ismethod, isfunction, istraceback, isframe, iscode
 
 # IPython's own modules
 from IPython import get_ipython
 from IPython.core import debugger
 from IPython.core.display_trap import DisplayTrap
 from IPython.core.excolors import exception_colors
 from IPython.utils import PyColorize
 from IPython.utils import openpy
 from IPython.utils import path as util_path
 from IPython.utils import py3compat
 from IPython.utils import ulinecache
 from IPython.utils.data import uniq_stable
 from IPython.utils.terminal import get_terminal_size
 from logging import info, error
 
 import IPython.utils.colorable as colorable
 
 # Globals
 # amount of space to put line numbers before verbose tracebacks
 INDENT_SIZE = 8
 
 # Default color scheme.  This is used, for example, by the traceback
 # formatter.  When running in an actual IPython instance, the user's rc.colors
 # value is used, but having a module global makes this functionality available
 # to users of ultratb who are NOT running inside ipython.
 DEFAULT_SCHEME = 'NoColor'
 
 # ---------------------------------------------------------------------------
 # Code begins
 
 # Utility functions
 def inspect_error():
     """Print a message about internal inspect errors.
 
     These are unfortunately quite common."""
 
     error('Internal Python error in the inspect module.\n'
           'Below is the traceback from this internal error.\n')
 
 
 # This function is a monkeypatch we apply to the Python inspect module. We have
 # now found when it's needed (see discussion on issue gh-1456), and we have a
 # test case (IPython.core.tests.test_ultratb.ChangedPyFileTest) that fails if
 # the monkeypatch is not applied. TK, Aug 2012.
 def findsource(object):
     """Return the entire source file and starting line number for an object.
 
     The argument may be a module, class, method, function, traceback, frame,
     or code object.  The source code is returned as a list of all the lines
     in the file and the line number indexes a line in that list.  An IOError
     is raised if the source code cannot be retrieved.
 
     FIXED version with which we monkeypatch the stdlib to work around a bug."""
 
     file = getsourcefile(object) or getfile(object)
     # If the object is a frame, then trying to get the globals dict from its
     # module won't work. Instead, the frame object itself has the globals
     # dictionary.
     globals_dict = None
     if inspect.isframe(object):
         # XXX: can this ever be false?
         globals_dict = object.f_globals
     else:
         module = getmodule(object, file)
         if module:
             globals_dict = module.__dict__
     lines = linecache.getlines(file, globals_dict)
     if not lines:
         raise IOError('could not get source code')
 
     if ismodule(object):
         return lines, 0
 
     if isclass(object):
         name = object.__name__
         pat = re.compile(r'^(\s*)class\s*' + name + r'\b')
         # make some effort to find the best matching class definition:
         # use the one with the least indentation, which is the one
         # that's most probably not inside a function definition.
         candidates = []
         for i in range(len(lines)):
             match = pat.match(lines[i])
             if match:
                 # if it's at toplevel, it's already the best one
                 if lines[i][0] == 'c':
                     return lines, i
                 # else add whitespace to candidate list
                 candidates.append((match.group(1), i))
         if candidates:
             # this will sort by whitespace, and by line number,
             # less whitespace first
             candidates.sort()
             return lines, candidates[0][1]
         else:
             raise IOError('could not find class definition')
 
     if ismethod(object):
         object = object.__func__
     if isfunction(object):
         object = object.__code__
     if istraceback(object):
         object = object.tb_frame
     if isframe(object):
         object = object.f_code
     if iscode(object):
         if not hasattr(object, 'co_firstlineno'):
             raise IOError('could not find function definition')
         pat = re.compile(r'^(\s*def\s)|(.*(?<!\w)lambda(:|\s))|^(\s*@)')
         pmatch = pat.match
         # fperez - fix: sometimes, co_firstlineno can give a number larger than
         # the length of lines, which causes an error.  Safeguard against that.
         lnum = min(object.co_firstlineno, len(lines)) - 1
         while lnum > 0:
             if pmatch(lines[lnum]):
                 break
             lnum -= 1
 
         return lines, lnum
     raise IOError('could not find code object')
 
 
 # This is a patched version of inspect.getargs that applies the (unmerged)
 # patch for http://bugs.python.org/issue14611 by Stefano Taschini.  This fixes
 # https://github.com/ipython/ipython/issues/8205 and
 # https://github.com/ipython/ipython/issues/8293
 def getargs(co):
     """Get information about the arguments accepted by a code object.
 
     Three things are returned: (args, varargs, varkw), where 'args' is
     a list of argument names (possibly containing nested lists), and
     'varargs' and 'varkw' are the names of the * and ** arguments or None."""
     if not iscode(co):
         raise TypeError('{!r} is not a code object'.format(co))
 
     nargs = co.co_argcount
     names = co.co_varnames
     args = list(names[:nargs])
     step = 0
 
     # The following acrobatics are for anonymous (tuple) arguments.
     for i in range(nargs):
         if args[i][:1] in ('', '.'):
             stack, remain, count = [], [], []
             while step < len(co.co_code):
                 op = ord(co.co_code[step])
                 step = step + 1
                 if op >= dis.HAVE_ARGUMENT:
                     opname = dis.opname[op]
                     value = ord(co.co_code[step]) + ord(co.co_code[step+1])*256
                     step = step + 2
                     if opname in ('UNPACK_TUPLE', 'UNPACK_SEQUENCE'):
                         remain.append(value)
                         count.append(value)
                     elif opname in ('STORE_FAST', 'STORE_DEREF'):
                         if op in dis.haslocal:
                             stack.append(co.co_varnames[value])
                         elif op in dis.hasfree:
                             stack.append((co.co_cellvars + co.co_freevars)[value])
                         # Special case for sublists of length 1: def foo((bar))
                         # doesn't generate the UNPACK_TUPLE bytecode, so if
                         # `remain` is empty here, we have such a sublist.
                         if not remain:
                             stack[0] = [stack[0]]
                             break
                         else:
                             remain[-1] = remain[-1] - 1
                             while remain[-1] == 0:
                                 remain.pop()
                                 size = count.pop()
                                 stack[-size:] = [stack[-size:]]
                                 if not remain:
                                     break
                                 remain[-1] = remain[-1] - 1
                             if not remain:
                                 break
             args[i] = stack[0]
 
     varargs = None
     if co.co_flags & inspect.CO_VARARGS:
         varargs = co.co_varnames[nargs]
         nargs = nargs + 1
     varkw = None
     if co.co_flags & inspect.CO_VARKEYWORDS:
         varkw = co.co_varnames[nargs]
     return inspect.Arguments(args, varargs, varkw)
 
 
 # Monkeypatch inspect to apply our bugfix.
 def with_patch_inspect(f):
-    """decorator for monkeypatching inspect.findsource"""
+    """
+    Deprecated since IPython 6.0
+    decorator for monkeypatching inspect.findsource
+    """
 
     def wrapped(*args, **kwargs):
         save_findsource = inspect.findsource
         save_getargs = inspect.getargs
         inspect.findsource = findsource
         inspect.getargs = getargs
         try:
             return f(*args, **kwargs)
         finally:
             inspect.findsource = save_findsource
             inspect.getargs = save_getargs
 
     return wrapped
 
 
-if py3compat.PY3:
-    fixed_getargvalues = inspect.getargvalues
-else:
-    # Fixes for https://github.com/ipython/ipython/issues/8293
-    #       and https://github.com/ipython/ipython/issues/8205.
-    # The relevant bug is caused by failure to correctly handle anonymous tuple
-    # unpacking, which only exists in Python 2.
-    fixed_getargvalues = with_patch_inspect(inspect.getargvalues)
-
-
 def fix_frame_records_filenames(records):
     """Try to fix the filenames in each record from inspect.getinnerframes().
 
     Particularly, modules loaded from within zip files have useless filenames
     attached to their code object, and inspect.getinnerframes() just uses it.
     """
     fixed_records = []
     for frame, filename, line_no, func_name, lines, index in records:
         # Look inside the frame's globals dictionary for __file__,
         # which should be better. However, keep Cython filenames since
         # we prefer the source filenames over the compiled .so file.
         filename = py3compat.cast_unicode_py2(filename, "utf-8")
         if not filename.endswith(('.pyx', '.pxd', '.pxi')):
             better_fn = frame.f_globals.get('__file__', None)
             if isinstance(better_fn, str):
                 # Check the type just in case someone did something weird with
                 # __file__. It might also be None if the error occurred during
                 # import.
                 filename = better_fn
         fixed_records.append((frame, filename, line_no, func_name, lines, index))
     return fixed_records
 
 
 @with_patch_inspect
 def _fixed_getinnerframes(etb, context=1, tb_offset=0):
     LNUM_POS, LINES_POS, INDEX_POS = 2, 4, 5
 
     records = fix_frame_records_filenames(inspect.getinnerframes(etb, context))
     # If the error is at the console, don't build any context, since it would
     # otherwise produce 5 blank lines printed out (there is no file at the
     # console)
     rec_check = records[tb_offset:]
     try:
         rname = rec_check[0][1]
         if rname == '<ipython console>' or rname.endswith('<string>'):
             return rec_check
     except IndexError:
         pass
 
     aux = traceback.extract_tb(etb)
     assert len(records) == len(aux)
     for i, (file, lnum, _, _) in zip(range(len(records)), aux):
         maybeStart = lnum - 1 - context // 2
         start = max(maybeStart, 0)
         end = start + context
         lines = ulinecache.getlines(file)[start:end]
         buf = list(records[i])
         buf[LNUM_POS] = lnum
         buf[INDEX_POS] = lnum - 1 - start
         buf[LINES_POS] = lines
         records[i] = tuple(buf)
     return records[tb_offset:]
 
 # Helper function -- largely belongs to VerboseTB, but we need the same
 # functionality to produce a pseudo verbose TB for SyntaxErrors, so that they
 # can be recognized properly by ipython.el's py-traceback-line-re
 # (SyntaxErrors have to be treated specially because they have no traceback)
 
 
 def _format_traceback_lines(lnum, index, lines, Colors, lvals=None,  _line_format=(lambda x,_:x,None)):
     numbers_width = INDENT_SIZE - 1
     res = []
     i = lnum - index
 
     for line in lines:
         line = py3compat.cast_unicode(line)
 
         new_line, err = _line_format(line, 'str')
         if not err: line = new_line
 
         if i == lnum:
             # This is the line with the error
             pad = numbers_width - len(str(i))
             num = '%s%s' % (debugger.make_arrow(pad), str(lnum))
             line = '%s%s%s %s%s' % (Colors.linenoEm, num,
                                     Colors.line, line, Colors.Normal)
         else:
             num = '%*s' % (numbers_width, i)
             line = '%s%s%s %s' % (Colors.lineno, num,
                                   Colors.Normal, line)
 
         res.append(line)
         if lvals and i == lnum:
             res.append(lvals + '\n')
         i = i + 1
     return res
 
 def is_recursion_error(etype, value, records):
     try:
         # RecursionError is new in Python 3.5
         recursion_error_type = RecursionError
     except NameError:
         recursion_error_type = RuntimeError
 
     # The default recursion limit is 1000, but some of that will be taken up
     # by stack frames in IPython itself. >500 frames probably indicates
     # a recursion error.
     return (etype is recursion_error_type) \
            and "recursion" in str(value).lower() \
            and len(records) > 500
 
 def find_recursion(etype, value, records):
     """Identify the repeating stack frames from a RecursionError traceback
 
     'records' is a list as returned by VerboseTB.get_records()
 
     Returns (last_unique, repeat_length)
     """
     # This involves a bit of guesswork - we want to show enough of the traceback
     # to indicate where the recursion is occurring. We guess that the innermost
     # quarter of the traceback (250 frames by default) is repeats, and find the
     # first frame (from in to out) that looks different.
     if not is_recursion_error(etype, value, records):
         return len(records), 0
 
     # Select filename, lineno, func_name to track frames with
     records = [r[1:4] for r in records]
     inner_frames = records[-(len(records)//4):]
     frames_repeated = set(inner_frames)
 
     last_seen_at = {}
     longest_repeat = 0
     i = len(records)
     for frame in reversed(records):
         i -= 1
         if frame not in frames_repeated:
             last_unique = i
             break
 
         if frame in last_seen_at:
             distance = last_seen_at[frame] - i
             longest_repeat = max(longest_repeat, distance)
 
         last_seen_at[frame] = i
     else:
         last_unique = 0 # The whole traceback was recursion
 
     return last_unique, longest_repeat
 
 #---------------------------------------------------------------------------
 # Module classes
 class TBTools(colorable.Colorable):
     """Basic tools used by all traceback printer classes."""
 
     # Number of frames to skip when reporting tracebacks
     tb_offset = 0
 
     def __init__(self, color_scheme='NoColor', call_pdb=False, ostream=None, parent=None, config=None):
         # Whether to call the interactive pdb debugger after printing
         # tracebacks or not
         super(TBTools, self).__init__(parent=parent, config=config)
         self.call_pdb = call_pdb
 
         # Output stream to write to.  Note that we store the original value in
         # a private attribute and then make the public ostream a property, so
         # that we can delay accessing sys.stdout until runtime.  The way
         # things are written now, the sys.stdout object is dynamically managed
         # so a reference to it should NEVER be stored statically.  This
         # property approach confines this detail to a single location, and all
         # subclasses can simply access self.ostream for writing.
         self._ostream = ostream
 
         # Create color table
         self.color_scheme_table = exception_colors()
 
         self.set_colors(color_scheme)
         self.old_scheme = color_scheme  # save initial value for toggles
 
         if call_pdb:
             self.pdb = debugger.Pdb()
         else:
             self.pdb = None
 
     def _get_ostream(self):
         """Output stream that exceptions are written to.
 
         Valid values are:
 
         - None: the default, which means that IPython will dynamically resolve
           to sys.stdout.  This ensures compatibility with most tools, including
           Windows (where plain stdout doesn't recognize ANSI escapes).
 
         - Any object with 'write' and 'flush' attributes.
         """
         return sys.stdout if self._ostream is None else self._ostream
 
     def _set_ostream(self, val):
         assert val is None or (hasattr(val, 'write') and hasattr(val, 'flush'))
         self._ostream = val
 
     ostream = property(_get_ostream, _set_ostream)
 
     def set_colors(self, *args, **kw):
         """Shorthand access to the color table scheme selector method."""
 
         # Set own color table
         self.color_scheme_table.set_active_scheme(*args, **kw)
         # for convenience, set Colors to the active scheme
         self.Colors = self.color_scheme_table.active_colors
         # Also set colors of debugger
         if hasattr(self, 'pdb') and self.pdb is not None:
             self.pdb.set_colors(*args, **kw)
 
     def color_toggle(self):
         """Toggle between the currently active color scheme and NoColor."""
 
         if self.color_scheme_table.active_scheme_name == 'NoColor':
             self.color_scheme_table.set_active_scheme(self.old_scheme)
             self.Colors = self.color_scheme_table.active_colors
         else:
             self.old_scheme = self.color_scheme_table.active_scheme_name
             self.color_scheme_table.set_active_scheme('NoColor')
             self.Colors = self.color_scheme_table.active_colors
 
     def stb2text(self, stb):
         """Convert a structured traceback (a list) to a string."""
         return '\n'.join(stb)
 
     def text(self, etype, value, tb, tb_offset=None, context=5):
         """Return formatted traceback.
 
         Subclasses may override this if they add extra arguments.
         """
         tb_list = self.structured_traceback(etype, value, tb,
                                             tb_offset, context)
         return self.stb2text(tb_list)
 
     def structured_traceback(self, etype, evalue, tb, tb_offset=None,
                              context=5, mode=None):
         """Return a list of traceback frames.
 
         Must be implemented by each class.
         """
         raise NotImplementedError()
 
 
 #---------------------------------------------------------------------------
 class ListTB(TBTools):
     """Print traceback information from a traceback list, with optional color.
 
     Calling requires 3 arguments: (etype, evalue, elist)
     as would be obtained by::
     
       etype, evalue, tb = sys.exc_info()
       if tb:
         elist = traceback.extract_tb(tb)
       else:
         elist = None
 
     It can thus be used by programs which need to process the traceback before
     printing (such as console replacements based on the code module from the
     standard library).
 
     Because they are meant to be called without a full traceback (only a
     list), instances of this class can't call the interactive pdb debugger."""
 
     def __init__(self, color_scheme='NoColor', call_pdb=False, ostream=None, parent=None, config=None):
         TBTools.__init__(self, color_scheme=color_scheme, call_pdb=call_pdb,
                          ostream=ostream, parent=parent,config=config)
 
     def __call__(self, etype, value, elist):
         self.ostream.flush()
         self.ostream.write(self.text(etype, value, elist))
         self.ostream.write('\n')
 
     def structured_traceback(self, etype, value, elist, tb_offset=None,
                              context=5):
         """Return a color formatted string with the traceback info.
 
         Parameters
         ----------
         etype : exception type
           Type of the exception raised.
 
         value : object
           Data stored in the exception
 
         elist : list
           List of frames, see class docstring for details.
 
         tb_offset : int, optional
           Number of frames in the traceback to skip.  If not given, the
           instance value is used (set in constructor).
 
         context : int, optional
           Number of lines of context information to print.
 
         Returns
         -------
         String with formatted exception.
         """
         tb_offset = self.tb_offset if tb_offset is None else tb_offset
         Colors = self.Colors
         out_list = []
         if elist:
 
             if tb_offset and len(elist) > tb_offset:
                 elist = elist[tb_offset:]
 
             out_list.append('Traceback %s(most recent call last)%s:' %
                             (Colors.normalEm, Colors.Normal) + '\n')
             out_list.extend(self._format_list(elist))
         # The exception info should be a single entry in the list.
         lines = ''.join(self._format_exception_only(etype, value))
         out_list.append(lines)
 
         # Note: this code originally read:
 
         ## for line in lines[:-1]:
         ##     out_list.append(" "+line)
         ## out_list.append(lines[-1])
 
         # This means it was indenting everything but the last line by a little
         # bit.  I've disabled this for now, but if we see ugliness somewhere we
         # can restore it.
 
         return out_list
 
     def _format_list(self, extracted_list):
         """Format a list of traceback entry tuples for printing.
 
         Given a list of tuples as returned by extract_tb() or
         extract_stack(), return a list of strings ready for printing.
         Each string in the resulting list corresponds to the item with the
         same index in the argument list.  Each string ends in a newline;
         the strings may contain internal newlines as well, for those items
         whose source text line is not None.
 
         Lifted almost verbatim from traceback.py
         """
 
         Colors = self.Colors
         list = []
         for filename, lineno, name, line in extracted_list[:-1]:
             item = '  File %s"%s"%s, line %s%d%s, in %s%s%s\n' % \
                    (Colors.filename, py3compat.cast_unicode_py2(filename, "utf-8"), Colors.Normal,
                     Colors.lineno, lineno, Colors.Normal,
                     Colors.name, py3compat.cast_unicode_py2(name, "utf-8"), Colors.Normal)
             if line:
                 item += '    %s\n' % line.strip()
             list.append(item)
         # Emphasize the last entry
         filename, lineno, name, line = extracted_list[-1]
         item = '%s  File %s"%s"%s, line %s%d%s, in %s%s%s%s\n' % \
                (Colors.normalEm,
                 Colors.filenameEm, py3compat.cast_unicode_py2(filename, "utf-8"), Colors.normalEm,
                 Colors.linenoEm, lineno, Colors.normalEm,
                 Colors.nameEm, py3compat.cast_unicode_py2(name, "utf-8"), Colors.normalEm,
                 Colors.Normal)
         if line:
             item += '%s    %s%s\n' % (Colors.line, line.strip(),
                                       Colors.Normal)
         list.append(item)
         return list
 
     def _format_exception_only(self, etype, value):
         """Format the exception part of a traceback.
 
         The arguments are the exception type and value such as given by
         sys.exc_info()[:2]. The return value is a list of strings, each ending
         in a newline.  Normally, the list contains a single string; however,
         for SyntaxError exceptions, it contains several lines that (when
         printed) display detailed information about where the syntax error
         occurred.  The message indicating which exception occurred is the
         always last string in the list.
 
         Also lifted nearly verbatim from traceback.py
         """
         have_filedata = False
         Colors = self.Colors
         list = []
         stype = py3compat.cast_unicode(Colors.excName + etype.__name__ + Colors.Normal)
         if value is None:
             # Not sure if this can still happen in Python 2.6 and above
             list.append(stype + '\n')
         else:
             if issubclass(etype, SyntaxError):
                 have_filedata = True
                 if not value.filename: value.filename = "<string>"
                 if value.lineno:
                     lineno = value.lineno
                     textline = ulinecache.getline(value.filename, value.lineno)
                 else:
                     lineno = 'unknown'
                     textline = ''
                 list.append('%s  File %s"%s"%s, line %s%s%s\n' % \
                             (Colors.normalEm,
                              Colors.filenameEm, py3compat.cast_unicode(value.filename), Colors.normalEm,
                              Colors.linenoEm, lineno, Colors.Normal  ))
                 if textline == '':
                     textline = py3compat.cast_unicode(value.text, "utf-8")
 
                 if textline is not None:
                     i = 0
                     while i < len(textline) and textline[i].isspace():
                         i += 1
                     list.append('%s    %s%s\n' % (Colors.line,
                                                   textline.strip(),
                                                   Colors.Normal))
                     if value.offset is not None:
                         s = '    '
                         for c in textline[i:value.offset - 1]:
                             if c.isspace():
                                 s += c
                             else:
                                 s += ' '
                         list.append('%s%s^%s\n' % (Colors.caret, s,
                                                    Colors.Normal))
 
             try:
                 s = value.msg
             except Exception:
                 s = self._some_str(value)
             if s:
                 list.append('%s%s:%s %s\n' % (stype, Colors.excName,
                                               Colors.Normal, s))
             else:
                 list.append('%s\n' % stype)
 
         # sync with user hooks
         if have_filedata:
             ipinst = get_ipython()
             if ipinst is not None:
                 ipinst.hooks.synchronize_with_editor(value.filename, value.lineno, 0)
 
         return list
 
     def get_exception_only(self, etype, value):
         """Only print the exception type and message, without a traceback.
 
         Parameters
         ----------
         etype : exception type
         value : exception value
         """
         return ListTB.structured_traceback(self, etype, value, [])
 
     def show_exception_only(self, etype, evalue):
         """Only print the exception type and message, without a traceback.
 
         Parameters
         ----------
         etype : exception type
         value : exception value
         """
         # This method needs to use __call__ from *this* class, not the one from
         # a subclass whose signature or behavior may be different
         ostream = self.ostream
         ostream.flush()
         ostream.write('\n'.join(self.get_exception_only(etype, evalue)))
         ostream.flush()
 
     def _some_str(self, value):
         # Lifted from traceback.py
         try:
             return py3compat.cast_unicode(str(value))
         except:
             return u'<unprintable %s object>' % type(value).__name__
 
 
 #----------------------------------------------------------------------------
 class VerboseTB(TBTools):
     """A port of Ka-Ping Yee's cgitb.py module that outputs color text instead
     of HTML.  Requires inspect and pydoc.  Crazy, man.
 
     Modified version which optionally strips the topmost entries from the
     traceback, to be used with alternate interpreters (because their own code
     would appear in the traceback)."""
 
     def __init__(self, color_scheme='Linux', call_pdb=False, ostream=None,
                  tb_offset=0, long_header=False, include_vars=True,
                  check_cache=None, debugger_cls = None,
                  parent=None, config=None):
         """Specify traceback offset, headers and color scheme.
 
         Define how many frames to drop from the tracebacks. Calling it with
         tb_offset=1 allows use of this handler in interpreters which will have
         their own code at the top of the traceback (VerboseTB will first
         remove that frame before printing the traceback info)."""
         TBTools.__init__(self, color_scheme=color_scheme, call_pdb=call_pdb,
                          ostream=ostream, parent=parent, config=config)
         self.tb_offset = tb_offset
         self.long_header = long_header
         self.include_vars = include_vars
         # By default we use linecache.checkcache, but the user can provide a
         # different check_cache implementation.  This is used by the IPython
         # kernel to provide tracebacks for interactive code that is cached,
         # by a compiler instance that flushes the linecache but preserves its
         # own code cache.
         if check_cache is None:
             check_cache = linecache.checkcache
         self.check_cache = check_cache
 
         self.debugger_cls = debugger_cls or debugger.Pdb
 
     def format_records(self, records, last_unique, recursion_repeat):
         """Format the stack frames of the traceback"""
         frames = []
         for r in records[:last_unique+recursion_repeat+1]:
             #print '*** record:',file,lnum,func,lines,index  # dbg
             frames.append(self.format_record(*r))
 
         if recursion_repeat:
             frames.append('... last %d frames repeated, from the frame below ...\n' % recursion_repeat)
             frames.append(self.format_record(*records[last_unique+recursion_repeat+1]))
 
         return frames
 
     def format_record(self, frame, file, lnum, func, lines, index):
         """Format a single stack frame"""
         Colors = self.Colors  # just a shorthand + quicker name lookup
         ColorsNormal = Colors.Normal  # used a lot
         col_scheme = self.color_scheme_table.active_scheme_name
         indent = ' ' * INDENT_SIZE
         em_normal = '%s\n%s%s' % (Colors.valEm, indent, ColorsNormal)
         undefined = '%sundefined%s' % (Colors.em, ColorsNormal)
         tpl_link = '%s%%s%s' % (Colors.filenameEm, ColorsNormal)
         tpl_call = 'in %s%%s%s%%s%s' % (Colors.vName, Colors.valEm,
                                         ColorsNormal)
         tpl_call_fail = 'in %s%%s%s(***failed resolving arguments***)%s' % \
                         (Colors.vName, Colors.valEm, ColorsNormal)
         tpl_local_var = '%s%%s%s' % (Colors.vName, ColorsNormal)
         tpl_global_var = '%sglobal%s %s%%s%s' % (Colors.em, ColorsNormal,
                                                  Colors.vName, ColorsNormal)
         tpl_name_val = '%%s %s= %%s%s' % (Colors.valEm, ColorsNormal)
 
         tpl_line = '%s%%s%s %%s' % (Colors.lineno, ColorsNormal)
         tpl_line_em = '%s%%s%s %%s%s' % (Colors.linenoEm, Colors.line,
                                          ColorsNormal)
 
         abspath = os.path.abspath
 
 
         if not file:
             file = '?'
         elif file.startswith(str("<")) and file.endswith(str(">")):
             # Not a real filename, no problem...
             pass
         elif not os.path.isabs(file):
             # Try to make the filename absolute by trying all
             # sys.path entries (which is also what linecache does)
             for dirname in sys.path:
                 try:
                     fullname = os.path.join(dirname, file)
                     if os.path.isfile(fullname):
                         file = os.path.abspath(fullname)
                         break
                 except Exception:
                     # Just in case that sys.path contains very
                     # strange entries...
                     pass
 
         file = py3compat.cast_unicode(file, util_path.fs_encoding)
         link = tpl_link % file
-        args, varargs, varkw, locals = fixed_getargvalues(frame)
+        args, varargs, varkw, locals = inspect.getargvalues(frame)
 
         if func == '?':
             call = ''
         else:
             # Decide whether to include variable details or not
             var_repr = self.include_vars and eqrepr or nullrepr
             try:
                 call = tpl_call % (func, inspect.formatargvalues(args,
                                                                  varargs, varkw,
                                                                  locals, formatvalue=var_repr))
             except KeyError:
                 # This happens in situations like errors inside generator
                 # expressions, where local variables are listed in the
                 # line, but can't be extracted from the frame.  I'm not
                 # 100% sure this isn't actually a bug in inspect itself,
                 # but since there's no info for us to compute with, the
                 # best we can do is report the failure and move on.  Here
                 # we must *not* call any traceback construction again,
                 # because that would mess up use of %debug later on.  So we
                 # simply report the failure and move on.  The only
                 # limitation will be that this frame won't have locals
                 # listed in the call signature.  Quite subtle problem...
                 # I can't think of a good way to validate this in a unit
                 # test, but running a script consisting of:
                 #  dict( (k,v.strip()) for (k,v) in range(10) )
                 # will illustrate the error, if this exception catch is
                 # disabled.
                 call = tpl_call_fail % func
 
         # Don't attempt to tokenize binary files.
         if file.endswith(('.so', '.pyd', '.dll')):
             return '%s %s\n' % (link, call)
 
         elif file.endswith(('.pyc', '.pyo')):
             # Look up the corresponding source file.
             try:
                 file = openpy.source_from_cache(file)
             except ValueError:
                 # Failed to get the source file for some reason
                 # E.g. https://github.com/ipython/ipython/issues/9486
                 return '%s %s\n' % (link, call)
 
         def linereader(file=file, lnum=[lnum], getline=ulinecache.getline):
             line = getline(file, lnum[0])
             lnum[0] += 1
             return line
 
         # Build the list of names on this line of code where the exception
         # occurred.
         try:
             names = []
             name_cont = False
 
             for token_type, token, start, end, line in generate_tokens(linereader):
                 # build composite names
                 if token_type == tokenize.NAME and token not in keyword.kwlist:
                     if name_cont:
                         # Continuation of a dotted name
                         try:
                             names[-1].append(token)
                         except IndexError:
                             names.append([token])
                         name_cont = False
                     else:
                         # Regular new names.  We append everything, the caller
                         # will be responsible for pruning the list later.  It's
                         # very tricky to try to prune as we go, b/c composite
                         # names can fool us.  The pruning at the end is easy
                         # to do (or the caller can print a list with repeated
                         # names if so desired.
                         names.append([token])
                 elif token == '.':
                     name_cont = True
                 elif token_type == tokenize.NEWLINE:
                     break
 
         except (IndexError, UnicodeDecodeError, SyntaxError):
             # signals exit of tokenizer
             # SyntaxError can occur if the file is not actually Python
             #  - see gh-6300
             pass
         except tokenize.TokenError as msg:
             _m = ("An unexpected error occurred while tokenizing input\n"
                   "The following traceback may be corrupted or invalid\n"
                   "The error message is: %s\n" % msg)
             error(_m)
 
         # Join composite names (e.g. "dict.fromkeys")
         names = ['.'.join(n) for n in names]
         # prune names list of duplicates, but keep the right order
         unique_names = uniq_stable(names)
 
         # Start loop over vars
         lvals = []
         if self.include_vars:
             for name_full in unique_names:
                 name_base = name_full.split('.', 1)[0]
                 if name_base in frame.f_code.co_varnames:
                     if name_base in locals:
                         try:
                             value = repr(eval(name_full, locals))
                         except:
                             value = undefined
                     else:
                         value = undefined
                     name = tpl_local_var % name_full
                 else:
                     if name_base in frame.f_globals:
                         try:
                             value = repr(eval(name_full, frame.f_globals))
                         except:
                             value = undefined
                     else:
                         value = undefined
                     name = tpl_global_var % name_full
                 lvals.append(tpl_name_val % (name, value))
         if lvals:
             lvals = '%s%s' % (indent, em_normal.join(lvals))
         else:
             lvals = ''
 
         level = '%s %s\n' % (link, call)
 
         if index is None:
             return level
         else:
             _line_format = PyColorize.Parser(style=col_scheme, parent=self).format2
             return '%s%s' % (level, ''.join(
                 _format_traceback_lines(lnum, index, lines, Colors, lvals,
                                          _line_format)))
 
     def prepare_chained_exception_message(self, cause):
         direct_cause = "\nThe above exception was the direct cause of the following exception:\n"
         exception_during_handling = "\nDuring handling of the above exception, another exception occurred:\n"
 
         if cause:
             message = [[direct_cause]]
         else:
             message = [[exception_during_handling]]
         return message
 
     def prepare_header(self, etype, long_version=False):
         colors = self.Colors  # just a shorthand + quicker name lookup
         colorsnormal = colors.Normal  # used a lot
         exc = '%s%s%s' % (colors.excName, etype, colorsnormal)
         width = min(75, get_terminal_size()[0])
         if long_version:
             # Header with the exception type, python version, and date
             pyver = 'Python ' + sys.version.split()[0] + ': ' + sys.executable
             date = time.ctime(time.time())
 
             head = '%s%s%s\n%s%s%s\n%s' % (colors.topline, '-' * width, colorsnormal,
                                            exc, ' ' * (width - len(str(etype)) - len(pyver)),
                                            pyver, date.rjust(width) )
             head += "\nA problem occurred executing Python code.  Here is the sequence of function" \
                     "\ncalls leading up to the error, with the most recent (innermost) call last."
         else:
             # Simplified header
             head = '%s%s' % (exc, 'Traceback (most recent call last)'. \
                              rjust(width - len(str(etype))) )
 
         return head
 
     def format_exception(self, etype, evalue):
         colors = self.Colors  # just a shorthand + quicker name lookup
         colorsnormal = colors.Normal  # used a lot
         indent = ' ' * INDENT_SIZE
         # Get (safely) a string form of the exception info
         try:
             etype_str, evalue_str = map(str, (etype, evalue))
         except:
             # User exception is improperly defined.
             etype, evalue = str, sys.exc_info()[:2]
             etype_str, evalue_str = map(str, (etype, evalue))
         # ... and format it
         return ['%s%s%s: %s' % (colors.excName, etype_str,
                                 colorsnormal, py3compat.cast_unicode(evalue_str))]
 
     def format_exception_as_a_whole(self, etype, evalue, etb, number_of_lines_of_context, tb_offset):
         """Formats the header, traceback and exception message for a single exception.
 
         This may be called multiple times by Python 3 exception chaining
         (PEP 3134).
         """
         # some locals
         orig_etype = etype
         try:
             etype = etype.__name__
         except AttributeError:
             pass
 
         tb_offset = self.tb_offset if tb_offset is None else tb_offset
         head = self.prepare_header(etype, self.long_header)
         records = self.get_records(etb, number_of_lines_of_context, tb_offset)
 
         if records is None:
             return ""
 
         last_unique, recursion_repeat = find_recursion(orig_etype, evalue, records)
 
         frames = self.format_records(records, last_unique, recursion_repeat)
 
         formatted_exception = self.format_exception(etype, evalue)
         if records:
             filepath, lnum = records[-1][1:3]
             filepath = os.path.abspath(filepath)
             ipinst = get_ipython()
             if ipinst is not None:
                 ipinst.hooks.synchronize_with_editor(filepath, lnum, 0)
 
         return [[head] + frames + [''.join(formatted_exception[0])]]
 
     def get_records(self, etb, number_of_lines_of_context, tb_offset):
         try:
             # Try the default getinnerframes and Alex's: Alex's fixes some
             # problems, but it generates empty tracebacks for console errors
             # (5 blanks lines) where none should be returned.
             return _fixed_getinnerframes(etb, number_of_lines_of_context, tb_offset)
         except UnicodeDecodeError:
             # This can occur if a file's encoding magic comment is wrong.
             # I can't see a way to recover without duplicating a bunch of code
             # from the stdlib traceback module. --TK
             error('\nUnicodeDecodeError while processing traceback.\n')
             return None
         except:
             # FIXME: I've been getting many crash reports from python 2.3
             # users, traceable to inspect.py.  If I can find a small test-case
             # to reproduce this, I should either write a better workaround or
             # file a bug report against inspect (if that's the real problem).
             # So far, I haven't been able to find an isolated example to
             # reproduce the problem.
             inspect_error()
             traceback.print_exc(file=self.ostream)
             info('\nUnfortunately, your original traceback can not be constructed.\n')
             return None
 
     def get_parts_of_chained_exception(self, evalue):
         def get_chained_exception(exception_value):
             cause = getattr(exception_value, '__cause__', None)
             if cause:
                 return cause
             if getattr(exception_value, '__suppress_context__', False):
                 return None
             return getattr(exception_value, '__context__', None)
 
         chained_evalue = get_chained_exception(evalue)
 
         if chained_evalue:
             return chained_evalue.__class__, chained_evalue, chained_evalue.__traceback__
 
     def structured_traceback(self, etype, evalue, etb, tb_offset=None,
                              number_of_lines_of_context=5):
         """Return a nice text document describing the traceback."""
 
         formatted_exception = self.format_exception_as_a_whole(etype, evalue, etb, number_of_lines_of_context,
                                                                tb_offset)
 
         colors = self.Colors  # just a shorthand + quicker name lookup
         colorsnormal = colors.Normal  # used a lot
         head = '%s%s%s' % (colors.topline, '-' * min(75, get_terminal_size()[0]), colorsnormal)
         structured_traceback_parts = [head]
         if py3compat.PY3:
             chained_exceptions_tb_offset = 0
             lines_of_context = 3
             formatted_exceptions = formatted_exception
             exception = self.get_parts_of_chained_exception(evalue)
             if exception:
                 formatted_exceptions += self.prepare_chained_exception_message(evalue.__cause__)
                 etype, evalue, etb = exception
             else:
                 evalue = None
             chained_exc_ids = set()
             while evalue:
                 formatted_exceptions += self.format_exception_as_a_whole(etype, evalue, etb, lines_of_context,
                                                                          chained_exceptions_tb_offset)
                 exception = self.get_parts_of_chained_exception(evalue)
 
                 if exception and not id(exception[1]) in chained_exc_ids:
                     chained_exc_ids.add(id(exception[1])) # trace exception to avoid infinite 'cause' loop
                     formatted_exceptions += self.prepare_chained_exception_message(evalue.__cause__)
                     etype, evalue, etb = exception
                 else:
                     evalue = None
 
             # we want to see exceptions in a reversed order:
             # the first exception should be on top
             for formatted_exception in reversed(formatted_exceptions):
                 structured_traceback_parts += formatted_exception
         else:
             structured_traceback_parts += formatted_exception[0]
 
         return structured_traceback_parts
 
     def debugger(self, force=False):
         """Call up the pdb debugger if desired, always clean up the tb
         reference.
 
         Keywords:
 
           - force(False): by default, this routine checks the instance call_pdb
             flag and does not actually invoke the debugger if the flag is false.
             The 'force' option forces the debugger to activate even if the flag
             is false.
 
         If the call_pdb flag is set, the pdb interactive debugger is
         invoked. In all cases, the self.tb reference to the current traceback
         is deleted to prevent lingering references which hamper memory
         management.
 
         Note that each call to pdb() does an 'import readline', so if your app
         requires a special setup for the readline completers, you'll have to
         fix that by hand after invoking the exception handler."""
 
         if force or self.call_pdb:
             if self.pdb is None:
                 self.pdb = self.debugger_cls()
             # the system displayhook may have changed, restore the original
             # for pdb
             display_trap = DisplayTrap(hook=sys.__displayhook__)
             with display_trap:
                 self.pdb.reset()
                 # Find the right frame so we don't pop up inside ipython itself
                 if hasattr(self, 'tb') and self.tb is not None:
                     etb = self.tb
                 else:
                     etb = self.tb = sys.last_traceback
                 while self.tb is not None and self.tb.tb_next is not None:
                     self.tb = self.tb.tb_next
                 if etb and etb.tb_next:
                     etb = etb.tb_next
                 self.pdb.botframe = etb.tb_frame
                 self.pdb.interaction(self.tb.tb_frame, self.tb)
 
         if hasattr(self, 'tb'):
             del self.tb
 
     def handler(self, info=None):
         (etype, evalue, etb) = info or sys.exc_info()
         self.tb = etb
         ostream = self.ostream
         ostream.flush()
         ostream.write(self.text(etype, evalue, etb))
         ostream.write('\n')
         ostream.flush()
 
     # Changed so an instance can just be called as VerboseTB_inst() and print
     # out the right info on its own.
     def __call__(self, etype=None, evalue=None, etb=None):
         """This hook can replace sys.excepthook (for Python 2.1 or higher)."""
         if etb is None:
             self.handler()
         else:
             self.handler((etype, evalue, etb))
         try:
             self.debugger()
         except KeyboardInterrupt:
             print("\nKeyboardInterrupt")
 
 
 #----------------------------------------------------------------------------
 class FormattedTB(VerboseTB, ListTB):
     """Subclass ListTB but allow calling with a traceback.
 
     It can thus be used as a sys.excepthook for Python > 2.1.
 
     Also adds 'Context' and 'Verbose' modes, not available in ListTB.
 
     Allows a tb_offset to be specified. This is useful for situations where
     one needs to remove a number of topmost frames from the traceback (such as
     occurs with python programs that themselves execute other python code,
     like Python shells).  """
 
     def __init__(self, mode='Plain', color_scheme='Linux', call_pdb=False,
                  ostream=None,
                  tb_offset=0, long_header=False, include_vars=False,
                  check_cache=None, debugger_cls=None,
                  parent=None, config=None):
 
         # NEVER change the order of this list. Put new modes at the end:
         self.valid_modes = ['Plain', 'Context', 'Verbose']
         self.verbose_modes = self.valid_modes[1:3]
 
         VerboseTB.__init__(self, color_scheme=color_scheme, call_pdb=call_pdb,
                            ostream=ostream, tb_offset=tb_offset,
                            long_header=long_header, include_vars=include_vars,
                            check_cache=check_cache, debugger_cls=debugger_cls,
                            parent=parent, config=config)
 
         # Different types of tracebacks are joined with different separators to
         # form a single string.  They are taken from this dict
         self._join_chars = dict(Plain='', Context='\n', Verbose='\n')
         # set_mode also sets the tb_join_char attribute
         self.set_mode(mode)
 
     def _extract_tb(self, tb):
         if tb:
             return traceback.extract_tb(tb)
         else:
             return None
 
     def structured_traceback(self, etype, value, tb, tb_offset=None, number_of_lines_of_context=5):
         tb_offset = self.tb_offset if tb_offset is None else tb_offset
         mode = self.mode
         if mode in self.verbose_modes:
             # Verbose modes need a full traceback
             return VerboseTB.structured_traceback(
                 self, etype, value, tb, tb_offset, number_of_lines_of_context
             )
         else:
             # We must check the source cache because otherwise we can print
             # out-of-date source code.
             self.check_cache()
             # Now we can extract and format the exception
             elist = self._extract_tb(tb)
             return ListTB.structured_traceback(
                 self, etype, value, elist, tb_offset, number_of_lines_of_context
             )
 
     def stb2text(self, stb):
         """Convert a structured traceback (a list) to a string."""
         return self.tb_join_char.join(stb)
 
 
     def set_mode(self, mode=None):
         """Switch to the desired mode.
 
         If mode is not specified, cycles through the available modes."""
 
         if not mode:
             new_idx = (self.valid_modes.index(self.mode) + 1 ) % \
                       len(self.valid_modes)
             self.mode = self.valid_modes[new_idx]
         elif mode not in self.valid_modes:
             raise ValueError('Unrecognized mode in FormattedTB: <' + mode + '>\n'
                                                                             'Valid modes: ' + str(self.valid_modes))
         else:
             self.mode = mode
         # include variable details only in 'Verbose' mode
         self.include_vars = (self.mode == self.valid_modes[2])
         # Set the join character for generating text tracebacks
         self.tb_join_char = self._join_chars[self.mode]
 
     # some convenient shortcuts
     def plain(self):
         self.set_mode(self.valid_modes[0])
 
     def context(self):
         self.set_mode(self.valid_modes[1])
 
     def verbose(self):
         self.set_mode(self.valid_modes[2])
 
 
 #----------------------------------------------------------------------------
 class AutoFormattedTB(FormattedTB):
     """A traceback printer which can be called on the fly.
 
     It will find out about exceptions by itself.
 
     A brief example::
 
         AutoTB = AutoFormattedTB(mode = 'Verbose',color_scheme='Linux')
         try:
           ...
         except:
           AutoTB()  # or AutoTB(out=logfile) where logfile is an open file object
     """
 
     def __call__(self, etype=None, evalue=None, etb=None,
                  out=None, tb_offset=None):
         """Print out a formatted exception traceback.
 
         Optional arguments:
           - out: an open file-like object to direct output to.
 
           - tb_offset: the number of frames to skip over in the stack, on a
           per-call basis (this overrides temporarily the instance's tb_offset
           given at initialization time.  """
 
         if out is None:
             out = self.ostream
         out.flush()
         out.write(self.text(etype, evalue, etb, tb_offset))
         out.write('\n')
         out.flush()
         # FIXME: we should remove the auto pdb behavior from here and leave
         # that to the clients.
         try:
             self.debugger()
         except KeyboardInterrupt:
             print("\nKeyboardInterrupt")
 
     def structured_traceback(self, etype=None, value=None, tb=None,
                              tb_offset=None, number_of_lines_of_context=5):
         if etype is None:
             etype, value, tb = sys.exc_info()
         self.tb = tb
         return FormattedTB.structured_traceback(
             self, etype, value, tb, tb_offset, number_of_lines_of_context)
 
 
 #---------------------------------------------------------------------------
 
 # A simple class to preserve Nathan's original functionality.
 class ColorTB(FormattedTB):
     """Shorthand to initialize a FormattedTB in Linux colors mode."""
 
     def __init__(self, color_scheme='Linux', call_pdb=0, **kwargs):
         FormattedTB.__init__(self, color_scheme=color_scheme,
                              call_pdb=call_pdb, **kwargs)
 
 
 class SyntaxTB(ListTB):
     """Extension which holds some state: the last exception value"""
 
     def __init__(self, color_scheme='NoColor', parent=None, config=None):
         ListTB.__init__(self, color_scheme, parent=parent, config=config)
         self.last_syntax_error = None
 
     def __call__(self, etype, value, elist):
         self.last_syntax_error = value
 
         ListTB.__call__(self, etype, value, elist)
 
     def structured_traceback(self, etype, value, elist, tb_offset=None,
                              context=5):
         # If the source file has been edited, the line in the syntax error can
         # be wrong (retrieved from an outdated cache). This replaces it with
         # the current value.
         if isinstance(value, SyntaxError) \
                 and isinstance(value.filename, str) \
                 and isinstance(value.lineno, int):
             linecache.checkcache(value.filename)
             newtext = ulinecache.getline(value.filename, value.lineno)
             if newtext:
                 value.text = newtext
         self.last_syntax_error = value
         return super(SyntaxTB, self).structured_traceback(etype, value, elist,
                                                           tb_offset=tb_offset, context=context)
 
     def clear_err_state(self):
         """Return the current error state and clear it"""
         e = self.last_syntax_error
         self.last_syntax_error = None
         return e
 
     def stb2text(self, stb):
         """Convert a structured traceback (a list) to a string."""
         return ''.join(stb)
 
 
 # some internal-use functions
 def text_repr(value):
     """Hopefully pretty robust repr equivalent."""
     # this is pretty horrible but should always return *something*
     try:
         return pydoc.text.repr(value)
     except KeyboardInterrupt:
         raise
     except:
         try:
             return repr(value)
         except KeyboardInterrupt:
             raise
         except:
             try:
                 # all still in an except block so we catch
                 # getattr raising
                 name = getattr(value, '__name__', None)
                 if name:
                     # ick, recursion
                     return text_repr(name)
                 klass = getattr(value, '__class__', None)
                 if klass:
                     return '%s instance' % text_repr(klass)
             except KeyboardInterrupt:
                 raise
             except:
                 return 'UNRECOVERABLE REPR FAILURE'
 
 
 def eqrepr(value, repr=text_repr):
     return '=%s' % repr(value)
 
 
 def nullrepr(value, repr=text_repr):
     return ''

IPython/terminal/ptutils.py

@@ -1,108 +1,106 b''
 """prompt-toolkit utilities
 
 Everything in this module is a private API,
 not to be used outside IPython.
 """
 
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
 import unicodedata
 from wcwidth import wcwidth
 
-from IPython.utils.py3compat import PY3
-
 from IPython.core.completer import IPCompleter
 from prompt_toolkit.completion import Completer, Completion
 from prompt_toolkit.layout.lexers import Lexer
 from prompt_toolkit.layout.lexers import PygmentsLexer
 
 import pygments.lexers as pygments_lexers
 
 
 class IPythonPTCompleter(Completer):
     """Adaptor to provide IPython completions to prompt_toolkit"""
     def __init__(self, ipy_completer=None, shell=None):
         if shell is None and ipy_completer is None:
             raise TypeError("Please pass shell=an InteractiveShell instance.")
         self._ipy_completer = ipy_completer
         self.shell = shell
 
     @property
     def ipy_completer(self):
         if self._ipy_completer:
             return self._ipy_completer
         else:
             return self.shell.Completer
 
     def get_completions(self, document, complete_event):
         if not document.current_line.strip():
             return
 
         used, matches = self.ipy_completer.complete(
                             line_buffer=document.current_line,
                             cursor_pos=document.cursor_position_col
         )
         start_pos = -len(used)
         for m in matches:
             if not m:
                 # Guard against completion machinery giving us an empty string.
                 continue
 
             m = unicodedata.normalize('NFC', m)
 
             # When the first character of the completion has a zero length,
             # then it's probably a decomposed unicode character. E.g. caused by
             # the "\dot" completion. Try to compose again with the previous
             # character.
             if wcwidth(m[0]) == 0:
                 if document.cursor_position + start_pos > 0:
                     char_before = document.text[document.cursor_position + start_pos - 1]
                     m = unicodedata.normalize('NFC', char_before + m)
 
                     # Yield the modified completion instead, if this worked.
                     if wcwidth(m[0:1]) == 1:
                         yield Completion(m, start_position=start_pos - 1)
                         continue
 
             # TODO: Use Jedi to determine meta_text
             # (Jedi currently has a bug that results in incorrect information.)
             # meta_text = ''
             # yield Completion(m, start_position=start_pos,
             #                  display_meta=meta_text)
             yield Completion(m, start_position=start_pos)
 
 class IPythonPTLexer(Lexer):
     """
     Wrapper around PythonLexer and BashLexer.
     """
     def __init__(self):
         l = pygments_lexers
-        self.python_lexer = PygmentsLexer(l.Python3Lexer if PY3 else l.PythonLexer)
+        self.python_lexer = PygmentsLexer(l.Python3Lexer)
         self.shell_lexer = PygmentsLexer(l.BashLexer)
 
         self.magic_lexers = {
             'HTML': PygmentsLexer(l.HtmlLexer),
             'html': PygmentsLexer(l.HtmlLexer),
             'javascript': PygmentsLexer(l.JavascriptLexer),
             'js': PygmentsLexer(l.JavascriptLexer),
             'perl': PygmentsLexer(l.PerlLexer),
             'ruby': PygmentsLexer(l.RubyLexer),
             'latex': PygmentsLexer(l.TexLexer),
         }
 
     def lex_document(self, cli, document):
         text = document.text.lstrip()
 
         lexer = self.python_lexer
 
         if text.startswith('!') or text.startswith('%%bash'):
             lexer = self.shell_lexer
 
         elif text.startswith('%%'):
             for magic, l in self.magic_lexers.items():
                 if text.startswith('%%' + magic):
                     lexer = l
                     break
 
         return lexer.lex_document(cli, document)

IPython/utils/tests/test_io.py

@@ -1,85 +1,81 b''
 # encoding: utf-8
 """Tests for io.py"""
 
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
 
 import io as stdlib_io
 import os.path
 import stat
 import sys
+from io import StringIO
 
 from subprocess import Popen, PIPE
 import unittest
 
 import nose.tools as nt
 
 from IPython.testing.decorators import skipif, skip_win32
 from IPython.utils.io import Tee, capture_output
-from IPython.utils.py3compat import doctest_refactor_print, PY3
+from IPython.utils.py3compat import doctest_refactor_print
 from IPython.utils.tempdir import TemporaryDirectory
 
-if PY3:
-    from io import StringIO
-else:
-    from StringIO import StringIO
-
 
 def test_tee_simple():
     "Very simple check with stdout only"
     chan = StringIO()
     text = 'Hello'
     tee = Tee(chan, channel='stdout')
     print(text, file=chan)
     nt.assert_equal(chan.getvalue(), text+"\n")
 
 
 class TeeTestCase(unittest.TestCase):
 
     def tchan(self, channel, check='close'):
         trap = StringIO()
         chan = StringIO()
         text = 'Hello'
         
         std_ori = getattr(sys, channel)
         setattr(sys, channel, trap)
 
         tee = Tee(chan, channel=channel)
         print(text, end='', file=chan)
         setattr(sys, channel, std_ori)
         trap_val = trap.getvalue()
         nt.assert_equal(chan.getvalue(), text)
         if check=='close':
             tee.close()
         else:
             del tee
 
     def test(self):
         for chan in ['stdout', 'stderr']:
             for check in ['close', 'del']:
                 self.tchan(chan, check)
 
 def test_io_init():
     """Test that io.stdin/out/err exist at startup"""
     for name in ('stdin', 'stdout', 'stderr'):
         cmd = doctest_refactor_print("from IPython.utils import io;print io.%s.__class__"%name)
         p = Popen([sys.executable, '-c', cmd],
                     stdout=PIPE)
         p.wait()
         classname = p.stdout.read().strip().decode('ascii')
         # __class__ is a reference to the class object in Python 3, so we can't
         # just test for string equality.
         assert 'IPython.utils.io.IOStream' in classname, classname
 
 def test_capture_output():
     """capture_output() context works"""
     
     with capture_output() as io:
         print('hi, stdout')
         print('hi, stderr', file=sys.stderr)
     
     nt.assert_equal(io.stdout, 'hi, stdout\n')
     nt.assert_equal(io.stderr, 'hi, stderr\n')