Start refactoring handling of color.

This is one more step into getting rid of the old way of handling
colors in IPython, in particular here we move the scheme/style as a
configuration option of the parser instead of passing it around into the
formats functions.

Also remove our own usage of deprecated arguments.

IPython/core/debugger.py

@@ -1,636 +1,637 b''
 # -*- coding: utf-8 -*-
 """
 Pdb debugger class.
 
 Modified from the standard pdb.Pdb class to avoid including readline, so that
 the command line completion of other programs which include this isn't
 damaged.
 
 In the future, this class will be expanded with improvements over the standard
 pdb.
 
 The code in this file is mainly lifted out of cmd.py in Python 2.2, with minor
 changes. Licensing should therefore be under the standard Python terms.  For
 details on the PSF (Python Software Foundation) standard license, see:
 
 http://www.python.org/2.2.3/license.html"""
 
 #*****************************************************************************
 #
 #       This file is licensed under the PSF license.
 #
 #       Copyright (C) 2001 Python Software Foundation, www.python.org
 #       Copyright (C) 2005-2006 Fernando Perez. <[e-mail]>
 #
 #
 #*****************************************************************************
 from __future__ import print_function
 
 import bdb
 import functools
 import inspect
 import sys
 import warnings
 
 from IPython import get_ipython
 from IPython.utils import PyColorize, ulinecache
 from IPython.utils import coloransi, py3compat
 from IPython.core.excolors import exception_colors
 from IPython.testing.skipdoctest import skip_doctest
 
 
 prompt = 'ipdb> '
 
 #We have to check this directly from sys.argv, config struct not yet available
 from pdb import Pdb as OldPdb
 
 # Allow the set_trace code to operate outside of an ipython instance, even if
 # it does so with some limitations.  The rest of this support is implemented in
 # the Tracer constructor.
 
 def make_arrow(pad):
     """generate the leading arrow in front of traceback or debugger"""
     if pad >= 2:
         return '-'*(pad-2) + '> '
     elif pad == 1:
         return '>'
     return ''
 
 
 def BdbQuit_excepthook(et, ev, tb, excepthook=None):
     """Exception hook which handles `BdbQuit` exceptions.
 
     All other exceptions are processed using the `excepthook`
     parameter.
     """
     warnings.warn("`BdbQuit_excepthook` is deprecated since version 5.1",
                   DeprecationWarning)
     if et==bdb.BdbQuit:
         print('Exiting Debugger.')
     elif excepthook is not None:
         excepthook(et, ev, tb)
     else:
         # Backwards compatibility. Raise deprecation warning?
         BdbQuit_excepthook.excepthook_ori(et,ev,tb)
 
 
 def BdbQuit_IPython_excepthook(self,et,ev,tb,tb_offset=None):
     warnings.warn(
         "`BdbQuit_IPython_excepthook` is deprecated since version 5.1",
         DeprecationWarning)
     print('Exiting Debugger.')
 
 
 class Tracer(object):
     """
     DEPRECATED
 
     Class for local debugging, similar to pdb.set_trace.
 
     Instances of this class, when called, behave like pdb.set_trace, but
     providing IPython's enhanced capabilities.
 
     This is implemented as a class which must be initialized in your own code
     and not as a standalone function because we need to detect at runtime
     whether IPython is already active or not.  That detection is done in the
     constructor, ensuring that this code plays nicely with a running IPython,
     while functioning acceptably (though with limitations) if outside of it.
     """
 
     @skip_doctest
     def __init__(self, colors=None):
         """
         DEPRECATED
 
         Create a local debugger instance.
 
         Parameters
         ----------
 
         colors : str, optional
             The name of the color scheme to use, it must be one of IPython's
             valid color schemes.  If not given, the function will default to
             the current IPython scheme when running inside IPython, and to
             'NoColor' otherwise.
 
         Examples
         --------
         ::
 
             from IPython.core.debugger import Tracer; debug_here = Tracer()
 
         Later in your code::
 
             debug_here()  # -> will open up the debugger at that point.
 
         Once the debugger activates, you can use all of its regular commands to
         step through code, set breakpoints, etc.  See the pdb documentation
         from the Python standard library for usage details.
         """
         warnings.warn("`Tracer` is deprecated since version 5.1, directly use "
                       "`IPython.core.debugger.Pdb.set_trace()`",
                       DeprecationWarning)
 
         ip = get_ipython()
         if ip is None:
             # Outside of ipython, we set our own exception hook manually
             sys.excepthook = functools.partial(BdbQuit_excepthook,
                                                excepthook=sys.excepthook)
             def_colors = 'NoColor'
         else:
             # In ipython, we use its custom exception handler mechanism
             def_colors = ip.colors
             ip.set_custom_exc((bdb.BdbQuit,), BdbQuit_IPython_excepthook)
 
         if colors is None:
             colors = def_colors
 
         # The stdlib debugger internally uses a modified repr from the `repr`
         # module, that limits the length of printed strings to a hardcoded
         # limit of 30 characters.  That much trimming is too aggressive, let's
         # at least raise that limit to 80 chars, which should be enough for
         # most interactive uses.
         try:
             try:
                 from reprlib import aRepr  # Py 3
             except ImportError:
                 from repr import aRepr  # Py 2
             aRepr.maxstring = 80
         except:
             # This is only a user-facing convenience, so any error we encounter
             # here can be warned about but can be otherwise ignored.  These
             # printouts will tell us about problems if this API changes
             import traceback
             traceback.print_exc()
 
         self.debugger = Pdb(colors)
 
     def __call__(self):
         """Starts an interactive debugger at the point where called.
 
         This is similar to the pdb.set_trace() function from the std lib, but
         using IPython's enhanced debugger."""
 
         self.debugger.set_trace(sys._getframe().f_back)
 
 
 def decorate_fn_with_doc(new_fn, old_fn, additional_text=""):
     """Make new_fn have old_fn's doc string. This is particularly useful
     for the ``do_...`` commands that hook into the help system.
     Adapted from from a comp.lang.python posting
     by Duncan Booth."""
     def wrapper(*args, **kw):
         return new_fn(*args, **kw)
     if old_fn.__doc__:
         wrapper.__doc__ = old_fn.__doc__ + additional_text
     return wrapper
 
 
 def _file_lines(fname):
     """Return the contents of a named file as a list of lines.
 
     This function never raises an IOError exception: if the file can't be
     read, it simply returns an empty list."""
 
     try:
         outfile = open(fname)
     except IOError:
         return []
     else:
         out = outfile.readlines()
         outfile.close()
         return out
 
 
 class Pdb(OldPdb, object):
     """Modified Pdb class, does not load readline.
 
     for a standalone version that uses prompt_toolkit, see
     `IPython.terminal.debugger.TerminalPdb` and
     `IPython.terminal.debugger.set_trace()`
     """
 
     def __init__(self, color_scheme=None, completekey=None,
                  stdin=None, stdout=None, context=5):
 
         # Parent constructor:
         try:
             self.context = int(context)
             if self.context <= 0:
                 raise ValueError("Context must be a positive integer")
         except (TypeError, ValueError):
                 raise ValueError("Context must be a positive integer")
 
         OldPdb.__init__(self, completekey, stdin, stdout)
 
         # IPython changes...
         self.shell = get_ipython()
 
         if self.shell is None:
             save_main = sys.modules['__main__']
             # No IPython instance running, we must create one
             from IPython.terminal.interactiveshell import \
                 TerminalInteractiveShell
             self.shell = TerminalInteractiveShell.instance()
             # needed by any code which calls __import__("__main__") after
             # the debugger was entered. See also #9941.
             sys.modules['__main__'] = save_main 
 
         if color_scheme is not None:
             warnings.warn(
                 "The `color_scheme` argument is deprecated since version 5.1",
-                DeprecationWarning)
+                DeprecationWarning, stacklevel=2)
         else:
             color_scheme = self.shell.colors
 
         self.aliases = {}
 
         # Create color table: we copy the default one from the traceback
         # module and add a few attributes needed for debugging
         self.color_scheme_table = exception_colors()
 
         # shorthands
         C = coloransi.TermColors
         cst = self.color_scheme_table
 
         cst['NoColor'].colors.prompt = C.NoColor
         cst['NoColor'].colors.breakpoint_enabled = C.NoColor
         cst['NoColor'].colors.breakpoint_disabled = C.NoColor
 
         cst['Linux'].colors.prompt = C.Green
         cst['Linux'].colors.breakpoint_enabled = C.LightRed
         cst['Linux'].colors.breakpoint_disabled = C.Red
 
         cst['LightBG'].colors.prompt = C.Blue
         cst['LightBG'].colors.breakpoint_enabled = C.LightRed
         cst['LightBG'].colors.breakpoint_disabled = C.Red
 
         cst['Neutral'].colors.prompt = C.Blue
         cst['Neutral'].colors.breakpoint_enabled = C.LightRed
         cst['Neutral'].colors.breakpoint_disabled = C.Red
 
-        self.set_colors(color_scheme)
 
         # Add a python parser so we can syntax highlight source while
         # debugging.
-        self.parser = PyColorize.Parser()
+        self.parser = PyColorize.Parser(style=color_scheme)
+        self.set_colors(color_scheme)
 
         # Set the prompt - the default prompt is '(Pdb)'
         self.prompt = prompt
 
     def set_colors(self, scheme):
         """Shorthand access to the color table scheme selector method."""
         self.color_scheme_table.set_active_scheme(scheme)
+        self.parser.style = scheme
 
     def trace_dispatch(self, frame, event, arg):
         try:
             return super(Pdb, self).trace_dispatch(frame, event, arg)
         except bdb.BdbQuit:
             pass
 
     def interaction(self, frame, traceback):
         try:
             OldPdb.interaction(self, frame, traceback)
         except KeyboardInterrupt:
             sys.stdout.write('\n' + self.shell.get_exception_only())
 
     def parseline(self, line):
         if line.startswith("!!"):
             # Force standard behavior.
             return super(Pdb, self).parseline(line[2:])
         # "Smart command mode" from pdb++: don't execute commands if a variable
         # with the same name exists.
         cmd, arg, newline = super(Pdb, self).parseline(line)
         # Fix for #9611: Do not trigger smart command if the command is `exit`
         # or `quit` and it would resolve to their *global* value (the
         # `ExitAutocall` object).  Just checking that it is not present in the
         # locals dict is not enough as locals and globals match at the
         # toplevel.
         if ((cmd in self.curframe.f_locals or cmd in self.curframe.f_globals)
                 and not (cmd in ["exit", "quit"]
                          and (self.curframe.f_locals is self.curframe.f_globals
                               or cmd not in self.curframe.f_locals))):
             return super(Pdb, self).parseline("!" + line)
         return super(Pdb, self).parseline(line)
 
     def new_do_up(self, arg):
         OldPdb.do_up(self, arg)
     do_u = do_up = decorate_fn_with_doc(new_do_up, OldPdb.do_up)
 
     def new_do_down(self, arg):
         OldPdb.do_down(self, arg)
 
     do_d = do_down = decorate_fn_with_doc(new_do_down, OldPdb.do_down)
 
     def new_do_frame(self, arg):
         OldPdb.do_frame(self, arg)
 
     def new_do_quit(self, arg):
 
         if hasattr(self, 'old_all_completions'):
             self.shell.Completer.all_completions=self.old_all_completions
 
         return OldPdb.do_quit(self, arg)
 
     do_q = do_quit = decorate_fn_with_doc(new_do_quit, OldPdb.do_quit)
 
     def new_do_restart(self, arg):
         """Restart command. In the context of ipython this is exactly the same
         thing as 'quit'."""
         self.msg("Restart doesn't make sense here. Using 'quit' instead.")
         return self.do_quit(arg)
 
     def print_stack_trace(self, context=None):
         if context is None:
             context = self.context
         try:
             context=int(context)
             if context <= 0:
                 raise ValueError("Context must be a positive integer")
         except (TypeError, ValueError):
                 raise ValueError("Context must be a positive integer")
         try:
             for frame_lineno in self.stack:
                 self.print_stack_entry(frame_lineno, context=context)
         except KeyboardInterrupt:
             pass
 
     def print_stack_entry(self,frame_lineno, prompt_prefix='\n-> ',
                           context=None):
         if context is None:
             context = self.context
         try:
             context=int(context)
             if context <= 0:
                 raise ValueError("Context must be a positive integer")
         except (TypeError, ValueError):
                 raise ValueError("Context must be a positive integer")
         print(self.format_stack_entry(frame_lineno, '', context))
 
         # vds: >>
         frame, lineno = frame_lineno
         filename = frame.f_code.co_filename
         self.shell.hooks.synchronize_with_editor(filename, lineno, 0)
         # vds: <<
 
     def format_stack_entry(self, frame_lineno, lprefix=': ', context=None):
         if context is None:
             context = self.context
         try:
             context=int(context)
             if context <= 0:
                 print("Context must be a positive integer")
         except (TypeError, ValueError):
                 print("Context must be a positive integer")
         try:
             import reprlib  # Py 3
         except ImportError:
             import repr as reprlib  # Py 2
 
         ret = []
 
         Colors = self.color_scheme_table.active_colors
         ColorsNormal = Colors.Normal
         tpl_link = u'%s%%s%s' % (Colors.filenameEm, ColorsNormal)
         tpl_call = u'%s%%s%s%%s%s' % (Colors.vName, Colors.valEm, ColorsNormal)
         tpl_line = u'%%s%s%%s %s%%s' % (Colors.lineno, ColorsNormal)
         tpl_line_em = u'%%s%s%%s %s%%s%s' % (Colors.linenoEm, Colors.line,
                                             ColorsNormal)
 
         frame, lineno = frame_lineno
 
         return_value = ''
         if '__return__' in frame.f_locals:
             rv = frame.f_locals['__return__']
             #return_value += '->'
             return_value += reprlib.repr(rv) + '\n'
         ret.append(return_value)
 
         #s = filename + '(' + `lineno` + ')'
         filename = self.canonic(frame.f_code.co_filename)
         link = tpl_link % py3compat.cast_unicode(filename)
 
         if frame.f_code.co_name:
             func = frame.f_code.co_name
         else:
             func = "<lambda>"
 
         call = ''
         if func != '?':
             if '__args__' in frame.f_locals:
                 args = reprlib.repr(frame.f_locals['__args__'])
             else:
                 args = '()'
             call = tpl_call % (func, args)
 
         # The level info should be generated in the same format pdb uses, to
         # avoid breaking the pdbtrack functionality of python-mode in *emacs.
         if frame is self.curframe:
             ret.append('> ')
         else:
             ret.append('  ')
         ret.append(u'%s(%s)%s\n' % (link,lineno,call))
 
         start = lineno - 1 - context//2
         lines = ulinecache.getlines(filename)
         start = min(start, len(lines) - context)
         start = max(start, 0)
         lines = lines[start : start + context]
 
         for i,line in enumerate(lines):
             show_arrow = (start + 1 + i == lineno)
             linetpl = (frame is self.curframe or show_arrow) \
                       and tpl_line_em \
                       or tpl_line
             ret.append(self.__format_line(linetpl, filename,
                                           start + 1 + i, line,
                                           arrow = show_arrow) )
         return ''.join(ret)
 
     def __format_line(self, tpl_line, filename, lineno, line, arrow = False):
         bp_mark = ""
         bp_mark_color = ""
 
-        scheme = self.color_scheme_table.active_scheme_name
-        new_line, err = self.parser.format2(line, 'str', scheme)
-        if not err: line = new_line
+        new_line, err = self.parser.format2(line, 'str')
+        if not err:
+            line = new_line
 
         bp = None
         if lineno in self.get_file_breaks(filename):
             bps = self.get_breaks(filename, lineno)
             bp = bps[-1]
 
         if bp:
             Colors = self.color_scheme_table.active_colors
             bp_mark = str(bp.number)
             bp_mark_color = Colors.breakpoint_enabled
             if not bp.enabled:
                 bp_mark_color = Colors.breakpoint_disabled
 
         numbers_width = 7
         if arrow:
             # This is the line with the error
             pad = numbers_width - len(str(lineno)) - len(bp_mark)
             num = '%s%s' % (make_arrow(pad), str(lineno))
         else:
             num = '%*s' % (numbers_width - len(bp_mark), str(lineno))
 
         return tpl_line % (bp_mark_color + bp_mark, num, line)
 
 
     def print_list_lines(self, filename, first, last):
         """The printing (as opposed to the parsing part of a 'list'
         command."""
         try:
             Colors = self.color_scheme_table.active_colors
             ColorsNormal = Colors.Normal
             tpl_line = '%%s%s%%s %s%%s' % (Colors.lineno, ColorsNormal)
             tpl_line_em = '%%s%s%%s %s%%s%s' % (Colors.linenoEm, Colors.line, ColorsNormal)
             src = []
             if filename == "<string>" and hasattr(self, "_exec_filename"):
                 filename = self._exec_filename
 
             for lineno in range(first, last+1):
                 line = ulinecache.getline(filename, lineno)
                 if not line:
                     break
 
                 if lineno == self.curframe.f_lineno:
                     line = self.__format_line(tpl_line_em, filename, lineno, line, arrow = True)
                 else:
                     line = self.__format_line(tpl_line, filename, lineno, line, arrow = False)
 
                 src.append(line)
                 self.lineno = lineno
 
             print(''.join(src))
 
         except KeyboardInterrupt:
             pass
 
     def do_list(self, arg):
         self.lastcmd = 'list'
         last = None
         if arg:
             try:
                 x = eval(arg, {}, {})
                 if type(x) == type(()):
                     first, last = x
                     first = int(first)
                     last = int(last)
                     if last < first:
                         # Assume it's a count
                         last = first + last
                 else:
                     first = max(1, int(x) - 5)
             except:
                 print('*** Error in argument:', repr(arg))
                 return
         elif self.lineno is None:
             first = max(1, self.curframe.f_lineno - 5)
         else:
             first = self.lineno + 1
         if last is None:
             last = first + 10
         self.print_list_lines(self.curframe.f_code.co_filename, first, last)
 
         # vds: >>
         lineno = first
         filename = self.curframe.f_code.co_filename
         self.shell.hooks.synchronize_with_editor(filename, lineno, 0)
         # vds: <<
 
     do_l = do_list
 
     def getsourcelines(self, obj):
         lines, lineno = inspect.findsource(obj)
         if inspect.isframe(obj) and obj.f_globals is obj.f_locals:
             # must be a module frame: do not try to cut a block out of it
             return lines, 1
         elif inspect.ismodule(obj):
             return lines, 1
         return inspect.getblock(lines[lineno:]), lineno+1
 
     def do_longlist(self, arg):
         self.lastcmd = 'longlist'
         try:
             lines, lineno = self.getsourcelines(self.curframe)
         except OSError as err:
             self.error(err)
             return
         last = lineno + len(lines)
         self.print_list_lines(self.curframe.f_code.co_filename, lineno, last)
     do_ll = do_longlist
 
     def do_pdef(self, arg):
         """Print the call signature for any callable object.
 
         The debugger interface to %pdef"""
         namespaces = [('Locals', self.curframe.f_locals),
                       ('Globals', self.curframe.f_globals)]
         self.shell.find_line_magic('pdef')(arg, namespaces=namespaces)
 
     def do_pdoc(self, arg):
         """Print the docstring for an object.
 
         The debugger interface to %pdoc."""
         namespaces = [('Locals', self.curframe.f_locals),
                       ('Globals', self.curframe.f_globals)]
         self.shell.find_line_magic('pdoc')(arg, namespaces=namespaces)
 
     def do_pfile(self, arg):
         """Print (or run through pager) the file where an object is defined.
 
         The debugger interface to %pfile.
         """
         namespaces = [('Locals', self.curframe.f_locals),
                       ('Globals', self.curframe.f_globals)]
         self.shell.find_line_magic('pfile')(arg, namespaces=namespaces)
 
     def do_pinfo(self, arg):
         """Provide detailed information about an object.
 
         The debugger interface to %pinfo, i.e., obj?."""
         namespaces = [('Locals', self.curframe.f_locals),
                       ('Globals', self.curframe.f_globals)]
         self.shell.find_line_magic('pinfo')(arg, namespaces=namespaces)
 
     def do_pinfo2(self, arg):
         """Provide extra detailed information about an object.
 
         The debugger interface to %pinfo2, i.e., obj??."""
         namespaces = [('Locals', self.curframe.f_locals),
                       ('Globals', self.curframe.f_globals)]
         self.shell.find_line_magic('pinfo2')(arg, namespaces=namespaces)
 
     def do_psource(self, arg):
         """Print (or run through pager) the source code for an object."""
         namespaces = [('Locals', self.curframe.f_locals),
                       ('Globals', self.curframe.f_globals)]
         self.shell.find_line_magic('psource')(arg, namespaces=namespaces)
 
     if sys.version_info > (3, ):
         def do_where(self, arg):
             """w(here)
             Print a stack trace, with the most recent frame at the bottom.
             An arrow indicates the "current frame", which determines the
             context of most commands. 'bt' is an alias for this command.
 
             Take a number as argument as an (optional) number of context line to
             print"""
             if arg:
                 context = int(arg)
                 self.print_stack_trace(context)
             else:
                 self.print_stack_trace()
 
         do_w = do_where
 
 
 def set_trace(frame=None):
     """
     Start debugging from `frame`.
 
     If frame is not specified, debugging starts from caller's frame.
     """
     Pdb().set_trace(frame or sys._getframe().f_back)

IPython/core/tests/test_completer.py

@@ -1,807 +1,806 b''
 # encoding: utf-8
 """Tests for the IPython tab-completion machinery."""
 
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
 import os
 import sys
 import unittest
 
 from contextlib import contextmanager
 
 import nose.tools as nt
 
 from traitlets.config.loader import Config
 from IPython import get_ipython
 from IPython.core import completer
 from IPython.external.decorators import knownfailureif
 from IPython.utils.tempdir import TemporaryDirectory, TemporaryWorkingDirectory
 from IPython.utils.generics import complete_object
 from IPython.utils.py3compat import string_types, unicode_type
 from IPython.testing import decorators as dec
 
 #-----------------------------------------------------------------------------
 # Test functions
 #-----------------------------------------------------------------------------
 
 @contextmanager
 def greedy_completion():
     ip = get_ipython()
     greedy_original = ip.Completer.greedy
     try:
         ip.Completer.greedy = True
         yield
     finally:
         ip.Completer.greedy = greedy_original
 
 def test_protect_filename():
     if sys.platform == 'win32':
         pairs = [('abc','abc'),
                  (' abc','" abc"'),
                  ('a bc','"a bc"'),
                  ('a  bc','"a  bc"'),
                  ('  bc','"  bc"'),
                  ]
     else:
         pairs = [('abc','abc'),
                  (' abc',r'\ abc'),
                  ('a bc',r'a\ bc'),
                  ('a  bc',r'a\ \ bc'),
                  ('  bc',r'\ \ bc'),
                  # On posix, we also protect parens and other special characters.
                  ('a(bc',r'a\(bc'),
                  ('a)bc',r'a\)bc'),
                  ('a( )bc',r'a\(\ \)bc'),
                  ('a[1]bc', r'a\[1\]bc'),
                  ('a{1}bc', r'a\{1\}bc'),
                  ('a#bc', r'a\#bc'),
                  ('a?bc', r'a\?bc'),
                  ('a=bc', r'a\=bc'),
                  ('a\\bc', r'a\\bc'),
                  ('a|bc', r'a\|bc'),
                  ('a;bc', r'a\;bc'),
                  ('a:bc', r'a\:bc'),
                  ("a'bc", r"a\'bc"),
                  ('a*bc', r'a\*bc'),
                  ('a"bc', r'a\"bc'),
                  ('a^bc', r'a\^bc'),
                  ('a&bc', r'a\&bc'),
                  ]
     # run the actual tests
     for s1, s2 in pairs:
         s1p = completer.protect_filename(s1)
         nt.assert_equal(s1p, s2)
 
 
 def check_line_split(splitter, test_specs):
     for part1, part2, split in test_specs:
         cursor_pos = len(part1)
         line = part1+part2
         out = splitter.split_line(line, cursor_pos)
         nt.assert_equal(out, split)
 
 
 def test_line_split():
     """Basic line splitter test with default specs."""
     sp = completer.CompletionSplitter()
     # The format of the test specs is: part1, part2, expected answer.  Parts 1
     # and 2 are joined into the 'line' sent to the splitter, as if the cursor
     # was at the end of part1.  So an empty part2 represents someone hitting
     # tab at the end of the line, the most common case.
     t = [('run some/scrip', '', 'some/scrip'),
          ('run scripts/er', 'ror.py foo', 'scripts/er'),
          ('echo $HOM', '', 'HOM'),
          ('print sys.pa', '', 'sys.pa'),
          ('print(sys.pa', '', 'sys.pa'),
          ("execfile('scripts/er", '', 'scripts/er'),
          ('a[x.', '', 'x.'),
          ('a[x.', 'y', 'x.'),
          ('cd "some_file/', '', 'some_file/'),
          ]
     check_line_split(sp, t)
     # Ensure splitting works OK with unicode by re-running the tests with
     # all inputs turned into unicode
     check_line_split(sp, [ map(unicode_type, p) for p in t] )
 
 
 def test_custom_completion_error():
     """Test that errors from custom attribute completers are silenced."""
     ip = get_ipython()
     class A(object): pass
     ip.user_ns['a'] = A()
     
     @complete_object.when_type(A)
     def complete_A(a, existing_completions):
         raise TypeError("this should be silenced")
     
     ip.complete("a.")
 
 
 def test_unicode_completions():
     ip = get_ipython()
     # Some strings that trigger different types of completion.  Check them both
     # in str and unicode forms
     s = ['ru', '%ru', 'cd /', 'floa', 'float(x)/']
     for t in s + list(map(unicode_type, s)):
         # We don't need to check exact completion values (they may change
         # depending on the state of the namespace, but at least no exceptions
         # should be thrown and the return value should be a pair of text, list
         # values.
         text, matches = ip.complete(t)
         nt.assert_true(isinstance(text, string_types))
         nt.assert_true(isinstance(matches, list))
 
[e-mail](sys.version_info[0] >= 3, 'This test only applies in Py>=3')
 def test_latex_completions():
     from IPython.core.latex_symbols import latex_symbols
     import random
     ip = get_ipython()
     # Test some random unicode symbols
     keys = random.sample(latex_symbols.keys(), 10)
     for k in keys:
         text, matches = ip.complete(k)
         nt.assert_equal(len(matches),1)
         nt.assert_equal(text, k)
         nt.assert_equal(matches[0], latex_symbols[k])
     # Test a more complex line
     text, matches = ip.complete(u'print(\\alpha')
     nt.assert_equals(text, u'\\alpha')
     nt.assert_equals(matches[0], latex_symbols['\\alpha'])
     # Test multiple matching latex symbols
     text, matches = ip.complete(u'\\al')
     nt.assert_in('\\alpha', matches)
     nt.assert_in('\\aleph', matches)
 
 
 
 
 @dec.onlyif(sys.version_info[0] >= 3, 'This test only apply on python3')
 def test_back_latex_completion():
     ip = get_ipython()
 
     # do not return more than 1 matches fro \beta, only the latex one.
     name, matches = ip.complete('\\β')
     nt.assert_equal(len(matches), 1)
     nt.assert_equal(matches[0], '\\beta')
 
 @dec.onlyif(sys.version_info[0] >= 3, 'This test only apply on python3')
 def test_back_unicode_completion():
     ip = get_ipython()
     
     name, matches = ip.complete('\\Ⅴ')
     nt.assert_equal(len(matches), 1)
     nt.assert_equal(matches[0], '\\ROMAN NUMERAL FIVE')
 
 
 @dec.onlyif(sys.version_info[0] >= 3, 'This test only apply on python3')
 def test_forward_unicode_completion():
     ip = get_ipython()
     
     name, matches = ip.complete('\\ROMAN NUMERAL FIVE')
     nt.assert_equal(len(matches), 1)
     nt.assert_equal(matches[0], 'Ⅴ')
 
 @dec.onlyif(sys.version_info[0] >= 3, 'This test only apply on python3')
 @dec.knownfailureif(sys.platform == 'win32', 'Fails if there is a C:\\j... path')
 def test_no_ascii_back_completion():
     ip = get_ipython()
     with TemporaryWorkingDirectory():  # Avoid any filename completions
         # single ascii letter that don't have yet completions
         for letter in 'jJ' :
             name, matches = ip.complete('\\'+letter)
             nt.assert_equal(matches, [])
 
 
 
 
 class CompletionSplitterTestCase(unittest.TestCase):
     def setUp(self):
         self.sp = completer.CompletionSplitter()
 
     def test_delim_setting(self):
         self.sp.delims = ' '
         nt.assert_equal(self.sp.delims, ' ')
         nt.assert_equal(self.sp._delim_expr, '[\ ]')
 
     def test_spaces(self):
         """Test with only spaces as split chars."""
         self.sp.delims = ' '
         t = [('foo', '', 'foo'),
              ('run foo', '', 'foo'),
              ('run foo', 'bar', 'foo'),
              ]
         check_line_split(self.sp, t)
 
 
 def test_has_open_quotes1():
     for s in ["'", "'''", "'hi' '"]:
         nt.assert_equal(completer.has_open_quotes(s), "'")
 
 
 def test_has_open_quotes2():
     for s in ['"', '"""', '"hi" "']:
         nt.assert_equal(completer.has_open_quotes(s), '"')
 
 
 def test_has_open_quotes3():
     for s in ["''", "''' '''", "'hi' 'ipython'"]:
         nt.assert_false(completer.has_open_quotes(s))
 
 
 def test_has_open_quotes4():
     for s in ['""', '""" """', '"hi" "ipython"']:
         nt.assert_false(completer.has_open_quotes(s))
 
 
 @knownfailureif(sys.platform == 'win32', "abspath completions fail on Windows")
 def test_abspath_file_completions():
     ip = get_ipython()
     with TemporaryDirectory() as tmpdir:
         prefix = os.path.join(tmpdir, 'foo')
         suffixes = ['1', '2']
         names = [prefix+s for s in suffixes]
         for n in names:
             open(n, 'w').close()
 
         # Check simple completion
         c = ip.complete(prefix)[1]
         nt.assert_equal(c, names)
 
         # Now check with a function call
         cmd = 'a = f("%s' % prefix
         c = ip.complete(prefix, cmd)[1]
         comp = [prefix+s for s in suffixes]
         nt.assert_equal(c, comp)
 
 
 def test_local_file_completions():
     ip = get_ipython()
     with TemporaryWorkingDirectory():
         prefix = './foo'
         suffixes = ['1', '2']
         names = [prefix+s for s in suffixes]
         for n in names:
             open(n, 'w').close()
 
         # Check simple completion
         c = ip.complete(prefix)[1]
         nt.assert_equal(c, names)
 
         # Now check with a function call
         cmd = 'a = f("%s' % prefix
         c = ip.complete(prefix, cmd)[1]
         comp = set(prefix+s for s in suffixes)
         nt.assert_true(comp.issubset(set(c)))
 
 
 def test_greedy_completions():
     ip = get_ipython()
     ip.ex('a=list(range(5))')
     _,c = ip.complete('.',line='a[0].')
     nt.assert_false('.real' in c,
                     "Shouldn't have completed on a[0]: %s"%c)
     with greedy_completion():
         def _(line, cursor_pos, expect, message):
             _,c = ip.complete('.', line=line, cursor_pos=cursor_pos)
             nt.assert_in(expect, c, message%c)
 
         yield _, 'a[0].', 5, 'a[0].real', "Should have completed on a[0].: %s"
         yield _, 'a[0].r', 6, 'a[0].real', "Should have completed on a[0].r: %s"
         
         if sys.version_info > (3,4):
             yield _, 'a[0].from_', 10, 'a[0].from_bytes', "Should have completed on a[0].from_: %s"
 
 
 
 def test_omit__names():
     # also happens to test IPCompleter as a configurable
     ip = get_ipython()
     ip._hidden_attr = 1
     ip._x = {}
     c = ip.Completer
     ip.ex('ip=get_ipython()')
     cfg = Config()
     cfg.IPCompleter.omit__names = 0
     c.update_config(cfg)
     s,matches = c.complete('ip.')
     nt.assert_in('ip.__str__', matches)
     nt.assert_in('ip._hidden_attr', matches)
     cfg = Config()
     cfg.IPCompleter.omit__names = 1
     c.update_config(cfg)
     s,matches = c.complete('ip.')
     nt.assert_not_in('ip.__str__', matches)
     nt.assert_in('ip._hidden_attr', matches)
     cfg = Config()
     cfg.IPCompleter.omit__names = 2
     c.update_config(cfg)
     s,matches = c.complete('ip.')
     nt.assert_not_in('ip.__str__', matches)
     nt.assert_not_in('ip._hidden_attr', matches)
     s,matches = c.complete('ip._x.')
     nt.assert_in('ip._x.keys', matches)
     del ip._hidden_attr
 
 
 def test_limit_to__all__False_ok():
     ip = get_ipython()
     c = ip.Completer
     ip.ex('class D: x=24')
     ip.ex('d=D()')
     cfg = Config()
     cfg.IPCompleter.limit_to__all__ = False
     c.update_config(cfg)
     s, matches = c.complete('d.')
     nt.assert_in('d.x', matches)
 
 
 def test_get__all__entries_ok():
     class A(object):
         __all__ = ['x', 1]
     words = completer.get__all__entries(A())
     nt.assert_equal(words, ['x'])
 
 
 def test_get__all__entries_no__all__ok():
     class A(object):
         pass
     words = completer.get__all__entries(A())
     nt.assert_equal(words, [])
 
 
 def test_func_kw_completions():
     ip = get_ipython()
     c = ip.Completer
     ip.ex('def myfunc(a=1,b=2): return a+b')
     s, matches = c.complete(None, 'myfunc(1,b')
     nt.assert_in('b=', matches)
     # Simulate completing with cursor right after b (pos==10):
     s, matches = c.complete(None, 'myfunc(1,b)', 10)
     nt.assert_in('b=', matches)
     s, matches = c.complete(None, 'myfunc(a="escaped\\")string",b')
     nt.assert_in('b=', matches)
     #builtin function
     s, matches = c.complete(None, 'min(k, k')
     nt.assert_in('key=', matches)
 
 
 def test_default_arguments_from_docstring():
     ip = get_ipython()
     c = ip.Completer
     kwd = c._default_arguments_from_docstring(
         'min(iterable[, key=func]) -> value')
     nt.assert_equal(kwd, ['key'])
     #with cython type etc
     kwd = c._default_arguments_from_docstring(
         'Minuit.migrad(self, int ncall=10000, resume=True, int nsplit=1)\n')
     nt.assert_equal(kwd, ['ncall', 'resume', 'nsplit'])
     #white spaces
     kwd = c._default_arguments_from_docstring(
         '\n Minuit.migrad(self, int ncall=10000, resume=True, int nsplit=1)\n')
     nt.assert_equal(kwd, ['ncall', 'resume', 'nsplit'])
 
 def test_line_magics():
     ip = get_ipython()
     c = ip.Completer
     s, matches = c.complete(None, 'lsmag')
     nt.assert_in('%lsmagic', matches)
     s, matches = c.complete(None, '%lsmag')
     nt.assert_in('%lsmagic', matches)
 
 
 def test_cell_magics():
     from IPython.core.magic import register_cell_magic
 
     @register_cell_magic
     def _foo_cellm(line, cell):
         pass
     
     ip = get_ipython()
     c = ip.Completer
 
     s, matches = c.complete(None, '_foo_ce')
     nt.assert_in('%%_foo_cellm', matches)
     s, matches = c.complete(None, '%%_foo_ce')
     nt.assert_in('%%_foo_cellm', matches)
 
 
 def test_line_cell_magics():
     from IPython.core.magic import register_line_cell_magic
 
     @register_line_cell_magic
     def _bar_cellm(line, cell):
         pass
     
     ip = get_ipython()
     c = ip.Completer
 
     # The policy here is trickier, see comments in completion code.  The
     # returned values depend on whether the user passes %% or not explicitly,
     # and this will show a difference if the same name is both a line and cell
     # magic.
     s, matches = c.complete(None, '_bar_ce')
     nt.assert_in('%_bar_cellm', matches)
     nt.assert_in('%%_bar_cellm', matches)
     s, matches = c.complete(None, '%_bar_ce')
     nt.assert_in('%_bar_cellm', matches)
     nt.assert_in('%%_bar_cellm', matches)
     s, matches = c.complete(None, '%%_bar_ce')
     nt.assert_not_in('%_bar_cellm', matches)
     nt.assert_in('%%_bar_cellm', matches)
 
 
 def test_magic_completion_order():
 
     ip = get_ipython()
     c = ip.Completer
 
     # Test ordering of magics and non-magics with the same name
     # We want the non-magic first
 
     # Before importing matplotlib, there should only be one option:
 
     text, matches = c.complete('mat')
     nt.assert_equal(matches, ["%matplotlib"])
 
 
     ip.run_cell("matplotlib = 1")  # introduce name into namespace
 
     # After the import, there should be two options, ordered like this:
     text, matches = c.complete('mat')
     nt.assert_equal(matches, ["matplotlib", "%matplotlib"])
 
 
     ip.run_cell("timeit = 1")  # define a user variable called 'timeit'
 
     # Order of user variable and line and cell magics with same name:
     text, matches = c.complete('timeit')
     nt.assert_equal(matches, ["timeit", "%timeit","%%timeit"])
 
 
 def test_dict_key_completion_string():
     """Test dictionary key completion for string keys"""
     ip = get_ipython()
     complete = ip.Completer.complete
 
     ip.user_ns['d'] = {'abc': None}
 
     # check completion at different stages
     _, matches = complete(line_buffer="d[")
     nt.assert_in("'abc'", matches)
     nt.assert_not_in("'abc']", matches)
 
     _, matches = complete(line_buffer="d['")
     nt.assert_in("abc", matches)
     nt.assert_not_in("abc']", matches)
 
     _, matches = complete(line_buffer="d['a")
     nt.assert_in("abc", matches)
     nt.assert_not_in("abc']", matches)
 
     # check use of different quoting
     _, matches = complete(line_buffer="d[\"")
     nt.assert_in("abc", matches)
     nt.assert_not_in('abc\"]', matches)
 
     _, matches = complete(line_buffer="d[\"a")
     nt.assert_in("abc", matches)
     nt.assert_not_in('abc\"]', matches)
 
     # check sensitivity to following context
     _, matches = complete(line_buffer="d[]", cursor_pos=2)
     nt.assert_in("'abc'", matches)
 
     _, matches = complete(line_buffer="d['']", cursor_pos=3)
     nt.assert_in("abc", matches)
     nt.assert_not_in("abc'", matches)
     nt.assert_not_in("abc']", matches)
 
     # check multiple solutions are correctly returned and that noise is not
     ip.user_ns['d'] = {'abc': None, 'abd': None, 'bad': None, object(): None,
                        5: None}
 
     _, matches = complete(line_buffer="d['a")
     nt.assert_in("abc", matches)
     nt.assert_in("abd", matches)
     nt.assert_not_in("bad", matches)
     assert not any(m.endswith((']', '"', "'")) for m in matches), matches
 
     # check escaping and whitespace
     ip.user_ns['d'] = {'a\nb': None, 'a\'b': None, 'a"b': None, 'a word': None}
     _, matches = complete(line_buffer="d['a")
     nt.assert_in("a\\nb", matches)
     nt.assert_in("a\\'b", matches)
     nt.assert_in("a\"b", matches)
     nt.assert_in("a word", matches)
     assert not any(m.endswith((']', '"', "'")) for m in matches), matches
 
     # - can complete on non-initial word of the string
     _, matches = complete(line_buffer="d['a w")
     nt.assert_in("word", matches)
 
     # - understands quote escaping
     _, matches = complete(line_buffer="d['a\\'")
     nt.assert_in("b", matches)
 
     # - default quoting should work like repr
     _, matches = complete(line_buffer="d[")
     nt.assert_in("\"a'b\"", matches)
 
     # - when opening quote with ", possible to match with unescaped apostrophe
     _, matches = complete(line_buffer="d[\"a'")
     nt.assert_in("b", matches)
 
     # need to not split at delims that readline won't split at
     if '-' not in ip.Completer.splitter.delims:
         ip.user_ns['d'] = {'before-after': None}
         _, matches = complete(line_buffer="d['before-af")
         nt.assert_in('before-after', matches)
 
 def test_dict_key_completion_contexts():
     """Test expression contexts in which dict key completion occurs"""
     ip = get_ipython()
     complete = ip.Completer.complete
     d = {'abc': None}
     ip.user_ns['d'] = d
 
     class C:
         data = d
     ip.user_ns['C'] = C
     ip.user_ns['get'] = lambda: d
 
     def assert_no_completion(**kwargs):
         _, matches = complete(**kwargs)
         nt.assert_not_in('abc', matches)
         nt.assert_not_in('abc\'', matches)
         nt.assert_not_in('abc\']', matches)
         nt.assert_not_in('\'abc\'', matches)
         nt.assert_not_in('\'abc\']', matches)
 
     def assert_completion(**kwargs):
         _, matches = complete(**kwargs)
         nt.assert_in("'abc'", matches)
         nt.assert_not_in("'abc']", matches)
 
     # no completion after string closed, even if reopened
     assert_no_completion(line_buffer="d['a'")
     assert_no_completion(line_buffer="d[\"a\"")
     assert_no_completion(line_buffer="d['a' + ")
     assert_no_completion(line_buffer="d['a' + '")
 
     # completion in non-trivial expressions
     assert_completion(line_buffer="+ d[")
     assert_completion(line_buffer="(d[")
     assert_completion(line_buffer="C.data[")
 
     # greedy flag
     def assert_completion(**kwargs):
         _, matches = complete(**kwargs)
         nt.assert_in("get()['abc']", matches)
     
     assert_no_completion(line_buffer="get()[")
     with greedy_completion():
         assert_completion(line_buffer="get()[")
         assert_completion(line_buffer="get()['")
         assert_completion(line_buffer="get()['a")
         assert_completion(line_buffer="get()['ab")
         assert_completion(line_buffer="get()['abc")
 
 
 
 @dec.onlyif(sys.version_info[0] >= 3, 'This test only applies in Py>=3')
 def test_dict_key_completion_bytes():
     """Test handling of bytes in dict key completion"""
     ip = get_ipython()
     complete = ip.Completer.complete
 
     ip.user_ns['d'] = {'abc': None, b'abd': None}
 
     _, matches = complete(line_buffer="d[")
     nt.assert_in("'abc'", matches)
     nt.assert_in("b'abd'", matches)
 
     if False:  # not currently implemented
         _, matches = complete(line_buffer="d[b")
         nt.assert_in("b'abd'", matches)
         nt.assert_not_in("b'abc'", matches)
 
         _, matches = complete(line_buffer="d[b'")
         nt.assert_in("abd", matches)
         nt.assert_not_in("abc", matches)
 
         _, matches = complete(line_buffer="d[B'")
         nt.assert_in("abd", matches)
         nt.assert_not_in("abc", matches)
 
         _, matches = complete(line_buffer="d['")
         nt.assert_in("abc", matches)
         nt.assert_not_in("abd", matches)
 
 
 @dec.onlyif(sys.version_info[0] < 3, 'This test only applies in Py<3')
 def test_dict_key_completion_unicode_py2():
     """Test handling of unicode in dict key completion"""
     ip = get_ipython()
     complete = ip.Completer.complete
 
     ip.user_ns['d'] = {u'abc': None,
                        u'a\u05d0b': None}
 
     _, matches = complete(line_buffer="d[")
     nt.assert_in("u'abc'", matches)
     nt.assert_in("u'a\\u05d0b'", matches)
 
     _, matches = complete(line_buffer="d['a")
     nt.assert_in("abc", matches)
     nt.assert_not_in("a\\u05d0b", matches)
 
     _, matches = complete(line_buffer="d[u'a")
     nt.assert_in("abc", matches)
     nt.assert_in("a\\u05d0b", matches)
 
     _, matches = complete(line_buffer="d[U'a")
     nt.assert_in("abc", matches)
     nt.assert_in("a\\u05d0b", matches)
 
     # query using escape
     if sys.platform != 'win32':
         # Known failure on Windows
         _, matches = complete(line_buffer=u"d[u'a\\u05d0")
         nt.assert_in("u05d0b", matches)  # tokenized after \\
 
     # query using character
     _, matches = complete(line_buffer=u"d[u'a\u05d0")
     nt.assert_in(u"a\u05d0b", matches)
     
     with greedy_completion():
         _, matches = complete(line_buffer="d[")
         nt.assert_in("d[u'abc']", matches)
         nt.assert_in("d[u'a\\u05d0b']", matches)
 
         _, matches = complete(line_buffer="d['a")
         nt.assert_in("d['abc']", matches)
         nt.assert_not_in("d[u'a\\u05d0b']", matches)
 
         _, matches = complete(line_buffer="d[u'a")
         nt.assert_in("d[u'abc']", matches)
         nt.assert_in("d[u'a\\u05d0b']", matches)
 
         _, matches = complete(line_buffer="d[U'a")
         nt.assert_in("d[U'abc']", matches)
         nt.assert_in("d[U'a\\u05d0b']", matches)
 
         # query using escape
         _, matches = complete(line_buffer=u"d[u'a\\u05d0")
         nt.assert_in("d[u'a\\u05d0b']", matches)  # tokenized after \\
 
         # query using character
         _, matches = complete(line_buffer=u"d[u'a\u05d0")
         nt.assert_in(u"d[u'a\u05d0b']", matches)
 
 
 @dec.onlyif(sys.version_info[0] >= 3, 'This test only applies in Py>=3')
 def test_dict_key_completion_unicode_py3():
     """Test handling of unicode in dict key completion"""
     ip = get_ipython()
     complete = ip.Completer.complete
 
     ip.user_ns['d'] = {u'a\u05d0': None}
 
     # query using escape
     if sys.platform != 'win32':
         # Known failure on Windows
         _, matches = complete(line_buffer="d['a\\u05d0")
         nt.assert_in("u05d0", matches)  # tokenized after \\
 
     # query using character
     _, matches = complete(line_buffer="d['a\u05d0")
     nt.assert_in(u"a\u05d0", matches)
     
     with greedy_completion():
         # query using escape
         _, matches = complete(line_buffer="d['a\\u05d0")
         nt.assert_in("d['a\\u05d0']", matches)  # tokenized after \\
 
         # query using character
         _, matches = complete(line_buffer="d['a\u05d0")
         nt.assert_in(u"d['a\u05d0']", matches)
         
 
 
 @dec.skip_without('numpy')
 def test_struct_array_key_completion():
     """Test dict key completion applies to numpy struct arrays"""
     import numpy
     ip = get_ipython()
     complete = ip.Completer.complete
     ip.user_ns['d'] = numpy.array([], dtype=[('hello', 'f'), ('world', 'f')])
     _, matches = complete(line_buffer="d['")
     nt.assert_in("hello", matches)
     nt.assert_in("world", matches)
     # complete on the numpy struct itself
     dt = numpy.dtype([('my_head', [('my_dt', '>u4'), ('my_df', '>u4')]),
                       ('my_data', '>f4', 5)])
     x = numpy.zeros(2, dtype=dt)
     ip.user_ns['d'] = x[1]
     _, matches = complete(line_buffer="d['")
     nt.assert_in("my_head", matches)
     nt.assert_in("my_data", matches)
     # complete on a nested level
     with greedy_completion():
         ip.user_ns['d'] = numpy.zeros(2, dtype=dt)
         _, matches = complete(line_buffer="d[1]['my_head']['")
         nt.assert_true(any(["my_dt" in m for m in matches]))
         nt.assert_true(any(["my_df" in m for m in matches]))
 
 
 @dec.skip_without('pandas')
 def test_dataframe_key_completion():
     """Test dict key completion applies to pandas DataFrames"""
     import pandas
     ip = get_ipython()
     complete = ip.Completer.complete
     ip.user_ns['d'] = pandas.DataFrame({'hello': [1], 'world': [2]})
     _, matches = complete(line_buffer="d['")
     nt.assert_in("hello", matches)
     nt.assert_in("world", matches)
 
 
 def test_dict_key_completion_invalids():
     """Smoke test cases dict key completion can't handle"""
     ip = get_ipython()
     complete = ip.Completer.complete
 
     ip.user_ns['no_getitem'] = None
     ip.user_ns['no_keys'] = []
     ip.user_ns['cant_call_keys'] = dict
     ip.user_ns['empty'] = {}
     ip.user_ns['d'] = {'abc': 5}
 
     _, matches = complete(line_buffer="no_getitem['")
     _, matches = complete(line_buffer="no_keys['")
     _, matches = complete(line_buffer="cant_call_keys['")
     _, matches = complete(line_buffer="empty['")
     _, matches = complete(line_buffer="name_error['")
     _, matches = complete(line_buffer="d['\\")  # incomplete escape
 
 class KeyCompletable(object):
     def __init__(self, things=()):
         self.things = things
 
     def _ipython_key_completions_(self):
         return list(self.things)
 
 def test_object_key_completion():
     ip = get_ipython()
     ip.user_ns['key_completable'] = KeyCompletable(['qwerty', 'qwick'])
 
     _, matches = ip.Completer.complete(line_buffer="key_completable['qw")
     nt.assert_in('qwerty', matches)
     nt.assert_in('qwick', matches)
 
 
 def test_aimport_module_completer():
     ip = get_ipython()
     _, matches = ip.complete('i', '%aimport i')
     nt.assert_in('io', matches)
     nt.assert_not_in('int', matches)
 
 def test_nested_import_module_completer():
     ip = get_ipython()
     _, matches = ip.complete(None, 'import IPython.co', 17)
     nt.assert_in('IPython.core', matches)
     nt.assert_not_in('import IPython.core', matches)
     nt.assert_not_in('IPython.display', matches)
 
 def test_import_module_completer():
     ip = get_ipython()
     _, matches = ip.complete('i', 'import i')
     nt.assert_in('io', matches)
     nt.assert_not_in('int', matches)
 
 def test_from_module_completer():
     ip = get_ipython()
     _, matches = ip.complete('B', 'from io import B', 16)
     nt.assert_in('BytesIO', matches)
     nt.assert_not_in('BaseException', matches)

IPython/core/tests/test_oinspect.py

@@ -1,452 +1,456 b''
 """Tests for the object inspection functionality.
 """
 
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
 from __future__ import print_function
 
 import os
 import re
 import sys
 
 import nose.tools as nt
 
 from .. import oinspect
 from IPython.core.magic import (Magics, magics_class, line_magic,
                                 cell_magic, line_cell_magic,
                                 register_line_magic, register_cell_magic,
                                 register_line_cell_magic)
 from decorator import decorator
 from IPython.testing.decorators import skipif
 from IPython.testing.tools import AssertPrints
 from IPython.utils.path import compress_user
 from IPython.utils import py3compat
 from IPython.utils.signatures import Signature, Parameter
 
 
 #-----------------------------------------------------------------------------
 # Globals and constants
 #-----------------------------------------------------------------------------
 
 inspector = oinspect.Inspector()
 ip = get_ipython()
 
 #-----------------------------------------------------------------------------
 # Local utilities
 #-----------------------------------------------------------------------------
 
 # WARNING: since this test checks the line number where a function is
 # defined, if any code is inserted above, the following line will need to be
 # updated.  Do NOT insert any whitespace between the next line and the function
 # definition below.
 THIS_LINE_NUMBER = 43  # Put here the actual number of this line
-def test_find_source_lines():
-    nt.assert_equal(oinspect.find_source_lines(test_find_source_lines), 
-                    THIS_LINE_NUMBER+1)
+
+from unittest import TestCase
+
+class Test(TestCase):
+
+    def test_find_source_lines(self):
+        self.assertEqual(oinspect.find_source_lines(Test.test_find_source_lines), 
+                    THIS_LINE_NUMBER+6)
 
 
 # A couple of utilities to ensure these tests work the same from a source or a
 # binary install
 def pyfile(fname):
     return os.path.normcase(re.sub('.py[co]$', '.py', fname))
 
 
 def match_pyfiles(f1, f2):
     nt.assert_equal(pyfile(f1), pyfile(f2))
 
 
 def test_find_file():
     match_pyfiles(oinspect.find_file(test_find_file), os.path.abspath(__file__))
 
 
 def test_find_file_decorated1():
 
     @decorator
     def noop1(f):
         def wrapper():
             return f(*a, **kw)
         return wrapper
 
     @noop1
     def f(x):
         "My docstring"
     
     match_pyfiles(oinspect.find_file(f), os.path.abspath(__file__))
     nt.assert_equal(f.__doc__, "My docstring")
 
 
 def test_find_file_decorated2():
 
     @decorator
     def noop2(f, *a, **kw):
         return f(*a, **kw)
 
     @noop2
     @noop2
     @noop2
     def f(x):
         "My docstring 2"
     
     match_pyfiles(oinspect.find_file(f), os.path.abspath(__file__))
     nt.assert_equal(f.__doc__, "My docstring 2")
     
 
 def test_find_file_magic():
     run = ip.find_line_magic('run')
     nt.assert_not_equal(oinspect.find_file(run), None)
 
 
 # A few generic objects we can then inspect in the tests below
 
 class Call(object):
     """This is the class docstring."""
 
     def __init__(self, x, y=1):
         """This is the constructor docstring."""
 
     def __call__(self, *a, **kw):
         """This is the call docstring."""
 
     def method(self, x, z=2):
         """Some method's docstring"""
 
 class HasSignature(object):
     """This is the class docstring."""
     __signature__ = Signature([Parameter('test', Parameter.POSITIONAL_OR_KEYWORD)])
 
     def __init__(self, *args):
         """This is the init docstring"""
 
 
 class SimpleClass(object):
     def method(self, x, z=2):
         """Some method's docstring"""
 
 
 class OldStyle:
     """An old-style class for testing."""
     pass
 
 
 def f(x, y=2, *a, **kw):
     """A simple function."""
 
 
 def g(y, z=3, *a, **kw):
     pass  # no docstring
 
 
 @register_line_magic
 def lmagic(line):
     "A line magic"
 
 
 @register_cell_magic
 def cmagic(line, cell):
     "A cell magic"
 
 
 @register_line_cell_magic
 def lcmagic(line, cell=None):
     "A line/cell magic"
 
 
 @magics_class
 class SimpleMagics(Magics):
     @line_magic
     def Clmagic(self, cline):
         "A class-based line magic"
         
     @cell_magic
     def Ccmagic(self, cline, ccell):
         "A class-based cell magic"
         
     @line_cell_magic
     def Clcmagic(self, cline, ccell=None):
         "A class-based line/cell magic"
 
 
 class Awkward(object):
     def __getattr__(self, name):
         raise Exception(name)
 
 class NoBoolCall:
     """
     callable with `__bool__` raising should still be inspect-able.
     """
 
     def __call__(self):
         """does nothing"""
         pass
 
     def __bool__(self):
         """just raise NotImplemented"""
         raise NotImplementedError('Must be implemented')
 
 
 class SerialLiar(object):
     """Attribute accesses always get another copy of the same class.
 
     unittest.mock.call does something similar, but it's not ideal for testing
     as the failure mode is to eat all your RAM. This gives up after 10k levels.
     """
     def __init__(self, max_fibbing_twig, lies_told=0):
         if lies_told > 10000:
             raise RuntimeError('Nose too long, honesty is the best policy')
         self.max_fibbing_twig = max_fibbing_twig
         self.lies_told = lies_told
         max_fibbing_twig[0] = max(max_fibbing_twig[0], lies_told)
 
     def __getattr__(self, item):
         return SerialLiar(self.max_fibbing_twig, self.lies_told + 1)
 
 
 def check_calltip(obj, name, call, docstring):
     """Generic check pattern all calltip tests will use"""
     info = inspector.info(obj, name)
     call_line, ds = oinspect.call_tip(info)
     nt.assert_equal(call_line, call)
     nt.assert_equal(ds, docstring)
 
 #-----------------------------------------------------------------------------
 # Tests
 #-----------------------------------------------------------------------------
 
 def test_calltip_class():
     check_calltip(Call, 'Call', 'Call(x, y=1)', Call.__init__.__doc__)
 
 
 def test_calltip_instance():
     c = Call(1)
     check_calltip(c, 'c', 'c(*a, **kw)', c.__call__.__doc__)
 
 
 def test_calltip_method():
     c = Call(1)
     check_calltip(c.method, 'c.method', 'c.method(x, z=2)', c.method.__doc__)
 
 
 def test_calltip_function():
     check_calltip(f, 'f', 'f(x, y=2, *a, **kw)', f.__doc__)
 
 
 def test_calltip_function2():
     check_calltip(g, 'g', 'g(y, z=3, *a, **kw)', '<no docstring>')
 
 
 @skipif(sys.version_info >= (3, 5))
 def test_calltip_builtin():
     check_calltip(sum, 'sum', None, sum.__doc__)
 
 
 def test_calltip_line_magic():
     check_calltip(lmagic, 'lmagic', 'lmagic(line)', "A line magic")
 
         
 def test_calltip_cell_magic():
     check_calltip(cmagic, 'cmagic', 'cmagic(line, cell)', "A cell magic")
 
         
 def test_calltip_line_cell_magic():
     check_calltip(lcmagic, 'lcmagic', 'lcmagic(line, cell=None)', 
                   "A line/cell magic")
         
 
 def test_class_magics():
     cm = SimpleMagics(ip)
     ip.register_magics(cm)
     check_calltip(cm.Clmagic, 'Clmagic', 'Clmagic(cline)',
                   "A class-based line magic")
     check_calltip(cm.Ccmagic, 'Ccmagic', 'Ccmagic(cline, ccell)',
                   "A class-based cell magic")
     check_calltip(cm.Clcmagic, 'Clcmagic', 'Clcmagic(cline, ccell=None)',
                   "A class-based line/cell magic")
     
 
 def test_info():
     "Check that Inspector.info fills out various fields as expected."
     i = inspector.info(Call, oname='Call')
     nt.assert_equal(i['type_name'], 'type')
     expted_class = str(type(type))  # <class 'type'> (Python 3) or <type 'type'>
     nt.assert_equal(i['base_class'], expted_class)
     if sys.version_info > (3,):
         nt.assert_regex(i['string_form'], "<class 'IPython.core.tests.test_oinspect.Call'( at 0x[0-9a-f]{1,9})?>")
     fname = __file__
     if fname.endswith(".pyc"):
         fname = fname[:-1]
     # case-insensitive comparison needed on some filesystems
     # e.g. Windows:
     nt.assert_equal(i['file'].lower(), compress_user(fname).lower())
     nt.assert_equal(i['definition'], None)
     nt.assert_equal(i['docstring'], Call.__doc__)
     nt.assert_equal(i['source'], None)
     nt.assert_true(i['isclass'])
     _self_py2 = '' if py3compat.PY3 else 'self, '
     nt.assert_equal(i['init_definition'], "Call(%sx, y=1)" % _self_py2)
     nt.assert_equal(i['init_docstring'], Call.__init__.__doc__)
 
     i = inspector.info(Call, detail_level=1)
     nt.assert_not_equal(i['source'], None)
     nt.assert_equal(i['docstring'], None)
 
     c = Call(1)
     c.__doc__ = "Modified instance docstring"
     i = inspector.info(c)
     nt.assert_equal(i['type_name'], 'Call')
     nt.assert_equal(i['docstring'], "Modified instance docstring")
     nt.assert_equal(i['class_docstring'], Call.__doc__)
     nt.assert_equal(i['init_docstring'], Call.__init__.__doc__)
     nt.assert_equal(i['call_docstring'], Call.__call__.__doc__)
 
     # Test old-style classes, which for example may not have an __init__ method.
     if not py3compat.PY3:
         i = inspector.info(OldStyle)
         nt.assert_equal(i['type_name'], 'classobj')
 
         i = inspector.info(OldStyle())
         nt.assert_equal(i['type_name'], 'instance')
         nt.assert_equal(i['docstring'], OldStyle.__doc__)
 
 def test_class_signature():
     info = inspector.info(HasSignature, 'HasSignature')
     nt.assert_equal(info['init_definition'], "HasSignature(test)")
     nt.assert_equal(info['init_docstring'], HasSignature.__init__.__doc__)
 
 def test_info_awkward():
     # Just test that this doesn't throw an error.
     inspector.info(Awkward())
 
 def test_bool_raise():
     inspector.info(NoBoolCall())
 
 def test_info_serialliar():
     fib_tracker = [0]
-    i = inspector.info(SerialLiar(fib_tracker))
+    inspector.info(SerialLiar(fib_tracker))
 
     # Nested attribute access should be cut off at 100 levels deep to avoid
     # infinite loops: https://github.com/ipython/ipython/issues/9122
     nt.assert_less(fib_tracker[0], 9000)
 
 def test_calldef_none():
     # We should ignore __call__ for all of these.
     for obj in [f, SimpleClass().method, any, str.upper]:
         print(obj)
         i = inspector.info(obj)
         nt.assert_is(i['call_def'], None)
 
-if py3compat.PY3:
-    exec("def f_kwarg(pos, *, kwonly): pass")
+def f_kwarg(pos, *, kwonly):
+    pass
 
-@skipif(not py3compat.PY3)
 def test_definition_kwonlyargs():
     i = inspector.info(f_kwarg, oname='f_kwarg')  # analysis:ignore
     nt.assert_equal(i['definition'], "f_kwarg(pos, *, kwonly)")
 
 def test_getdoc():
     class A(object):
         """standard docstring"""
         pass
 
     class B(object):
         """standard docstring"""
         def getdoc(self):
             return "custom docstring"
     
     class C(object):
         """standard docstring"""
         def getdoc(self):
             return None
     
     a = A()
     b = B()
     c = C()
     
     nt.assert_equal(oinspect.getdoc(a), "standard docstring")
     nt.assert_equal(oinspect.getdoc(b), "custom docstring")
     nt.assert_equal(oinspect.getdoc(c), "standard docstring")
 
 
 def test_empty_property_has_no_source():
     i = inspector.info(property(), detail_level=1)
     nt.assert_is(i['source'], None)
 
 
 def test_property_sources():
     import zlib
 
     class A(object):
         @property
         def foo(self):
             return 'bar'
 
         foo = foo.setter(lambda self, v: setattr(self, 'bar', v))
 
         id = property(id)
         compress = property(zlib.compress)
 
     i = inspector.info(A.foo, detail_level=1)
     nt.assert_in('def foo(self):', i['source'])
     nt.assert_in('lambda self, v:', i['source'])
 
     i = inspector.info(A.id, detail_level=1)
     nt.assert_in('fget = <function id>', i['source'])
 
     i = inspector.info(A.compress, detail_level=1)
     nt.assert_in('fget = <function zlib.compress>', i['source'])
 
 
 def test_property_docstring_is_in_info_for_detail_level_0():
     class A(object):
         @property
         def foobar(self):
             """This is `foobar` property."""
             pass
 
     ip.user_ns['a_obj'] = A()
     nt.assert_equals(
         'This is `foobar` property.',
         ip.object_inspect('a_obj.foobar', detail_level=0)['docstring'])
 
     ip.user_ns['a_cls'] = A
     nt.assert_equals(
         'This is `foobar` property.',
         ip.object_inspect('a_cls.foobar', detail_level=0)['docstring'])
 
 
 def test_pdef():
     # See gh-1914
     def foo(): pass
     inspector.pdef(foo, 'foo')
 
 
 def test_pinfo_nonascii():
     # See gh-1177
     from . import nonascii2
     ip.user_ns['nonascii2'] = nonascii2
     ip._inspect('pinfo', 'nonascii2', detail_level=1)
 
 
 def test_pinfo_magic():
     with AssertPrints('Docstring:'):
         ip._inspect('pinfo', 'lsmagic', detail_level=0)
 
     with AssertPrints('Source:'):
         ip._inspect('pinfo', 'lsmagic', detail_level=1)
 
 
 def test_init_colors():
     # ensure colors are not present in signature info
     info = inspector.info(HasSignature)
     init_def = info['init_definition']
     nt.assert_not_in('[0m', init_def)
 
 
 def test_builtin_init():
     info = inspector.info(list)
     init_def = info['init_definition']
     # Python < 3.4 can't get init definition from builtins,
     # but still exercise the inspection in case of error-raising bugs.
     if sys.version_info >= (3,4):
         nt.assert_is_not_none(init_def)
 

IPython/core/ultratb.py

@@ -1,1500 +1,1489 b''
 # -*- coding: utf-8 -*-
 """
 Verbose and colourful traceback formatting.
 
 **ColorTB**
 
 I've always found it a bit hard to visually parse tracebacks in Python.  The
 ColorTB class is a solution to that problem.  It colors the different parts of a
 traceback in a manner similar to what you would expect from a syntax-highlighting
 text editor.
 
 Installation instructions for ColorTB::
 
     import sys,ultratb
     sys.excepthook = ultratb.ColorTB()
 
 **VerboseTB**
 
 I've also included a port of Ka-Ping Yee's "cgitb.py" that produces all kinds
 of useful info when a traceback occurs.  Ping originally had it spit out HTML
 and intended it for CGI programmers, but why should they have all the fun?  I
 altered it to spit out colored text to the terminal.  It's a bit overwhelming,
 but kind of neat, and maybe useful for long-running programs that you believe
 are bug-free.  If a crash *does* occur in that type of program you want details.
 Give it a shot--you'll love it or you'll hate it.
 
 .. note::
 
   The Verbose mode prints the variables currently visible where the exception
   happened (shortening their strings if too long). This can potentially be
   very slow, if you happen to have a huge data structure whose string
   representation is complex to compute. Your computer may appear to freeze for
   a while with cpu usage at 100%. If this occurs, you can cancel the traceback
   with Ctrl-C (maybe hitting it more than once).
 
   If you encounter this kind of situation often, you may want to use the
   Verbose_novars mode instead of the regular Verbose, which avoids formatting
   variables (but otherwise includes the information and context given by
   Verbose).
 
 .. note::
 
   The verbose mode print all variables in the stack, which means it can
   potentially leak sensitive information like access keys, or unencryted
   password.
 
 Installation instructions for VerboseTB::
 
     import sys,ultratb
     sys.excepthook = ultratb.VerboseTB()
 
 Note:  Much of the code in this module was lifted verbatim from the standard
 library module 'traceback.py' and Ka-Ping Yee's 'cgitb.py'.
 
 Color schemes
 -------------
 
 The colors are defined in the class TBTools through the use of the
 ColorSchemeTable class. Currently the following exist:
 
   - NoColor: allows all of this module to be used in any terminal (the color
     escapes are just dummy blank strings).
 
   - Linux: is meant to look good in a terminal like the Linux console (black
     or very dark background).
 
   - LightBG: similar to Linux but swaps dark/light colors to be more readable
     in light background terminals.
 
   - Neutral: a neutral color scheme that should be readable on both light and
     dark background
 
 You can implement other color schemes easily, the syntax is fairly
 self-explanatory. Please send back new schemes you develop to the author for
 possible inclusion in future releases.
 
 Inheritance diagram:
 
 .. inheritance-diagram:: IPython.core.ultratb
    :parts: 3
 """
 
 #*****************************************************************************
 # Copyright (C) 2001 Nathaniel Gray <[e-mail]>
 # Copyright (C) 2001-2004 Fernando Perez <[e-mail]>
 #
 # Distributed under the terms of the BSD License.  The full license is in
 # the file COPYING, distributed as part of this software.
 #*****************************************************************************
 
 from __future__ import absolute_import
 from __future__ import unicode_literals
 from __future__ import print_function
 
 import dis
 import inspect
 import keyword
 import linecache
 import os
 import pydoc
 import re
 import sys
 import time
 import tokenize
 import traceback
 import types
 
 try:  # Python 2
     generate_tokens = tokenize.generate_tokens
 except AttributeError:  # Python 3
     generate_tokens = tokenize.tokenize
 
 # For purposes of monkeypatching inspect to fix a bug in it.
 from inspect import getsourcefile, getfile, getmodule, \
     ismodule, isclass, ismethod, isfunction, istraceback, isframe, iscode
 
 # IPython's own modules
 from IPython import get_ipython
 from IPython.core import debugger
 from IPython.core.display_trap import DisplayTrap
 from IPython.core.excolors import exception_colors
 from IPython.utils import PyColorize
 from IPython.utils import openpy
 from IPython.utils import path as util_path
 from IPython.utils import py3compat
 from IPython.utils import ulinecache
 from IPython.utils.data import uniq_stable
 from IPython.utils.terminal import get_terminal_size
 from logging import info, error
 
 import IPython.utils.colorable as colorable
 
 # Globals
 # amount of space to put line numbers before verbose tracebacks
 INDENT_SIZE = 8
 
 # Default color scheme.  This is used, for example, by the traceback
 # formatter.  When running in an actual IPython instance, the user's rc.colors
 # value is used, but having a module global makes this functionality available
 # to users of ultratb who are NOT running inside ipython.
 DEFAULT_SCHEME = 'NoColor'
 
 # ---------------------------------------------------------------------------
 # Code begins
 
 # Utility functions
 def inspect_error():
     """Print a message about internal inspect errors.
 
     These are unfortunately quite common."""
 
     error('Internal Python error in the inspect module.\n'
           'Below is the traceback from this internal error.\n')
 
 
 # This function is a monkeypatch we apply to the Python inspect module. We have
 # now found when it's needed (see discussion on issue gh-1456), and we have a
 # test case (IPython.core.tests.test_ultratb.ChangedPyFileTest) that fails if
 # the monkeypatch is not applied. TK, Aug 2012.
 def findsource(object):
     """Return the entire source file and starting line number for an object.
 
     The argument may be a module, class, method, function, traceback, frame,
     or code object.  The source code is returned as a list of all the lines
     in the file and the line number indexes a line in that list.  An IOError
     is raised if the source code cannot be retrieved.
 
     FIXED version with which we monkeypatch the stdlib to work around a bug."""
 
     file = getsourcefile(object) or getfile(object)
     # If the object is a frame, then trying to get the globals dict from its
     # module won't work. Instead, the frame object itself has the globals
     # dictionary.
     globals_dict = None
     if inspect.isframe(object):
         # XXX: can this ever be false?
         globals_dict = object.f_globals
     else:
         module = getmodule(object, file)
         if module:
             globals_dict = module.__dict__
     lines = linecache.getlines(file, globals_dict)
     if not lines:
         raise IOError('could not get source code')
 
     if ismodule(object):
         return lines, 0
 
     if isclass(object):
         name = object.__name__
         pat = re.compile(r'^(\s*)class\s*' + name + r'\b')
         # make some effort to find the best matching class definition:
         # use the one with the least indentation, which is the one
         # that's most probably not inside a function definition.
         candidates = []
         for i in range(len(lines)):
             match = pat.match(lines[i])
             if match:
                 # if it's at toplevel, it's already the best one
                 if lines[i][0] == 'c':
                     return lines, i
                 # else add whitespace to candidate list
                 candidates.append((match.group(1), i))
         if candidates:
             # this will sort by whitespace, and by line number,
             # less whitespace first
             candidates.sort()
             return lines, candidates[0][1]
         else:
             raise IOError('could not find class definition')
 
     if ismethod(object):
         object = object.__func__
     if isfunction(object):
         object = object.__code__
     if istraceback(object):
         object = object.tb_frame
     if isframe(object):
         object = object.f_code
     if iscode(object):
         if not hasattr(object, 'co_firstlineno'):
             raise IOError('could not find function definition')
         pat = re.compile(r'^(\s*def\s)|(.*(?<!\w)lambda(:|\s))|^(\s*@)')
         pmatch = pat.match
         # fperez - fix: sometimes, co_firstlineno can give a number larger than
         # the length of lines, which causes an error.  Safeguard against that.
         lnum = min(object.co_firstlineno, len(lines)) - 1
         while lnum > 0:
             if pmatch(lines[lnum]):
                 break
             lnum -= 1
 
         return lines, lnum
     raise IOError('could not find code object')
 
 
 # This is a patched version of inspect.getargs that applies the (unmerged)
 # patch for http://bugs.python.org/issue14611 by Stefano Taschini.  This fixes
 # https://github.com/ipython/ipython/issues/8205 and
 # https://github.com/ipython/ipython/issues/8293
 def getargs(co):
     """Get information about the arguments accepted by a code object.
 
     Three things are returned: (args, varargs, varkw), where 'args' is
     a list of argument names (possibly containing nested lists), and
     'varargs' and 'varkw' are the names of the * and ** arguments or None."""
     if not iscode(co):
         raise TypeError('{!r} is not a code object'.format(co))
 
     nargs = co.co_argcount
     names = co.co_varnames
     args = list(names[:nargs])
     step = 0
 
     # The following acrobatics are for anonymous (tuple) arguments.
     for i in range(nargs):
         if args[i][:1] in ('', '.'):
             stack, remain, count = [], [], []
             while step < len(co.co_code):
                 op = ord(co.co_code[step])
                 step = step + 1
                 if op >= dis.HAVE_ARGUMENT:
                     opname = dis.opname[op]
                     value = ord(co.co_code[step]) + ord(co.co_code[step+1])*256
                     step = step + 2
                     if opname in ('UNPACK_TUPLE', 'UNPACK_SEQUENCE'):
                         remain.append(value)
                         count.append(value)
                     elif opname in ('STORE_FAST', 'STORE_DEREF'):
                         if op in dis.haslocal:
                             stack.append(co.co_varnames[value])
                         elif op in dis.hasfree:
                             stack.append((co.co_cellvars + co.co_freevars)[value])
                         # Special case for sublists of length 1: def foo((bar))
                         # doesn't generate the UNPACK_TUPLE bytecode, so if
                         # `remain` is empty here, we have such a sublist.
                         if not remain:
                             stack[0] = [stack[0]]
                             break
                         else:
                             remain[-1] = remain[-1] - 1
                             while remain[-1] == 0:
                                 remain.pop()
                                 size = count.pop()
                                 stack[-size:] = [stack[-size:]]
                                 if not remain:
                                     break
                                 remain[-1] = remain[-1] - 1
                             if not remain:
                                 break
             args[i] = stack[0]
 
     varargs = None
     if co.co_flags & inspect.CO_VARARGS:
         varargs = co.co_varnames[nargs]
         nargs = nargs + 1
     varkw = None
     if co.co_flags & inspect.CO_VARKEYWORDS:
         varkw = co.co_varnames[nargs]
     return inspect.Arguments(args, varargs, varkw)
 
 
 # Monkeypatch inspect to apply our bugfix.
 def with_patch_inspect(f):
     """decorator for monkeypatching inspect.findsource"""
 
     def wrapped(*args, **kwargs):
         save_findsource = inspect.findsource
         save_getargs = inspect.getargs
         inspect.findsource = findsource
         inspect.getargs = getargs
         try:
             return f(*args, **kwargs)
         finally:
             inspect.findsource = save_findsource
             inspect.getargs = save_getargs
 
     return wrapped
 
 
 if py3compat.PY3:
     fixed_getargvalues = inspect.getargvalues
 else:
     # Fixes for https://github.com/ipython/ipython/issues/8293
     #       and https://github.com/ipython/ipython/issues/8205.
     # The relevant bug is caused by failure to correctly handle anonymous tuple
     # unpacking, which only exists in Python 2.
     fixed_getargvalues = with_patch_inspect(inspect.getargvalues)
 
 
 def fix_frame_records_filenames(records):
     """Try to fix the filenames in each record from inspect.getinnerframes().
 
     Particularly, modules loaded from within zip files have useless filenames
     attached to their code object, and inspect.getinnerframes() just uses it.
     """
     fixed_records = []
     for frame, filename, line_no, func_name, lines, index in records:
         # Look inside the frame's globals dictionary for __file__,
         # which should be better. However, keep Cython filenames since
         # we prefer the source filenames over the compiled .so file.
         filename = py3compat.cast_unicode_py2(filename, "utf-8")
         if not filename.endswith(('.pyx', '.pxd', '.pxi')):
             better_fn = frame.f_globals.get('__file__', None)
             if isinstance(better_fn, str):
                 # Check the type just in case someone did something weird with
                 # __file__. It might also be None if the error occurred during
                 # import.
                 filename = better_fn
         fixed_records.append((frame, filename, line_no, func_name, lines, index))
     return fixed_records
 
 
 @with_patch_inspect
 def _fixed_getinnerframes(etb, context=1, tb_offset=0):
     LNUM_POS, LINES_POS, INDEX_POS = 2, 4, 5
 
     records = fix_frame_records_filenames(inspect.getinnerframes(etb, context))
     # If the error is at the console, don't build any context, since it would
     # otherwise produce 5 blank lines printed out (there is no file at the
     # console)
     rec_check = records[tb_offset:]
     try:
         rname = rec_check[0][1]
         if rname == '<ipython console>' or rname.endswith('<string>'):
             return rec_check
     except IndexError:
         pass
 
     aux = traceback.extract_tb(etb)
     assert len(records) == len(aux)
     for i, (file, lnum, _, _) in zip(range(len(records)), aux):
         maybeStart = lnum - 1 - context // 2
         start = max(maybeStart, 0)
         end = start + context
         lines = ulinecache.getlines(file)[start:end]
         buf = list(records[i])
         buf[LNUM_POS] = lnum
         buf[INDEX_POS] = lnum - 1 - start
         buf[LINES_POS] = lines
         records[i] = tuple(buf)
     return records[tb_offset:]
 
 # Helper function -- largely belongs to VerboseTB, but we need the same
 # functionality to produce a pseudo verbose TB for SyntaxErrors, so that they
 # can be recognized properly by ipython.el's py-traceback-line-re
 # (SyntaxErrors have to be treated specially because they have no traceback)
 
-_parser = PyColorize.Parser()
-
 
 def _format_traceback_lines(lnum, index, lines, Colors, lvals=None, scheme=None):
     numbers_width = INDENT_SIZE - 1
     res = []
     i = lnum - index
 
-    # This lets us get fully syntax-highlighted tracebacks.
-    if scheme is None:
-        ipinst = get_ipython()
-        if ipinst is not None:
-            scheme = ipinst.colors
-        else:
-            scheme = DEFAULT_SCHEME
-
-    _line_format = _parser.format2
+    _line_format = PyColorize.Parser(style=scheme).format2
 
     for line in lines:
         line = py3compat.cast_unicode(line)
 
-        new_line, err = _line_format(line, 'str', scheme)
+        new_line, err = _line_format(line, 'str')
         if not err: line = new_line
 
         if i == lnum:
             # This is the line with the error
             pad = numbers_width - len(str(i))
             num = '%s%s' % (debugger.make_arrow(pad), str(lnum))
             line = '%s%s%s %s%s' % (Colors.linenoEm, num,
                                     Colors.line, line, Colors.Normal)
         else:
             num = '%*s' % (numbers_width, i)
             line = '%s%s%s %s' % (Colors.lineno, num,
                                   Colors.Normal, line)
 
         res.append(line)
         if lvals and i == lnum:
             res.append(lvals + '\n')
         i = i + 1
     return res
 
 def is_recursion_error(etype, value, records):
     try:
         # RecursionError is new in Python 3.5
         recursion_error_type = RecursionError
     except NameError:
         recursion_error_type = RuntimeError
 
     # The default recursion limit is 1000, but some of that will be taken up
     # by stack frames in IPython itself. >500 frames probably indicates
     # a recursion error.
     return (etype is recursion_error_type) \
            and "recursion" in str(value).lower() \
            and len(records) > 500
 
 def find_recursion(etype, value, records):
     """Identify the repeating stack frames from a RecursionError traceback
 
     'records' is a list as returned by VerboseTB.get_records()
 
     Returns (last_unique, repeat_length)
     """
     # This involves a bit of guesswork - we want to show enough of the traceback
     # to indicate where the recursion is occurring. We guess that the innermost
     # quarter of the traceback (250 frames by default) is repeats, and find the
     # first frame (from in to out) that looks different.
     if not is_recursion_error(etype, value, records):
         return len(records), 0
 
     # Select filename, lineno, func_name to track frames with
     records = [r[1:4] for r in records]
     inner_frames = records[-(len(records)//4):]
     frames_repeated = set(inner_frames)
 
     last_seen_at = {}
     longest_repeat = 0
     i = len(records)
     for frame in reversed(records):
         i -= 1
         if frame not in frames_repeated:
             last_unique = i
             break
 
         if frame in last_seen_at:
             distance = last_seen_at[frame] - i
             longest_repeat = max(longest_repeat, distance)
 
         last_seen_at[frame] = i
     else:
         last_unique = 0 # The whole traceback was recursion
 
     return last_unique, longest_repeat
 
 #---------------------------------------------------------------------------
 # Module classes
 class TBTools(colorable.Colorable):
     """Basic tools used by all traceback printer classes."""
 
     # Number of frames to skip when reporting tracebacks
     tb_offset = 0
 
     def __init__(self, color_scheme='NoColor', call_pdb=False, ostream=None, parent=None, config=None):
         # Whether to call the interactive pdb debugger after printing
         # tracebacks or not
         super(TBTools, self).__init__(parent=parent, config=config)
         self.call_pdb = call_pdb
 
         # Output stream to write to.  Note that we store the original value in
         # a private attribute and then make the public ostream a property, so
         # that we can delay accessing sys.stdout until runtime.  The way
         # things are written now, the sys.stdout object is dynamically managed
         # so a reference to it should NEVER be stored statically.  This
         # property approach confines this detail to a single location, and all
         # subclasses can simply access self.ostream for writing.
         self._ostream = ostream
 
         # Create color table
         self.color_scheme_table = exception_colors()
 
         self.set_colors(color_scheme)
         self.old_scheme = color_scheme  # save initial value for toggles
 
         if call_pdb:
             self.pdb = debugger.Pdb()
         else:
             self.pdb = None
 
     def _get_ostream(self):
         """Output stream that exceptions are written to.
 
         Valid values are:
 
         - None: the default, which means that IPython will dynamically resolve
           to sys.stdout.  This ensures compatibility with most tools, including
           Windows (where plain stdout doesn't recognize ANSI escapes).
 
         - Any object with 'write' and 'flush' attributes.
         """
         return sys.stdout if self._ostream is None else self._ostream
 
     def _set_ostream(self, val):
         assert val is None or (hasattr(val, 'write') and hasattr(val, 'flush'))
         self._ostream = val
 
     ostream = property(_get_ostream, _set_ostream)
 
     def set_colors(self, *args, **kw):
         """Shorthand access to the color table scheme selector method."""
 
         # Set own color table
         self.color_scheme_table.set_active_scheme(*args, **kw)
         # for convenience, set Colors to the active scheme
         self.Colors = self.color_scheme_table.active_colors
         # Also set colors of debugger
         if hasattr(self, 'pdb') and self.pdb is not None:
             self.pdb.set_colors(*args, **kw)
 
     def color_toggle(self):
         """Toggle between the currently active color scheme and NoColor."""
 
         if self.color_scheme_table.active_scheme_name == 'NoColor':
             self.color_scheme_table.set_active_scheme(self.old_scheme)
             self.Colors = self.color_scheme_table.active_colors
         else:
             self.old_scheme = self.color_scheme_table.active_scheme_name
             self.color_scheme_table.set_active_scheme('NoColor')
             self.Colors = self.color_scheme_table.active_colors
 
     def stb2text(self, stb):
         """Convert a structured traceback (a list) to a string."""
         return '\n'.join(stb)
 
     def text(self, etype, value, tb, tb_offset=None, context=5):
         """Return formatted traceback.
 
         Subclasses may override this if they add extra arguments.
         """
         tb_list = self.structured_traceback(etype, value, tb,
                                             tb_offset, context)
         return self.stb2text(tb_list)
 
     def structured_traceback(self, etype, evalue, tb, tb_offset=None,
                              context=5, mode=None):
         """Return a list of traceback frames.
 
         Must be implemented by each class.
         """
         raise NotImplementedError()
 
 
 #---------------------------------------------------------------------------
 class ListTB(TBTools):
     """Print traceback information from a traceback list, with optional color.
 
     Calling requires 3 arguments: (etype, evalue, elist)
     as would be obtained by::
     
       etype, evalue, tb = sys.exc_info()
       if tb:
         elist = traceback.extract_tb(tb)
       else:
         elist = None
 
     It can thus be used by programs which need to process the traceback before
     printing (such as console replacements based on the code module from the
     standard library).
 
     Because they are meant to be called without a full traceback (only a
     list), instances of this class can't call the interactive pdb debugger."""
 
     def __init__(self, color_scheme='NoColor', call_pdb=False, ostream=None, parent=None):
         TBTools.__init__(self, color_scheme=color_scheme, call_pdb=call_pdb,
                          ostream=ostream, parent=parent)
 
     def __call__(self, etype, value, elist):
         self.ostream.flush()
         self.ostream.write(self.text(etype, value, elist))
         self.ostream.write('\n')
 
     def structured_traceback(self, etype, value, elist, tb_offset=None,
                              context=5):
         """Return a color formatted string with the traceback info.
 
         Parameters
         ----------
         etype : exception type
           Type of the exception raised.
 
         value : object
           Data stored in the exception
 
         elist : list
           List of frames, see class docstring for details.
 
         tb_offset : int, optional
           Number of frames in the traceback to skip.  If not given, the
           instance value is used (set in constructor).
 
         context : int, optional
           Number of lines of context information to print.
 
         Returns
         -------
         String with formatted exception.
         """
         tb_offset = self.tb_offset if tb_offset is None else tb_offset
         Colors = self.Colors
         out_list = []
         if elist:
 
             if tb_offset and len(elist) > tb_offset:
                 elist = elist[tb_offset:]
 
             out_list.append('Traceback %s(most recent call last)%s:' %
                             (Colors.normalEm, Colors.Normal) + '\n')
             out_list.extend(self._format_list(elist))
         # The exception info should be a single entry in the list.
         lines = ''.join(self._format_exception_only(etype, value))
         out_list.append(lines)
 
         # Note: this code originally read:
 
         ## for line in lines[:-1]:
         ##     out_list.append(" "+line)
         ## out_list.append(lines[-1])
 
         # This means it was indenting everything but the last line by a little
         # bit.  I've disabled this for now, but if we see ugliness somewhere we
         # can restore it.
 
         return out_list
 
     def _format_list(self, extracted_list):
         """Format a list of traceback entry tuples for printing.
 
         Given a list of tuples as returned by extract_tb() or
         extract_stack(), return a list of strings ready for printing.
         Each string in the resulting list corresponds to the item with the
         same index in the argument list.  Each string ends in a newline;
         the strings may contain internal newlines as well, for those items
         whose source text line is not None.
 
         Lifted almost verbatim from traceback.py
         """
 
         Colors = self.Colors
         list = []
         for filename, lineno, name, line in extracted_list[:-1]:
             item = '  File %s"%s"%s, line %s%d%s, in %s%s%s\n' % \
                    (Colors.filename, py3compat.cast_unicode_py2(filename, "utf-8"), Colors.Normal,
                     Colors.lineno, lineno, Colors.Normal,
                     Colors.name, py3compat.cast_unicode_py2(name, "utf-8"), Colors.Normal)
             if line:
                 item += '    %s\n' % line.strip()
             list.append(item)
         # Emphasize the last entry
         filename, lineno, name, line = extracted_list[-1]
         item = '%s  File %s"%s"%s, line %s%d%s, in %s%s%s%s\n' % \
                (Colors.normalEm,
                 Colors.filenameEm, py3compat.cast_unicode_py2(filename, "utf-8"), Colors.normalEm,
                 Colors.linenoEm, lineno, Colors.normalEm,
                 Colors.nameEm, py3compat.cast_unicode_py2(name, "utf-8"), Colors.normalEm,
                 Colors.Normal)
         if line:
             item += '%s    %s%s\n' % (Colors.line, line.strip(),
                                       Colors.Normal)
         list.append(item)
         return list
 
     def _format_exception_only(self, etype, value):
         """Format the exception part of a traceback.
 
         The arguments are the exception type and value such as given by
         sys.exc_info()[:2]. The return value is a list of strings, each ending
         in a newline.  Normally, the list contains a single string; however,
         for SyntaxError exceptions, it contains several lines that (when
         printed) display detailed information about where the syntax error
         occurred.  The message indicating which exception occurred is the
         always last string in the list.
 
         Also lifted nearly verbatim from traceback.py
         """
         have_filedata = False
         Colors = self.Colors
         list = []
         stype = py3compat.cast_unicode(Colors.excName + etype.__name__ + Colors.Normal)
         if value is None:
             # Not sure if this can still happen in Python 2.6 and above
             list.append(stype + '\n')
         else:
             if issubclass(etype, SyntaxError):
                 have_filedata = True
                 if not value.filename: value.filename = "<string>"
                 if value.lineno:
                     lineno = value.lineno
                     textline = ulinecache.getline(value.filename, value.lineno)
                 else:
                     lineno = 'unknown'
                     textline = ''
                 list.append('%s  File %s"%s"%s, line %s%s%s\n' % \
                             (Colors.normalEm,
                              Colors.filenameEm, py3compat.cast_unicode(value.filename), Colors.normalEm,
                              Colors.linenoEm, lineno, Colors.Normal  ))
                 if textline == '':
                     textline = py3compat.cast_unicode(value.text, "utf-8")
 
                 if textline is not None:
                     i = 0
                     while i < len(textline) and textline[i].isspace():
                         i += 1
                     list.append('%s    %s%s\n' % (Colors.line,
                                                   textline.strip(),
                                                   Colors.Normal))
                     if value.offset is not None:
                         s = '    '
                         for c in textline[i:value.offset - 1]:
                             if c.isspace():
                                 s += c
                             else:
                                 s += ' '
                         list.append('%s%s^%s\n' % (Colors.caret, s,
                                                    Colors.Normal))
 
             try:
                 s = value.msg
             except Exception:
                 s = self._some_str(value)
             if s:
                 list.append('%s%s:%s %s\n' % (stype, Colors.excName,
                                               Colors.Normal, s))
             else:
                 list.append('%s\n' % stype)
 
         # sync with user hooks
         if have_filedata:
             ipinst = get_ipython()
             if ipinst is not None:
                 ipinst.hooks.synchronize_with_editor(value.filename, value.lineno, 0)
 
         return list
 
     def get_exception_only(self, etype, value):
         """Only print the exception type and message, without a traceback.
 
         Parameters
         ----------
         etype : exception type
         value : exception value
         """
         return ListTB.structured_traceback(self, etype, value, [])
 
     def show_exception_only(self, etype, evalue):
         """Only print the exception type and message, without a traceback.
 
         Parameters
         ----------
         etype : exception type
         value : exception value
         """
         # This method needs to use __call__ from *this* class, not the one from
         # a subclass whose signature or behavior may be different
         ostream = self.ostream
         ostream.flush()
         ostream.write('\n'.join(self.get_exception_only(etype, evalue)))
         ostream.flush()
 
     def _some_str(self, value):
         # Lifted from traceback.py
         try:
             return py3compat.cast_unicode(str(value))
         except:
             return u'<unprintable %s object>' % type(value).__name__
 
 
 #----------------------------------------------------------------------------
 class VerboseTB(TBTools):
     """A port of Ka-Ping Yee's cgitb.py module that outputs color text instead
     of HTML.  Requires inspect and pydoc.  Crazy, man.
 
     Modified version which optionally strips the topmost entries from the
     traceback, to be used with alternate interpreters (because their own code
     would appear in the traceback)."""
 
     def __init__(self, color_scheme='Linux', call_pdb=False, ostream=None,
                  tb_offset=0, long_header=False, include_vars=True,
                  check_cache=None, debugger_cls = None):
         """Specify traceback offset, headers and color scheme.
 
         Define how many frames to drop from the tracebacks. Calling it with
         tb_offset=1 allows use of this handler in interpreters which will have
         their own code at the top of the traceback (VerboseTB will first
         remove that frame before printing the traceback info)."""
         TBTools.__init__(self, color_scheme=color_scheme, call_pdb=call_pdb,
                          ostream=ostream)
         self.tb_offset = tb_offset
         self.long_header = long_header
         self.include_vars = include_vars
         # By default we use linecache.checkcache, but the user can provide a
         # different check_cache implementation.  This is used by the IPython
         # kernel to provide tracebacks for interactive code that is cached,
         # by a compiler instance that flushes the linecache but preserves its
         # own code cache.
         if check_cache is None:
             check_cache = linecache.checkcache
         self.check_cache = check_cache
 
         self.debugger_cls = debugger_cls or debugger.Pdb
 
     def format_records(self, records, last_unique, recursion_repeat):
         """Format the stack frames of the traceback"""
         frames = []
         for r in records[:last_unique+recursion_repeat+1]:
             #print '*** record:',file,lnum,func,lines,index  # dbg
             frames.append(self.format_record(*r))
 
         if recursion_repeat:
             frames.append('... last %d frames repeated, from the frame below ...\n' % recursion_repeat)
             frames.append(self.format_record(*records[last_unique+recursion_repeat+1]))
 
         return frames
 
     def format_record(self, frame, file, lnum, func, lines, index):
         """Format a single stack frame"""
         Colors = self.Colors  # just a shorthand + quicker name lookup
         ColorsNormal = Colors.Normal  # used a lot
         col_scheme = self.color_scheme_table.active_scheme_name
         indent = ' ' * INDENT_SIZE
         em_normal = '%s\n%s%s' % (Colors.valEm, indent, ColorsNormal)
         undefined = '%sundefined%s' % (Colors.em, ColorsNormal)
         tpl_link = '%s%%s%s' % (Colors.filenameEm, ColorsNormal)
         tpl_call = 'in %s%%s%s%%s%s' % (Colors.vName, Colors.valEm,
                                         ColorsNormal)
         tpl_call_fail = 'in %s%%s%s(***failed resolving arguments***)%s' % \
                         (Colors.vName, Colors.valEm, ColorsNormal)
         tpl_local_var = '%s%%s%s' % (Colors.vName, ColorsNormal)
         tpl_global_var = '%sglobal%s %s%%s%s' % (Colors.em, ColorsNormal,
                                                  Colors.vName, ColorsNormal)
         tpl_name_val = '%%s %s= %%s%s' % (Colors.valEm, ColorsNormal)
 
         tpl_line = '%s%%s%s %%s' % (Colors.lineno, ColorsNormal)
         tpl_line_em = '%s%%s%s %%s%s' % (Colors.linenoEm, Colors.line,
                                          ColorsNormal)
 
         abspath = os.path.abspath
 
 
         if not file:
             file = '?'
         elif file.startswith(str("<")) and file.endswith(str(">")):
             # Not a real filename, no problem...
             pass
         elif not os.path.isabs(file):
             # Try to make the filename absolute by trying all
             # sys.path entries (which is also what linecache does)
             for dirname in sys.path:
                 try:
                     fullname = os.path.join(dirname, file)
                     if os.path.isfile(fullname):
                         file = os.path.abspath(fullname)
                         break
                 except Exception:
                     # Just in case that sys.path contains very
                     # strange entries...
                     pass
 
         file = py3compat.cast_unicode(file, util_path.fs_encoding)
         link = tpl_link % file
         args, varargs, varkw, locals = fixed_getargvalues(frame)
 
         if func == '?':
             call = ''
         else:
             # Decide whether to include variable details or not
             var_repr = self.include_vars and eqrepr or nullrepr
             try:
                 call = tpl_call % (func, inspect.formatargvalues(args,
                                                                  varargs, varkw,
                                                                  locals, formatvalue=var_repr))
             except KeyError:
                 # This happens in situations like errors inside generator
                 # expressions, where local variables are listed in the
                 # line, but can't be extracted from the frame.  I'm not
                 # 100% sure this isn't actually a bug in inspect itself,
                 # but since there's no info for us to compute with, the
                 # best we can do is report the failure and move on.  Here
                 # we must *not* call any traceback construction again,
                 # because that would mess up use of %debug later on.  So we
                 # simply report the failure and move on.  The only
                 # limitation will be that this frame won't have locals
                 # listed in the call signature.  Quite subtle problem...
                 # I can't think of a good way to validate this in a unit
                 # test, but running a script consisting of:
                 #  dict( (k,v.strip()) for (k,v) in range(10) )
                 # will illustrate the error, if this exception catch is
                 # disabled.
                 call = tpl_call_fail % func
 
         # Don't attempt to tokenize binary files.
         if file.endswith(('.so', '.pyd', '.dll')):
             return '%s %s\n' % (link, call)
 
         elif file.endswith(('.pyc', '.pyo')):
             # Look up the corresponding source file.
             try:
                 file = openpy.source_from_cache(file)
             except ValueError:
                 # Failed to get the source file for some reason
                 # E.g. https://github.com/ipython/ipython/issues/9486
                 return '%s %s\n' % (link, call)
 
         def linereader(file=file, lnum=[lnum], getline=ulinecache.getline):
             line = getline(file, lnum[0])
             lnum[0] += 1
             return line
 
         # Build the list of names on this line of code where the exception
         # occurred.
         try:
             names = []
             name_cont = False
 
             for token_type, token, start, end, line in generate_tokens(linereader):
                 # build composite names
                 if token_type == tokenize.NAME and token not in keyword.kwlist:
                     if name_cont:
                         # Continuation of a dotted name
                         try:
                             names[-1].append(token)
                         except IndexError:
                             names.append([token])
                         name_cont = False
                     else:
                         # Regular new names.  We append everything, the caller
                         # will be responsible for pruning the list later.  It's
                         # very tricky to try to prune as we go, b/c composite
                         # names can fool us.  The pruning at the end is easy
                         # to do (or the caller can print a list with repeated
                         # names if so desired.
                         names.append([token])
                 elif token == '.':
                     name_cont = True
                 elif token_type == tokenize.NEWLINE:
                     break
 
         except (IndexError, UnicodeDecodeError, SyntaxError):
             # signals exit of tokenizer
             # SyntaxError can occur if the file is not actually Python
             #  - see gh-6300
             pass
         except tokenize.TokenError as msg:
             _m = ("An unexpected error occurred while tokenizing input\n"
                   "The following traceback may be corrupted or invalid\n"
                   "The error message is: %s\n" % msg)
             error(_m)
 
         # Join composite names (e.g. "dict.fromkeys")
         names = ['.'.join(n) for n in names]
         # prune names list of duplicates, but keep the right order
         unique_names = uniq_stable(names)
 
         # Start loop over vars
         lvals = []
         if self.include_vars:
             for name_full in unique_names:
                 name_base = name_full.split('.', 1)[0]
                 if name_base in frame.f_code.co_varnames:
                     if name_base in locals:
                         try:
                             value = repr(eval(name_full, locals))
                         except:
                             value = undefined
                     else:
                         value = undefined
                     name = tpl_local_var % name_full
                 else:
                     if name_base in frame.f_globals:
                         try:
                             value = repr(eval(name_full, frame.f_globals))
                         except:
                             value = undefined
                     else:
                         value = undefined
                     name = tpl_global_var % name_full
                 lvals.append(tpl_name_val % (name, value))
         if lvals:
             lvals = '%s%s' % (indent, em_normal.join(lvals))
         else:
             lvals = ''
 
         level = '%s %s\n' % (link, call)
 
         if index is None:
             return level
         else:
             return '%s%s' % (level, ''.join(
                 _format_traceback_lines(lnum, index, lines, Colors, lvals,
                                         col_scheme)))
 
     def prepare_chained_exception_message(self, cause):
         direct_cause = "\nThe above exception was the direct cause of the following exception:\n"
         exception_during_handling = "\nDuring handling of the above exception, another exception occurred:\n"
 
         if cause:
             message = [[direct_cause]]
         else:
             message = [[exception_during_handling]]
         return message
 
     def prepare_header(self, etype, long_version=False):
         colors = self.Colors  # just a shorthand + quicker name lookup
         colorsnormal = colors.Normal  # used a lot
         exc = '%s%s%s' % (colors.excName, etype, colorsnormal)
         width = min(75, get_terminal_size()[0])
         if long_version:
             # Header with the exception type, python version, and date
             pyver = 'Python ' + sys.version.split()[0] + ': ' + sys.executable
             date = time.ctime(time.time())
 
             head = '%s%s%s\n%s%s%s\n%s' % (colors.topline, '-' * width, colorsnormal,
                                            exc, ' ' * (width - len(str(etype)) - len(pyver)),
                                            pyver, date.rjust(width) )
             head += "\nA problem occurred executing Python code.  Here is the sequence of function" \
                     "\ncalls leading up to the error, with the most recent (innermost) call last."
         else:
             # Simplified header
             head = '%s%s' % (exc, 'Traceback (most recent call last)'. \
                              rjust(width - len(str(etype))) )
 
         return head
 
     def format_exception(self, etype, evalue):
         colors = self.Colors  # just a shorthand + quicker name lookup
         colorsnormal = colors.Normal  # used a lot
         indent = ' ' * INDENT_SIZE
         # Get (safely) a string form of the exception info
         try:
             etype_str, evalue_str = map(str, (etype, evalue))
         except:
             # User exception is improperly defined.
             etype, evalue = str, sys.exc_info()[:2]
             etype_str, evalue_str = map(str, (etype, evalue))
         # ... and format it
         exception = ['%s%s%s: %s' % (colors.excName, etype_str,
                                      colorsnormal, py3compat.cast_unicode(evalue_str))]
 
         if (not py3compat.PY3) and type(evalue) is types.InstanceType:
             try:
                 names = [w for w in dir(evalue) if isinstance(w, py3compat.string_types)]
             except:
                 # Every now and then, an object with funny internals blows up
                 # when dir() is called on it.  We do the best we can to report
                 # the problem and continue
                 _m = '%sException reporting error (object with broken dir())%s:'
                 exception.append(_m % (colors.excName, colorsnormal))
                 etype_str, evalue_str = map(str, sys.exc_info()[:2])
                 exception.append('%s%s%s: %s' % (colors.excName, etype_str,
                                                  colorsnormal, py3compat.cast_unicode(evalue_str)))
                 names = []
             for name in names:
                 value = text_repr(getattr(evalue, name))
                 exception.append('\n%s%s = %s' % (indent, name, value))
 
         return exception
 
     def format_exception_as_a_whole(self, etype, evalue, etb, number_of_lines_of_context, tb_offset):
         """Formats the header, traceback and exception message for a single exception.
 
         This may be called multiple times by Python 3 exception chaining
         (PEP 3134).
         """
         # some locals
         orig_etype = etype
         try:
             etype = etype.__name__
         except AttributeError:
             pass
 
         tb_offset = self.tb_offset if tb_offset is None else tb_offset
         head = self.prepare_header(etype, self.long_header)
         records = self.get_records(etb, number_of_lines_of_context, tb_offset)
 
         if records is None:
             return ""
 
         last_unique, recursion_repeat = find_recursion(orig_etype, evalue, records)
 
         frames = self.format_records(records, last_unique, recursion_repeat)
 
         formatted_exception = self.format_exception(etype, evalue)
         if records:
             filepath, lnum = records[-1][1:3]
             filepath = os.path.abspath(filepath)
             ipinst = get_ipython()
             if ipinst is not None:
                 ipinst.hooks.synchronize_with_editor(filepath, lnum, 0)
 
         return [[head] + frames + [''.join(formatted_exception[0])]]
 
     def get_records(self, etb, number_of_lines_of_context, tb_offset):
         try:
             # Try the default getinnerframes and Alex's: Alex's fixes some
             # problems, but it generates empty tracebacks for console errors
             # (5 blanks lines) where none should be returned.
             return _fixed_getinnerframes(etb, number_of_lines_of_context, tb_offset)
         except UnicodeDecodeError:
             # This can occur if a file's encoding magic comment is wrong.
             # I can't see a way to recover without duplicating a bunch of code
             # from the stdlib traceback module. --TK
             error('\nUnicodeDecodeError while processing traceback.\n')
             return None
         except:
             # FIXME: I've been getting many crash reports from python 2.3
             # users, traceable to inspect.py.  If I can find a small test-case
             # to reproduce this, I should either write a better workaround or
             # file a bug report against inspect (if that's the real problem).
             # So far, I haven't been able to find an isolated example to
             # reproduce the problem.
             inspect_error()
             traceback.print_exc(file=self.ostream)
             info('\nUnfortunately, your original traceback can not be constructed.\n')
             return None
 
     def get_parts_of_chained_exception(self, evalue):
         def get_chained_exception(exception_value):
             cause = getattr(exception_value, '__cause__', None)
             if cause:
                 return cause
             if getattr(exception_value, '__suppress_context__', False):
                 return None
             return getattr(exception_value, '__context__', None)
 
         chained_evalue = get_chained_exception(evalue)
 
         if chained_evalue:
             return chained_evalue.__class__, chained_evalue, chained_evalue.__traceback__
 
     def structured_traceback(self, etype, evalue, etb, tb_offset=None,
                              number_of_lines_of_context=5):
         """Return a nice text document describing the traceback."""
 
         formatted_exception = self.format_exception_as_a_whole(etype, evalue, etb, number_of_lines_of_context,
                                                                tb_offset)
 
         colors = self.Colors  # just a shorthand + quicker name lookup
         colorsnormal = colors.Normal  # used a lot
         head = '%s%s%s' % (colors.topline, '-' * min(75, get_terminal_size()[0]), colorsnormal)
         structured_traceback_parts = [head]
         if py3compat.PY3:
             chained_exceptions_tb_offset = 0
             lines_of_context = 3
             formatted_exceptions = formatted_exception
             exception = self.get_parts_of_chained_exception(evalue)
             if exception:
                 formatted_exceptions += self.prepare_chained_exception_message(evalue.__cause__)
                 etype, evalue, etb = exception
             else:
                 evalue = None
             chained_exc_ids = set()
             while evalue:
                 formatted_exceptions += self.format_exception_as_a_whole(etype, evalue, etb, lines_of_context,
                                                                          chained_exceptions_tb_offset)
                 exception = self.get_parts_of_chained_exception(evalue)
 
                 if exception and not id(exception[1]) in chained_exc_ids:
                     chained_exc_ids.add(id(exception[1])) # trace exception to avoid infinite 'cause' loop
                     formatted_exceptions += self.prepare_chained_exception_message(evalue.__cause__)
                     etype, evalue, etb = exception
                 else:
                     evalue = None
 
             # we want to see exceptions in a reversed order:
             # the first exception should be on top
             for formatted_exception in reversed(formatted_exceptions):
                 structured_traceback_parts += formatted_exception
         else:
             structured_traceback_parts += formatted_exception[0]
 
         return structured_traceback_parts
 
     def debugger(self, force=False):
         """Call up the pdb debugger if desired, always clean up the tb
         reference.
 
         Keywords:
 
           - force(False): by default, this routine checks the instance call_pdb
             flag and does not actually invoke the debugger if the flag is false.
             The 'force' option forces the debugger to activate even if the flag
             is false.
 
         If the call_pdb flag is set, the pdb interactive debugger is
         invoked. In all cases, the self.tb reference to the current traceback
         is deleted to prevent lingering references which hamper memory
         management.
 
         Note that each call to pdb() does an 'import readline', so if your app
         requires a special setup for the readline completers, you'll have to
         fix that by hand after invoking the exception handler."""
 
         if force or self.call_pdb:
             if self.pdb is None:
-                self.pdb = self.debugger_cls(
-                    self.color_scheme_table.active_scheme_name)
+                self.pdb = self.debugger_cls()
             # the system displayhook may have changed, restore the original
             # for pdb
             display_trap = DisplayTrap(hook=sys.__displayhook__)
             with display_trap:
                 self.pdb.reset()
                 # Find the right frame so we don't pop up inside ipython itself
                 if hasattr(self, 'tb') and self.tb is not None:
                     etb = self.tb
                 else:
                     etb = self.tb = sys.last_traceback
                 while self.tb is not None and self.tb.tb_next is not None:
                     self.tb = self.tb.tb_next
                 if etb and etb.tb_next:
                     etb = etb.tb_next
                 self.pdb.botframe = etb.tb_frame
                 self.pdb.interaction(self.tb.tb_frame, self.tb)
 
         if hasattr(self, 'tb'):
             del self.tb
 
     def handler(self, info=None):
         (etype, evalue, etb) = info or sys.exc_info()
         self.tb = etb
         ostream = self.ostream
         ostream.flush()
         ostream.write(self.text(etype, evalue, etb))
         ostream.write('\n')
         ostream.flush()
 
     # Changed so an instance can just be called as VerboseTB_inst() and print
     # out the right info on its own.
     def __call__(self, etype=None, evalue=None, etb=None):
         """This hook can replace sys.excepthook (for Python 2.1 or higher)."""
         if etb is None:
             self.handler()
         else:
             self.handler((etype, evalue, etb))
         try:
             self.debugger()
         except KeyboardInterrupt:
             print("\nKeyboardInterrupt")
 
 
 #----------------------------------------------------------------------------
 class FormattedTB(VerboseTB, ListTB):
     """Subclass ListTB but allow calling with a traceback.
 
     It can thus be used as a sys.excepthook for Python > 2.1.
 
     Also adds 'Context' and 'Verbose' modes, not available in ListTB.
 
     Allows a tb_offset to be specified. This is useful for situations where
     one needs to remove a number of topmost frames from the traceback (such as
     occurs with python programs that themselves execute other python code,
     like Python shells).  """
 
     def __init__(self, mode='Plain', color_scheme='Linux', call_pdb=False,
                  ostream=None,
                  tb_offset=0, long_header=False, include_vars=False,
                  check_cache=None, debugger_cls=None):
 
         # NEVER change the order of this list. Put new modes at the end:
         self.valid_modes = ['Plain', 'Context', 'Verbose']
         self.verbose_modes = self.valid_modes[1:3]
 
         VerboseTB.__init__(self, color_scheme=color_scheme, call_pdb=call_pdb,
                            ostream=ostream, tb_offset=tb_offset,
                            long_header=long_header, include_vars=include_vars,
                            check_cache=check_cache, debugger_cls=debugger_cls)
 
         # Different types of tracebacks are joined with different separators to
         # form a single string.  They are taken from this dict
         self._join_chars = dict(Plain='', Context='\n', Verbose='\n')
         # set_mode also sets the tb_join_char attribute
         self.set_mode(mode)
 
     def _extract_tb(self, tb):
         if tb:
             return traceback.extract_tb(tb)
         else:
             return None
 
     def structured_traceback(self, etype, value, tb, tb_offset=None, number_of_lines_of_context=5):
         tb_offset = self.tb_offset if tb_offset is None else tb_offset
         mode = self.mode
         if mode in self.verbose_modes:
             # Verbose modes need a full traceback
             return VerboseTB.structured_traceback(
                 self, etype, value, tb, tb_offset, number_of_lines_of_context
             )
         else:
             # We must check the source cache because otherwise we can print
             # out-of-date source code.
             self.check_cache()
             # Now we can extract and format the exception
             elist = self._extract_tb(tb)
             return ListTB.structured_traceback(
                 self, etype, value, elist, tb_offset, number_of_lines_of_context
             )
 
     def stb2text(self, stb):
         """Convert a structured traceback (a list) to a string."""
         return self.tb_join_char.join(stb)
 
 
     def set_mode(self, mode=None):
         """Switch to the desired mode.
 
         If mode is not specified, cycles through the available modes."""
 
         if not mode:
             new_idx = (self.valid_modes.index(self.mode) + 1 ) % \
                       len(self.valid_modes)
             self.mode = self.valid_modes[new_idx]
         elif mode not in self.valid_modes:
             raise ValueError('Unrecognized mode in FormattedTB: <' + mode + '>\n'
                                                                             'Valid modes: ' + str(self.valid_modes))
         else:
             self.mode = mode
         # include variable details only in 'Verbose' mode
         self.include_vars = (self.mode == self.valid_modes[2])
         # Set the join character for generating text tracebacks
         self.tb_join_char = self._join_chars[self.mode]
 
     # some convenient shortcuts
     def plain(self):
         self.set_mode(self.valid_modes[0])
 
     def context(self):
         self.set_mode(self.valid_modes[1])
 
     def verbose(self):
         self.set_mode(self.valid_modes[2])
 
 
 #----------------------------------------------------------------------------
 class AutoFormattedTB(FormattedTB):
     """A traceback printer which can be called on the fly.
 
     It will find out about exceptions by itself.
 
     A brief example::
 
         AutoTB = AutoFormattedTB(mode = 'Verbose',color_scheme='Linux')
         try:
           ...
         except:
           AutoTB()  # or AutoTB(out=logfile) where logfile is an open file object
     """
 
     def __call__(self, etype=None, evalue=None, etb=None,
                  out=None, tb_offset=None):
         """Print out a formatted exception traceback.
 
         Optional arguments:
           - out: an open file-like object to direct output to.
 
           - tb_offset: the number of frames to skip over in the stack, on a
           per-call basis (this overrides temporarily the instance's tb_offset
           given at initialization time.  """
 
         if out is None:
             out = self.ostream
         out.flush()
         out.write(self.text(etype, evalue, etb, tb_offset))
         out.write('\n')
         out.flush()
         # FIXME: we should remove the auto pdb behavior from here and leave
         # that to the clients.
         try:
             self.debugger()
         except KeyboardInterrupt:
             print("\nKeyboardInterrupt")
 
     def structured_traceback(self, etype=None, value=None, tb=None,
                              tb_offset=None, number_of_lines_of_context=5):
         if etype is None:
             etype, value, tb = sys.exc_info()
         self.tb = tb
         return FormattedTB.structured_traceback(
             self, etype, value, tb, tb_offset, number_of_lines_of_context)
 
 
 #---------------------------------------------------------------------------
 
 # A simple class to preserve Nathan's original functionality.
 class ColorTB(FormattedTB):
     """Shorthand to initialize a FormattedTB in Linux colors mode."""
 
     def __init__(self, color_scheme='Linux', call_pdb=0, **kwargs):
         FormattedTB.__init__(self, color_scheme=color_scheme,
                              call_pdb=call_pdb, **kwargs)
 
 
 class SyntaxTB(ListTB):
     """Extension which holds some state: the last exception value"""
 
     def __init__(self, color_scheme='NoColor'):
         ListTB.__init__(self, color_scheme)
         self.last_syntax_error = None
 
     def __call__(self, etype, value, elist):
         self.last_syntax_error = value
 
         ListTB.__call__(self, etype, value, elist)
 
     def structured_traceback(self, etype, value, elist, tb_offset=None,
                              context=5):
         # If the source file has been edited, the line in the syntax error can
         # be wrong (retrieved from an outdated cache). This replaces it with
         # the current value.
         if isinstance(value, SyntaxError) \
                 and isinstance(value.filename, py3compat.string_types) \
                 and isinstance(value.lineno, int):
             linecache.checkcache(value.filename)
             newtext = ulinecache.getline(value.filename, value.lineno)
             if newtext:
                 value.text = newtext
         self.last_syntax_error = value
         return super(SyntaxTB, self).structured_traceback(etype, value, elist,
                                                           tb_offset=tb_offset, context=context)
 
     def clear_err_state(self):
         """Return the current error state and clear it"""
         e = self.last_syntax_error
         self.last_syntax_error = None
         return e
 
     def stb2text(self, stb):
         """Convert a structured traceback (a list) to a string."""
         return ''.join(stb)
 
 
 # some internal-use functions
 def text_repr(value):
     """Hopefully pretty robust repr equivalent."""
     # this is pretty horrible but should always return *something*
     try:
         return pydoc.text.repr(value)
     except KeyboardInterrupt:
         raise
     except:
         try:
             return repr(value)
         except KeyboardInterrupt:
             raise
         except:
             try:
                 # all still in an except block so we catch
                 # getattr raising
                 name = getattr(value, '__name__', None)
                 if name:
                     # ick, recursion
                     return text_repr(name)
                 klass = getattr(value, '__class__', None)
                 if klass:
                     return '%s instance' % text_repr(klass)
             except KeyboardInterrupt:
                 raise
             except:
                 return 'UNRECOVERABLE REPR FAILURE'
 
 
 def eqrepr(value, repr=text_repr):
     return '=%s' % repr(value)
 
 
 def nullrepr(value, repr=text_repr):
     return ''

IPython/utils/PyColorize.py

@@ -1,382 +1,332 b''
 # -*- coding: utf-8 -*-
 """
 Class and program to colorize python source code for ANSI terminals.
 
 Based on an HTML code highlighter by Jurgen Hermann found at:
 http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/52298
 
 Modifications by Fernando Perez ([e-mail]).
 
 Information on the original HTML highlighter follows:
 
 MoinMoin - Python Source Parser
 
 Title: Colorize Python source using the built-in tokenizer
 
 Submitter: Jurgen Hermann
 Last Updated:2001/04/06
 
 Version no:1.2
 
 Description:
 
 This code is part of MoinMoin (http://moin.sourceforge.net/) and converts
 Python source code to HTML markup, rendering comments, keywords,
 operators, numeric and string literals in different colors.
 
 It shows how to use the built-in keyword, token and tokenize modules to
 scan Python source code and re-emit it with no changes to its original
 formatting (which is the hard part).
 """
 from __future__ import print_function
 from __future__ import absolute_import
 from __future__ import unicode_literals
 
 __all__ = ['ANSICodeColors','Parser']
 
 _scheme_default = 'Linux'
 
 
 # Imports
 import keyword
 import os
 import sys
 import token
 import tokenize
 
 try:
     generate_tokens = tokenize.generate_tokens
 except AttributeError:
     # Python 3. Note that we use the undocumented _tokenize because it expects
     # strings, not bytes. See also Python issue #9969.
     generate_tokens = tokenize._tokenize
 
 from IPython.utils.coloransi import TermColors, InputTermColors ,ColorScheme, ColorSchemeTable
 from IPython.utils.py3compat import PY3
 
 from .colorable import Colorable
 
 if PY3:
     from io import StringIO
 else:
     from StringIO import StringIO
 
 #############################################################################
 ### Python Source Parser (does Hilighting)
 #############################################################################
 
 _KEYWORD = token.NT_OFFSET + 1
 _TEXT    = token.NT_OFFSET + 2
 
 #****************************************************************************
 # Builtin color schemes
 
 Colors = TermColors  # just a shorthand
 
 # Build a few color schemes
 NoColor = ColorScheme(
     'NoColor',{
     'header'         : Colors.NoColor,
     token.NUMBER     : Colors.NoColor,
     token.OP         : Colors.NoColor,
     token.STRING     : Colors.NoColor,
     tokenize.COMMENT : Colors.NoColor,
     token.NAME       : Colors.NoColor,
     token.ERRORTOKEN : Colors.NoColor,
 
     _KEYWORD         : Colors.NoColor,
     _TEXT            : Colors.NoColor,
 
     'in_prompt'      : InputTermColors.NoColor,  # Input prompt
     'in_number'      : InputTermColors.NoColor,  # Input prompt number
     'in_prompt2'     : InputTermColors.NoColor, # Continuation prompt
     'in_normal'      : InputTermColors.NoColor,  # color off (usu. Colors.Normal)
 
     'out_prompt'     : Colors.NoColor, # Output prompt
     'out_number'     : Colors.NoColor, # Output prompt number
 
     'normal'         : Colors.NoColor  # color off (usu. Colors.Normal)
     }  )
 
 LinuxColors = ColorScheme(
     'Linux',{
     'header'         : Colors.LightRed,
     token.NUMBER     : Colors.LightCyan,
     token.OP         : Colors.Yellow,
     token.STRING     : Colors.LightBlue,
     tokenize.COMMENT : Colors.LightRed,
     token.NAME       : Colors.Normal,
     token.ERRORTOKEN : Colors.Red,
 
     _KEYWORD         : Colors.LightGreen,
     _TEXT            : Colors.Yellow,
 
     'in_prompt'      : InputTermColors.Green,
     'in_number'      : InputTermColors.LightGreen,
     'in_prompt2'     : InputTermColors.Green,
     'in_normal'      : InputTermColors.Normal,  # color off (usu. Colors.Normal)
 
     'out_prompt'     : Colors.Red,
     'out_number'     : Colors.LightRed,
 
     'normal'         : Colors.Normal  # color off (usu. Colors.Normal)
     } )
 
 NeutralColors = ColorScheme(
     'Neutral',{
     'header'         : Colors.Red,
     token.NUMBER     : Colors.Cyan,
     token.OP         : Colors.Blue,
     token.STRING     : Colors.Blue,
     tokenize.COMMENT : Colors.Red,
     token.NAME       : Colors.Normal,
     token.ERRORTOKEN : Colors.Red,
 
     _KEYWORD         : Colors.Green,
     _TEXT            : Colors.Blue,
 
     'in_prompt'      : InputTermColors.Blue,
     'in_number'      : InputTermColors.LightBlue,
     'in_prompt2'     : InputTermColors.Blue,
     'in_normal'      : InputTermColors.Normal,  # color off (usu. Colors.Normal)
 
     'out_prompt'     : Colors.Red,
     'out_number'     : Colors.LightRed,
 
     'normal'         : Colors.Normal  # color off (usu. Colors.Normal)
     }  )
 
 # Hack: the 'neutral' colours are not very visible on a dark background on
 # Windows. Since Windows command prompts have a dark background by default, and
 # relatively few users are likely to alter that, we will use the 'Linux' colours,
 # designed for a dark background, as the default on Windows. Changing it here
 # avoids affecting the prompt colours rendered by prompt_toolkit, where the
 # neutral defaults do work OK.
 
 if os.name == 'nt':
     NeutralColors = LinuxColors.copy(name='Neutral')
 
 LightBGColors = ColorScheme(
     'LightBG',{
     'header'         : Colors.Red,
     token.NUMBER     : Colors.Cyan,
     token.OP         : Colors.Blue,
     token.STRING     : Colors.Blue,
     tokenize.COMMENT : Colors.Red,
     token.NAME       : Colors.Normal,
     token.ERRORTOKEN : Colors.Red,
 
 
     _KEYWORD         : Colors.Green,
     _TEXT            : Colors.Blue,
 
     'in_prompt'      : InputTermColors.Blue,
     'in_number'      : InputTermColors.LightBlue,
     'in_prompt2'     : InputTermColors.Blue,
     'in_normal'      : InputTermColors.Normal,  # color off (usu. Colors.Normal)
 
     'out_prompt'     : Colors.Red,
     'out_number'     : Colors.LightRed,
 
     'normal'         : Colors.Normal  # color off (usu. Colors.Normal)
     }  )
 
 # Build table of color schemes (needed by the parser)
 ANSICodeColors = ColorSchemeTable([NoColor,LinuxColors,LightBGColors, NeutralColors],
                                   _scheme_default)
 
+Undefined = object()
+
 class Parser(Colorable):
     """ Format colored Python source.
     """
 
     def __init__(self, color_table=None, out = sys.stdout, parent=None, style=None):
         """ Create a parser with a specified color table and output channel.
 
         Call format() to process code.
         """
 
         super(Parser, self).__init__(parent=parent)
 
         self.color_table = color_table and color_table or ANSICodeColors
         self.out = out
+        if not style:
+            self.style = self.default_style
+        else:
+            self.style = style
+
 
-    def format(self, raw, out = None, scheme = ''):
-        return self.format2(raw, out, scheme)[0]
+    def format(self, raw, out=None, scheme=Undefined):
+        import warnings
+        if scheme is not Undefined:
+            warnings.warn('The `scheme` argument of IPython.utils.PyColorize:Parser.format is deprecated since IPython 6.0.'
+                          'It will have no effect. Set the parser `style` directly.',
+                          stacklevel=2)
+        return self.format2(raw, out)[0]
 
-    def format2(self, raw, out = None, scheme = ''):
+    def format2(self, raw, out = None):
         """ Parse and send the colored source.
 
         If out and scheme are not specified, the defaults (given to
         constructor) are used.
 
         out should be a file-type object. Optionally, out can be given as the
         string 'str' and the parser will automatically return the output in a
         string."""
 
         string_output = 0
         if out == 'str' or self.out == 'str' or \
            isinstance(self.out,StringIO):
             # XXX - I don't really like this state handling logic, but at this
             # point I don't want to make major changes, so adding the
             # isinstance() check is the simplest I can do to ensure correct
             # behavior.
             out_old = self.out
             self.out = StringIO()
             string_output = 1
         elif out is not None:
             self.out = out
 
         # Fast return of the unmodified input for NoColor scheme
-        if scheme == 'NoColor':
+        if self.style == 'NoColor':
             error = False
             self.out.write(raw)
             if string_output:
                 return raw,error
             else:
                 return None,error
 
         # local shorthands
-        colors = self.color_table[scheme].colors
+        colors = self.color_table[self.style].colors
         self.colors = colors # put in object so __call__ sees it
 
         # Remove trailing whitespace and normalize tabs
         self.raw = raw.expandtabs().rstrip()
 
         # store line offsets in self.lines
         self.lines = [0, 0]
         pos = 0
         raw_find = self.raw.find
         lines_append = self.lines.append
         while 1:
             pos = raw_find('\n', pos) + 1
             if not pos: break
             lines_append(pos)
         lines_append(len(self.raw))
 
         # parse the source and write it
         self.pos = 0
         text = StringIO(self.raw)
 
         error = False
         try:
             for atoken in generate_tokens(text.readline):
                 self(*atoken)
         except tokenize.TokenError as ex:
             msg = ex.args[0]
             line = ex.args[1][0]
             self.out.write("%s\n\n*** ERROR: %s%s%s\n" %
                            (colors[token.ERRORTOKEN],
                             msg, self.raw[self.lines[line]:],
                             colors.normal)
                            )
             error = True
         self.out.write(colors.normal+'\n')
         if string_output:
             output = self.out.getvalue()
             self.out = out_old
             return (output, error)
         return (None, error)
 
     def __call__(self, toktype, toktext, start_pos, end_pos, line):
         """ Token handler, with syntax highlighting."""
         (srow,scol) = start_pos
         (erow,ecol) = end_pos
         colors = self.colors
         owrite = self.out.write
 
         # line separator, so this works across platforms
         linesep = os.linesep
 
         # calculate new positions
         oldpos = self.pos
         newpos = self.lines[srow] + scol
         self.pos = newpos + len(toktext)
 
         # send the original whitespace, if needed
         if newpos > oldpos:
             owrite(self.raw[oldpos:newpos])
 
         # skip indenting tokens
         if toktype in [token.INDENT, token.DEDENT]:
             self.pos = newpos
             return
 
         # map token type to a color group
         if token.LPAR <= toktype <= token.OP:
             toktype = token.OP
         elif toktype == token.NAME and keyword.iskeyword(toktext):
             toktype = _KEYWORD
         color = colors.get(toktype, colors[_TEXT])
 
         #print '<%s>' % toktext,    # dbg
 
         # Triple quoted strings must be handled carefully so that backtracking
         # in pagers works correctly. We need color terminators on _each_ line.
         if linesep in toktext:
             toktext = toktext.replace(linesep, '%s%s%s' %
                                       (colors.normal,linesep,color))
 
         # send text
         owrite('%s%s%s' % (color,toktext,colors.normal))
-
-def main(argv=None):
-    """Run as a command-line script: colorize a python file or stdin using ANSI
-    color escapes and print to stdout.
-
-    Inputs:
-
-      - argv(None): a list of strings like sys.argv[1:] giving the command-line
-        arguments. If None, use sys.argv[1:].
-    """
-
-    usage_msg = """%prog [options] [filename]
-
-Colorize a python file or stdin using ANSI color escapes and print to stdout.
-If no filename is given, or if filename is -, read standard input."""
-
-    import optparse
-    parser = optparse.OptionParser(usage=usage_msg)
-    newopt = parser.add_option
-    newopt('-s','--scheme',metavar='NAME',dest='scheme_name',action='store',
-           choices=['Linux','LightBG','NoColor'],default=_scheme_default,
-           help="give the color scheme to use. Currently only 'Linux'\
- (default) and 'LightBG' and 'NoColor' are implemented (give without\
- quotes)")
-
-    opts,args = parser.parse_args(argv)
-
-    if len(args) > 1:
-        parser.error("you must give at most one filename.")
-
-    if len(args) == 0:
-        fname = '-' # no filename given; setup to read from stdin
-    else:
-        fname = args[0]
-
-    if fname == '-':
-        stream = sys.stdin
-    else:
-        try:
-            stream = open(fname)
-        except IOError as msg:
-            print(msg, file=sys.stderr)
-            sys.exit(1)
-
-    parser = Parser()
-
-    # we need nested try blocks because pre-2.5 python doesn't support unified
-    # try-except-finally
-    try:
-        try:
-            # write colorized version to stdout
-            parser.format(stream.read(),scheme=opts.scheme_name)
-        except IOError as msg:
-            # if user reads through a pager and quits, don't print traceback
-            if msg.args != (32,'Broken pipe'):
-                raise
-    finally:
-        if stream is not sys.stdin:
-            stream.close() # in case a non-handled exception happened above
-
-if __name__ == "__main__":
-    main()

IPython/utils/colorable.py

@@ -1,26 +1,26 b''
 #*****************************************************************************
 # Copyright (C) 2016 The IPython Team <[e-mail]>
 #
 # Distributed under the terms of the BSD License.  The full license is in
 # the file COPYING, distributed as part of this software.
 #*****************************************************************************
 from __future__ import absolute_import
 
 """
 Color managing related utilities
 """
 
 import pygments
 
 from traitlets.config import Configurable
 from traitlets import Unicode
 
 
 available_themes = lambda : [s for s in pygments.styles.get_all_styles()]+['NoColor','LightBG','Linux', 'Neutral']
 
 class Colorable(Configurable):
     """
     A subclass of configurable for all the classes that have a `default_scheme`
     """
-    default_style=Unicode('lightbg').tag(config=True)
+    default_style=Unicode('LightBG').tag(config=True)
 

IPython/utils/tests/test_pycolorize.py

@@ -1,78 +1,78 b''
 # coding: utf-8
 """Test suite for our color utilities.
 
 Authors
 -------
 
 * Min RK
 """
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING.txt, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 # third party
 import nose.tools as nt
 
 # our own
 from IPython.utils.PyColorize import Parser
 import io
 
 #-----------------------------------------------------------------------------
 # Test functions
 #-----------------------------------------------------------------------------
 
 sample = u"""
 def function(arg, *args, kwarg=True, **kwargs):
     '''
     this is docs
     '''
     pass is True
     False == None
 
     with io.open(ru'unicode'):
         raise ValueError("\n escape \r sequence")
 
     print("wěird ünicoðe")
 
 class Bar(Super):
 
     def __init__(self):
         super(Bar, self).__init__(1**2, 3^4, 5 or 6)
 """
 
 def test_loop_colors():
 
-    for scheme in ('Linux', 'NoColor','LightBG', 'Neutral'):
+    for style in ('Linux', 'NoColor','LightBG', 'Neutral'):
 
         def test_unicode_colorize():
-            p = Parser()
-            f1 = p.format('1/0', 'str', scheme=scheme)
-            f2 = p.format(u'1/0', 'str', scheme=scheme)
+            p = Parser(style=style)
+            f1 = p.format('1/0', 'str')
+            f2 = p.format(u'1/0', 'str')
             nt.assert_equal(f1, f2)
 
         def test_parse_sample():
             """and test writing to a buffer"""
             buf = io.StringIO()
-            p = Parser()
-            p.format(sample, buf, scheme=scheme)
+            p = Parser(style=style)
+            p.format(sample, buf)
             buf.seek(0)
             f1 = buf.read()
 
             nt.assert_not_in('ERROR', f1)
 
         def test_parse_error():
-            p = Parser()
-            f1 = p.format(')', 'str', scheme=scheme)
-            if scheme != 'NoColor':
+            p = Parser(style=style)
+            f1 = p.format(')', 'str')
+            if style != 'NoColor':
                 nt.assert_in('ERROR', f1)
 
         yield test_unicode_colorize
         yield test_parse_sample
         yield test_parse_error