Merge pull request #12110 from terrdavis/remove-py2-workarounds-1of2

Remove py2 workarounds 1/2

IPython/core/oinspect.py

@@ -1,1032 +1,1031
 # -*- coding: utf-8 -*-
 """Tools for inspecting Python objects.
 
 Uses syntax highlighting for presenting the various information elements.
 
 Similar in spirit to the inspect module, but all calls take a name argument to
 reference the name under which an object is being read.
 """
 
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
 __all__ = ['Inspector','InspectColors']
 
 # stdlib modules
 import ast
 import inspect
 from inspect import signature
 import linecache
 import warnings
 import os
 from textwrap import dedent
 import types
 import io as stdlib_io
 
 from typing import Union
 
 # IPython's own
 from IPython.core import page
 from IPython.lib.pretty import pretty
 from IPython.testing.skipdoctest import skip_doctest
 from IPython.utils import PyColorize
 from IPython.utils import openpy
 from IPython.utils import py3compat
 from IPython.utils.dir2 import safe_hasattr
 from IPython.utils.path import compress_user
 from IPython.utils.text import indent
 from IPython.utils.wildcard import list_namespace
 from IPython.utils.wildcard import typestr2type
 from IPython.utils.coloransi import TermColors, ColorScheme, ColorSchemeTable
 from IPython.utils.py3compat import cast_unicode
 from IPython.utils.colorable import Colorable
 from IPython.utils.decorators import undoc
 
 from pygments import highlight
 from pygments.lexers import PythonLexer
 from pygments.formatters import HtmlFormatter
 
 def pylight(code):
     return highlight(code, PythonLexer(), HtmlFormatter(noclasses=True))
 
 # builtin docstrings to ignore
 _func_call_docstring = types.FunctionType.__call__.__doc__
 _object_init_docstring = object.__init__.__doc__
 _builtin_type_docstrings = {
     inspect.getdoc(t) for t in (types.ModuleType, types.MethodType,
                                 types.FunctionType, property)
 }
 
 _builtin_func_type = type(all)
 _builtin_meth_type = type(str.upper)  # Bound methods have the same type as builtin functions
 #****************************************************************************
 # Builtin color schemes
 
 Colors = TermColors  # just a shorthand
 
 InspectColors = PyColorize.ANSICodeColors
 
 #****************************************************************************
 # Auxiliary functions and objects
 
 # See the messaging spec for the definition of all these fields.  This list
 # effectively defines the order of display
 info_fields = ['type_name', 'base_class', 'string_form', 'namespace',
                'length', 'file', 'definition', 'docstring', 'source',
                'init_definition', 'class_docstring', 'init_docstring',
                'call_def', 'call_docstring',
                # These won't be printed but will be used to determine how to
                # format the object
                'ismagic', 'isalias', 'isclass', 'found', 'name'
                ]
 
 
 def object_info(**kw):
     """Make an object info dict with all fields present."""
     infodict = {k:None for k in info_fields}
     infodict.update(kw)
     return infodict
 
 
 def get_encoding(obj):
     """Get encoding for python source file defining obj
 
     Returns None if obj is not defined in a sourcefile.
     """
     ofile = find_file(obj)
     # run contents of file through pager starting at line where the object
     # is defined, as long as the file isn't binary and is actually on the
     # filesystem.
     if ofile is None:
         return None
     elif ofile.endswith(('.so', '.dll', '.pyd')):
         return None
     elif not os.path.isfile(ofile):
         return None
     else:
         # Print only text files, not extension binaries.  Note that
         # getsourcelines returns lineno with 1-offset and page() uses
         # 0-offset, so we must adjust.
         with stdlib_io.open(ofile, 'rb') as buffer:   # Tweaked to use io.open for Python 2
             encoding, lines = openpy.detect_encoding(buffer.readline)
         return encoding
 
 def getdoc(obj) -> Union[str,None]:
     """Stable wrapper around inspect.getdoc.
 
     This can't crash because of attribute problems.
 
     It also attempts to call a getdoc() method on the given object.  This
     allows objects which provide their docstrings via non-standard mechanisms
     (like Pyro proxies) to still be inspected by ipython's ? system.
     """
     # Allow objects to offer customized documentation via a getdoc method:
     try:
         ds = obj.getdoc()
     except Exception:
         pass
     else:
         if isinstance(ds, str):
             return inspect.cleandoc(ds)
     docstr = inspect.getdoc(obj)
     return docstr
 
 
 def getsource(obj, oname='') -> Union[str,None]:
     """Wrapper around inspect.getsource.
 
     This can be modified by other projects to provide customized source
     extraction.
 
     Parameters
     ----------
     obj : object
         an object whose source code we will attempt to extract
     oname : str
         (optional) a name under which the object is known
 
     Returns
     -------
     src : unicode or None
 
     """
 
     if isinstance(obj, property):
         sources = []
         for attrname in ['fget', 'fset', 'fdel']:
             fn = getattr(obj, attrname)
             if fn is not None:
                 encoding = get_encoding(fn)
                 oname_prefix = ('%s.' % oname) if oname else ''
                 sources.append(''.join(('# ', oname_prefix, attrname)))
                 if inspect.isfunction(fn):
                     sources.append(dedent(getsource(fn)))
                 else:
                     # Default str/repr only prints function name,
                     # pretty.pretty prints module name too.
                     sources.append(
                         '%s%s = %s\n' % (oname_prefix, attrname, pretty(fn))
                     )
         if sources:
             return '\n'.join(sources)
         else:
             return None
 
     else:
         # Get source for non-property objects.
 
         obj = _get_wrapped(obj)
 
         try:
             src = inspect.getsource(obj)
         except TypeError:
             # The object itself provided no meaningful source, try looking for
             # its class definition instead.
             if hasattr(obj, '__class__'):
                 try:
                     src = inspect.getsource(obj.__class__)
                 except TypeError:
                     return None
 
         return src
 
 
 def is_simple_callable(obj):
     """True if obj is a function ()"""
     return (inspect.isfunction(obj) or inspect.ismethod(obj) or \
             isinstance(obj, _builtin_func_type) or isinstance(obj, _builtin_meth_type))
 
 @undoc
 def getargspec(obj):
-    """Wrapper around :func:`inspect.getfullargspec` on Python 3, and
-    :func:inspect.getargspec` on Python 2.
+    """Wrapper around :func:`inspect.getfullargspec`
 
     In addition to functions and methods, this can also handle objects with a
     ``__call__`` attribute.
 
     DEPRECATED: Deprecated since 7.10. Do not use, will be removed.
     """
 
     warnings.warn('`getargspec` function is deprecated as of IPython 7.10'
                   'and will be removed in future versions.', DeprecationWarning, stacklevel=2)
 
     if safe_hasattr(obj, '__call__') and not is_simple_callable(obj):
         obj = obj.__call__
 
     return inspect.getfullargspec(obj)
 
 @undoc
 def format_argspec(argspec):
     """Format argspect, convenience wrapper around inspect's.
 
     This takes a dict instead of ordered arguments and calls
     inspect.format_argspec with the arguments in the necessary order.
 
     DEPRECATED: Do not use; will be removed in future versions.
     """
     
     warnings.warn('`format_argspec` function is deprecated as of IPython 7.10'
                   'and will be removed in future versions.', DeprecationWarning, stacklevel=2)
 
 
     return inspect.formatargspec(argspec['args'], argspec['varargs'],
                                  argspec['varkw'], argspec['defaults'])
 
 @undoc
 def call_tip(oinfo, format_call=True):
     """DEPRECATED. Extract call tip data from an oinfo dict.
     """
     warnings.warn('`call_tip` function is deprecated as of IPython 6.0'
                   'and will be removed in future versions.', DeprecationWarning, stacklevel=2)
     # Get call definition
     argspec = oinfo.get('argspec')
     if argspec is None:
         call_line = None
     else:
         # Callable objects will have 'self' as their first argument, prune
         # it out if it's there for clarity (since users do *not* pass an
         # extra first argument explicitly).
         try:
             has_self = argspec['args'][0] == 'self'
         except (KeyError, IndexError):
             pass
         else:
             if has_self:
                 argspec['args'] = argspec['args'][1:]
 
         call_line = oinfo['name']+format_argspec(argspec)
 
     # Now get docstring.
     # The priority is: call docstring, constructor docstring, main one.
     doc = oinfo.get('call_docstring')
     if doc is None:
         doc = oinfo.get('init_docstring')
     if doc is None:
         doc = oinfo.get('docstring','')
 
     return call_line, doc
 
 
 def _get_wrapped(obj):
     """Get the original object if wrapped in one or more @decorators
 
     Some objects automatically construct similar objects on any unrecognised
     attribute access (e.g. unittest.mock.call). To protect against infinite loops,
     this will arbitrarily cut off after 100 levels of obj.__wrapped__
     attribute access. --TK, Jan 2016
     """
     orig_obj = obj
     i = 0
     while safe_hasattr(obj, '__wrapped__'):
         obj = obj.__wrapped__
         i += 1
         if i > 100:
             # __wrapped__ is probably a lie, so return the thing we started with
             return orig_obj
     return obj
 
 def find_file(obj) -> str:
     """Find the absolute path to the file where an object was defined.
 
     This is essentially a robust wrapper around `inspect.getabsfile`.
 
     Returns None if no file can be found.
 
     Parameters
     ----------
     obj : any Python object
 
     Returns
     -------
     fname : str
       The absolute path to the file where the object was defined.
     """
     obj = _get_wrapped(obj)
 
     fname = None
     try:
         fname = inspect.getabsfile(obj)
     except TypeError:
         # For an instance, the file that matters is where its class was
         # declared.
         if hasattr(obj, '__class__'):
             try:
                 fname = inspect.getabsfile(obj.__class__)
             except TypeError:
                 # Can happen for builtins
                 pass
     except:
         pass
     return cast_unicode(fname)
 
 
 def find_source_lines(obj):
     """Find the line number in a file where an object was defined.
 
     This is essentially a robust wrapper around `inspect.getsourcelines`.
 
     Returns None if no file can be found.
 
     Parameters
     ----------
     obj : any Python object
 
     Returns
     -------
     lineno : int
       The line number where the object definition starts.
     """
     obj = _get_wrapped(obj)
 
     try:
         try:
             lineno = inspect.getsourcelines(obj)[1]
         except TypeError:
             # For instances, try the class object like getsource() does
             if hasattr(obj, '__class__'):
                 lineno = inspect.getsourcelines(obj.__class__)[1]
             else:
                 lineno = None
     except:
         return None
 
     return lineno
 
 class Inspector(Colorable):
 
     def __init__(self, color_table=InspectColors,
                  code_color_table=PyColorize.ANSICodeColors,
                  scheme=None,
                  str_detail_level=0,
                  parent=None, config=None):
         super(Inspector, self).__init__(parent=parent, config=config)
         self.color_table = color_table
         self.parser = PyColorize.Parser(out='str', parent=self, style=scheme)
         self.format = self.parser.format
         self.str_detail_level = str_detail_level
         self.set_active_scheme(scheme)
 
     def _getdef(self,obj,oname='') -> Union[str,None]:
         """Return the call signature for any callable object.
 
         If any exception is generated, None is returned instead and the
         exception is suppressed."""
         try:
             return _render_signature(signature(obj), oname)
         except:
             return None
 
     def __head(self,h) -> str:
         """Return a header string with proper colors."""
         return '%s%s%s' % (self.color_table.active_colors.header,h,
                            self.color_table.active_colors.normal)
 
     def set_active_scheme(self, scheme):
         if scheme is not None:
             self.color_table.set_active_scheme(scheme)
             self.parser.color_table.set_active_scheme(scheme)
 
     def noinfo(self, msg, oname):
         """Generic message when no information is found."""
         print('No %s found' % msg, end=' ')
         if oname:
             print('for %s' % oname)
         else:
             print()
 
     def pdef(self, obj, oname=''):
         """Print the call signature for any callable object.
 
         If the object is a class, print the constructor information."""
 
         if not callable(obj):
             print('Object is not callable.')
             return
 
         header = ''
 
         if inspect.isclass(obj):
             header = self.__head('Class constructor information:\n')
 
 
         output = self._getdef(obj,oname)
         if output is None:
             self.noinfo('definition header',oname)
         else:
             print(header,self.format(output), end=' ')
 
     # In Python 3, all classes are new-style, so they all have __init__.
     @skip_doctest
     def pdoc(self, obj, oname='', formatter=None):
         """Print the docstring for any object.
 
         Optional:
         -formatter: a function to run the docstring through for specially
         formatted docstrings.
 
         Examples
         --------
 
         In [1]: class NoInit:
            ...:     pass
 
         In [2]: class NoDoc:
            ...:     def __init__(self):
            ...:         pass
 
         In [3]: %pdoc NoDoc
         No documentation found for NoDoc
 
         In [4]: %pdoc NoInit
         No documentation found for NoInit
 
         In [5]: obj = NoInit()
 
         In [6]: %pdoc obj
         No documentation found for obj
 
         In [5]: obj2 = NoDoc()
 
         In [6]: %pdoc obj2
         No documentation found for obj2
         """
 
         head = self.__head  # For convenience
         lines = []
         ds = getdoc(obj)
         if formatter:
             ds = formatter(ds).get('plain/text', ds)
         if ds:
             lines.append(head("Class docstring:"))
             lines.append(indent(ds))
         if inspect.isclass(obj) and hasattr(obj, '__init__'):
             init_ds = getdoc(obj.__init__)
             if init_ds is not None:
                 lines.append(head("Init docstring:"))
                 lines.append(indent(init_ds))
         elif hasattr(obj,'__call__'):
             call_ds = getdoc(obj.__call__)
             if call_ds:
                 lines.append(head("Call docstring:"))
                 lines.append(indent(call_ds))
 
         if not lines:
             self.noinfo('documentation',oname)
         else:
             page.page('\n'.join(lines))
 
     def psource(self, obj, oname=''):
         """Print the source code for an object."""
 
         # Flush the source cache because inspect can return out-of-date source
         linecache.checkcache()
         try:
             src = getsource(obj, oname=oname)
         except Exception:
             src = None
 
         if src is None:
             self.noinfo('source', oname)
         else:
             page.page(self.format(src))
 
     def pfile(self, obj, oname=''):
         """Show the whole file where an object was defined."""
 
         lineno = find_source_lines(obj)
         if lineno is None:
             self.noinfo('file', oname)
             return
 
         ofile = find_file(obj)
         # run contents of file through pager starting at line where the object
         # is defined, as long as the file isn't binary and is actually on the
         # filesystem.
         if ofile.endswith(('.so', '.dll', '.pyd')):
             print('File %r is binary, not printing.' % ofile)
         elif not os.path.isfile(ofile):
             print('File %r does not exist, not printing.' % ofile)
         else:
             # Print only text files, not extension binaries.  Note that
             # getsourcelines returns lineno with 1-offset and page() uses
             # 0-offset, so we must adjust.
             page.page(self.format(openpy.read_py_file(ofile, skip_encoding_cookie=False)), lineno - 1)
 
 
     def _mime_format(self, text:str, formatter=None) -> dict:
         """Return a mime bundle representation of the input text.
 
         - if `formatter` is None, the returned mime bundle has
            a `text/plain` field, with the input text.
            a `text/html` field with a `<pre>` tag containing the input text.
 
         - if `formatter` is not None, it must be a callable transforming the
           input text into a mime bundle. Default values for `text/plain` and
           `text/html` representations are the ones described above.
 
         Note:
 
         Formatters returning strings are supported but this behavior is deprecated.
 
         """
         defaults = {
             'text/plain': text,
             'text/html': '<pre>' + text + '</pre>'
         }
 
         if formatter is None:
             return defaults
         else:
             formatted = formatter(text)
 
             if not isinstance(formatted, dict):
                 # Handle the deprecated behavior of a formatter returning
                 # a string instead of a mime bundle.
                 return {
                     'text/plain': formatted,
                     'text/html': '<pre>' + formatted + '</pre>'
                 }
 
             else:
                 return dict(defaults, **formatted)
 
 
     def format_mime(self, bundle):
 
         text_plain = bundle['text/plain']
 
         text = ''
         heads, bodies = list(zip(*text_plain))
         _len = max(len(h) for h in heads)
 
         for head, body in zip(heads, bodies):
             body = body.strip('\n')
             delim = '\n' if '\n' in body else ' '
             text += self.__head(head+':') + (_len - len(head))*' ' +delim + body +'\n'
 
         bundle['text/plain'] = text
         return bundle
 
     def _get_info(self, obj, oname='', formatter=None, info=None, detail_level=0):
         """Retrieve an info dict and format it.
 
         Parameters
         ==========
 
         obj: any
             Object to inspect and return info from
         oname: str (default: ''):
             Name of the variable pointing to `obj`.
         formatter: callable
         info:
             already computed information
         detail_level: integer
             Granularity of detail level, if set to 1, give more information.
         """
 
         info = self._info(obj, oname=oname, info=info, detail_level=detail_level)
 
         _mime = {
             'text/plain': [],
             'text/html': '',
         }
 
         def append_field(bundle, title:str, key:str, formatter=None):
             field = info[key]
             if field is not None:
                 formatted_field = self._mime_format(field, formatter)
                 bundle['text/plain'].append((title, formatted_field['text/plain']))
                 bundle['text/html'] += '<h1>' + title + '</h1>\n' + formatted_field['text/html'] + '\n'
 
         def code_formatter(text):
             return {
                 'text/plain': self.format(text),
                 'text/html': pylight(text)
             }
 
         if info['isalias']:
             append_field(_mime, 'Repr', 'string_form')
 
         elif info['ismagic']:
             if detail_level > 0:
                 append_field(_mime, 'Source', 'source', code_formatter)
             else:
                 append_field(_mime, 'Docstring', 'docstring', formatter)
             append_field(_mime, 'File', 'file')
 
         elif info['isclass'] or is_simple_callable(obj):
             # Functions, methods, classes
             append_field(_mime, 'Signature', 'definition', code_formatter)
             append_field(_mime, 'Init signature', 'init_definition', code_formatter)
             append_field(_mime, 'Docstring', 'docstring', formatter)
             if detail_level > 0 and info['source']:
                 append_field(_mime, 'Source', 'source', code_formatter)
             else:
                 append_field(_mime, 'Init docstring', 'init_docstring', formatter)
 
             append_field(_mime, 'File', 'file')
             append_field(_mime, 'Type', 'type_name')
             append_field(_mime, 'Subclasses', 'subclasses')
 
         else:
             # General Python objects
             append_field(_mime, 'Signature', 'definition', code_formatter)
             append_field(_mime, 'Call signature', 'call_def', code_formatter)
             append_field(_mime, 'Type', 'type_name')
             append_field(_mime, 'String form', 'string_form')
 
             # Namespace
             if info['namespace'] != 'Interactive':
                 append_field(_mime, 'Namespace', 'namespace')
 
             append_field(_mime, 'Length', 'length')
             append_field(_mime, 'File', 'file')
 
             # Source or docstring, depending on detail level and whether
             # source found.
             if detail_level > 0 and info['source']:
                 append_field(_mime, 'Source', 'source', code_formatter)
             else:
                 append_field(_mime, 'Docstring', 'docstring', formatter)
 
             append_field(_mime, 'Class docstring', 'class_docstring', formatter)
             append_field(_mime, 'Init docstring', 'init_docstring', formatter)
             append_field(_mime, 'Call docstring', 'call_docstring', formatter)
 
 
         return self.format_mime(_mime)
 
     def pinfo(self, obj, oname='', formatter=None, info=None, detail_level=0, enable_html_pager=True):
         """Show detailed information about an object.
 
         Optional arguments:
 
         - oname: name of the variable pointing to the object.
 
         - formatter: callable (optional)
               A special formatter for docstrings.
 
               The formatter is a callable that takes a string as an input
               and returns either a formatted string or a mime type bundle
               in the form of a dictionary.
 
               Although the support of custom formatter returning a string
               instead of a mime type bundle is deprecated.
 
         - info: a structure with some information fields which may have been
           precomputed already.
 
         - detail_level: if set to 1, more information is given.
         """
         info = self._get_info(obj, oname, formatter, info, detail_level)
         if not enable_html_pager:
             del info['text/html']
         page.page(info)
 
     def info(self, obj, oname='', formatter=None, info=None, detail_level=0):
         """DEPRECATED. Compute a dict with detailed information about an object.
         """
         if formatter is not None:
             warnings.warn('The `formatter` keyword argument to `Inspector.info`'
                      'is deprecated as of IPython 5.0 and will have no effects.',
                       DeprecationWarning, stacklevel=2)
         return self._info(obj, oname=oname, info=info, detail_level=detail_level)
 
     def _info(self, obj, oname='', info=None, detail_level=0) -> dict:
         """Compute a dict with detailed information about an object.
 
         Parameters
         ==========
 
         obj: any
             An object to find information about
         oname: str (default: ''):
             Name of the variable pointing to `obj`.
         info: (default: None)
             A struct (dict like with attr access) with some information fields
             which may have been precomputed already.
         detail_level: int (default:0)
             If set to 1, more information is given.
 
         Returns
         =======
 
         An object info dict with known fields from `info_fields`. Keys are
         strings, values are string or None.
         """
 
         if info is None:
             ismagic = False
             isalias = False
             ospace = ''
         else:
             ismagic = info.ismagic
             isalias = info.isalias
             ospace = info.namespace
 
         # Get docstring, special-casing aliases:
         if isalias:
             if not callable(obj):
                 try:
                     ds = "Alias to the system command:\n  %s" % obj[1]
                 except:
                     ds = "Alias: " + str(obj)
             else:
                 ds = "Alias to " + str(obj)
                 if obj.__doc__:
                     ds += "\nDocstring:\n" + obj.__doc__
         else:
             ds = getdoc(obj)
             if ds is None:
                 ds = '<no docstring>'
 
         # store output in a dict, we initialize it here and fill it as we go
         out = dict(name=oname, found=True, isalias=isalias, ismagic=ismagic, subclasses=None)
 
         string_max = 200 # max size of strings to show (snipped if longer)
         shalf = int((string_max - 5) / 2)
 
         if ismagic:
             out['type_name'] = 'Magic function'
         elif isalias:
             out['type_name'] = 'System alias'
         else:
             out['type_name'] = type(obj).__name__
 
         try:
             bclass = obj.__class__
             out['base_class'] = str(bclass)
         except:
             pass
 
         # String form, but snip if too long in ? form (full in ??)
         if detail_level >= self.str_detail_level:
             try:
                 ostr = str(obj)
                 str_head = 'string_form'
                 if not detail_level and len(ostr)>string_max:
                     ostr = ostr[:shalf] + ' <...> ' + ostr[-shalf:]
                     ostr = ("\n" + " " * len(str_head.expandtabs())).\
                             join(q.strip() for q in ostr.split("\n"))
                 out[str_head] = ostr
             except:
                 pass
 
         if ospace:
             out['namespace'] = ospace
 
         # Length (for strings and lists)
         try:
             out['length'] = str(len(obj))
         except Exception:
             pass
 
         # Filename where object was defined
         binary_file = False
         fname = find_file(obj)
         if fname is None:
             # if anything goes wrong, we don't want to show source, so it's as
             # if the file was binary
             binary_file = True
         else:
             if fname.endswith(('.so', '.dll', '.pyd')):
                 binary_file = True
             elif fname.endswith('<string>'):
                 fname = 'Dynamically generated function. No source code available.'
             out['file'] = compress_user(fname)
 
         # Original source code for a callable, class or property.
         if detail_level:
             # Flush the source cache because inspect can return out-of-date
             # source
             linecache.checkcache()
             try:
                 if isinstance(obj, property) or not binary_file:
                     src = getsource(obj, oname)
                     if src is not None:
                         src = src.rstrip()
                     out['source'] = src
 
             except Exception:
                 pass
 
         # Add docstring only if no source is to be shown (avoid repetitions).
         if ds and not self._source_contains_docstring(out.get('source'), ds):
             out['docstring'] = ds
 
         # Constructor docstring for classes
         if inspect.isclass(obj):
             out['isclass'] = True
 
             # get the init signature:
             try:
                 init_def = self._getdef(obj, oname)
             except AttributeError:
                 init_def = None
 
             # get the __init__ docstring
             try:
                 obj_init = obj.__init__
             except AttributeError:
                 init_ds = None
             else:
                 if init_def is None:
                     # Get signature from init if top-level sig failed.
                     # Can happen for built-in types (list, etc.).
                     try:
                         init_def = self._getdef(obj_init, oname)
                     except AttributeError:
                         pass
                 init_ds = getdoc(obj_init)
                 # Skip Python's auto-generated docstrings
                 if init_ds == _object_init_docstring:
                     init_ds = None
 
             if init_def:
                 out['init_definition'] = init_def
 
             if init_ds:
                 out['init_docstring'] = init_ds
 
             names = [sub.__name__ for sub in type.__subclasses__(obj)]
             if len(names) < 10:
                 all_names = ', '.join(names)
             else:
                 all_names = ', '.join(names[:10]+['...'])
             out['subclasses'] = all_names
         # and class docstring for instances:
         else:
             # reconstruct the function definition and print it:
             defln = self._getdef(obj, oname)
             if defln:
                 out['definition'] = defln
 
             # First, check whether the instance docstring is identical to the
             # class one, and print it separately if they don't coincide.  In
             # most cases they will, but it's nice to print all the info for
             # objects which use instance-customized docstrings.
             if ds:
                 try:
                     cls = getattr(obj,'__class__')
                 except:
                     class_ds = None
                 else:
                     class_ds = getdoc(cls)
                 # Skip Python's auto-generated docstrings
                 if class_ds in _builtin_type_docstrings:
                     class_ds = None
                 if class_ds and ds != class_ds:
                     out['class_docstring'] = class_ds
 
             # Next, try to show constructor docstrings
             try:
                 init_ds = getdoc(obj.__init__)
                 # Skip Python's auto-generated docstrings
                 if init_ds == _object_init_docstring:
                     init_ds = None
             except AttributeError:
                 init_ds = None
             if init_ds:
                 out['init_docstring'] = init_ds
 
             # Call form docstring for callable instances
             if safe_hasattr(obj, '__call__') and not is_simple_callable(obj):
                 call_def = self._getdef(obj.__call__, oname)
                 if call_def and (call_def != out.get('definition')):
                     # it may never be the case that call def and definition differ,
                     # but don't include the same signature twice
                     out['call_def'] = call_def
                 call_ds = getdoc(obj.__call__)
                 # Skip Python's auto-generated docstrings
                 if call_ds == _func_call_docstring:
                     call_ds = None
                 if call_ds:
                     out['call_docstring'] = call_ds
 
         return object_info(**out)
 
     @staticmethod
     def _source_contains_docstring(src, doc):
         """
         Check whether the source *src* contains the docstring *doc*.
 
         This is is helper function to skip displaying the docstring if the
         source already contains it, avoiding repetition of information.
         """
         try:
             def_node, = ast.parse(dedent(src)).body
             return ast.get_docstring(def_node) == doc
         except Exception:
             # The source can become invalid or even non-existent (because it
             # is re-fetched from the source file) so the above code fail in
             # arbitrary ways.
             return False
 
     def psearch(self,pattern,ns_table,ns_search=[],
                 ignore_case=False,show_all=False, *, list_types=False):
         """Search namespaces with wildcards for objects.
 
         Arguments:
 
         - pattern: string containing shell-like wildcards to use in namespace
           searches and optionally a type specification to narrow the search to
           objects of that type.
 
         - ns_table: dict of name->namespaces for search.
 
         Optional arguments:
 
           - ns_search: list of namespace names to include in search.
 
           - ignore_case(False): make the search case-insensitive.
 
           - show_all(False): show all names, including those starting with
             underscores.
             
           - list_types(False): list all available object types for object matching.
         """
         #print 'ps pattern:<%r>' % pattern # dbg
 
         # defaults
         type_pattern = 'all'
         filter = ''
 
         # list all object types
         if list_types:
             page.page('\n'.join(sorted(typestr2type)))
             return
 
         cmds = pattern.split()
         len_cmds  =  len(cmds)
         if len_cmds == 1:
             # Only filter pattern given
             filter = cmds[0]
         elif len_cmds == 2:
             # Both filter and type specified
             filter,type_pattern = cmds
         else:
             raise ValueError('invalid argument string for psearch: <%s>' %
                              pattern)
 
         # filter search namespaces
         for name in ns_search:
             if name not in ns_table:
                 raise ValueError('invalid namespace <%s>. Valid names: %s' %
                                  (name,ns_table.keys()))
 
         #print 'type_pattern:',type_pattern # dbg
         search_result, namespaces_seen = set(), set()
         for ns_name in ns_search:
             ns = ns_table[ns_name]
             # Normally, locals and globals are the same, so we just check one.
             if id(ns) in namespaces_seen:
                 continue
             namespaces_seen.add(id(ns))
             tmp_res = list_namespace(ns, type_pattern, filter,
                                     ignore_case=ignore_case, show_all=show_all)
             search_result.update(tmp_res)
 
         page.page('\n'.join(sorted(search_result)))
 
 
 def _render_signature(obj_signature, obj_name) -> str:
     """
     This was mostly taken from inspect.Signature.__str__.
     Look there for the comments.
     The only change is to add linebreaks when this gets too long.
     """
     result = []
     pos_only = False
     kw_only = True
     for param in obj_signature.parameters.values():
         if param.kind == inspect._POSITIONAL_ONLY:
             pos_only = True
         elif pos_only:
             result.append('/')
             pos_only = False
 
         if param.kind == inspect._VAR_POSITIONAL:
             kw_only = False
         elif param.kind == inspect._KEYWORD_ONLY and kw_only:
             result.append('*')
             kw_only = False
 
         result.append(str(param))
 
     if pos_only:
         result.append('/')
 
     # add up name, parameters, braces (2), and commas
     if len(obj_name) + sum(len(r) + 2 for r in result) > 75:
         # This doesn’t fit behind “Signature: ” in an inspect window.
         rendered = '{}(\n{})'.format(obj_name, ''.join(
             '    {},\n'.format(r) for r in result)
         )
     else:
         rendered = '{}({})'.format(obj_name, ', '.join(result))
 
     if obj_signature.return_annotation is not inspect._empty:
         anno = inspect.formatannotation(obj_signature.return_annotation)
         rendered += ' -> {}'.format(anno)
 
     return rendered

IPython/core/ultratb.py

@@ -1,1506 +1,1503
 # -*- coding: utf-8 -*-
 """
 Verbose and colourful traceback formatting.
 
 **ColorTB**
 
 I've always found it a bit hard to visually parse tracebacks in Python.  The
 ColorTB class is a solution to that problem.  It colors the different parts of a
 traceback in a manner similar to what you would expect from a syntax-highlighting
 text editor.
 
 Installation instructions for ColorTB::
 
     import sys,ultratb
     sys.excepthook = ultratb.ColorTB()
 
 **VerboseTB**
 
 I've also included a port of Ka-Ping Yee's "cgitb.py" that produces all kinds
 of useful info when a traceback occurs.  Ping originally had it spit out HTML
 and intended it for CGI programmers, but why should they have all the fun?  I
 altered it to spit out colored text to the terminal.  It's a bit overwhelming,
 but kind of neat, and maybe useful for long-running programs that you believe
 are bug-free.  If a crash *does* occur in that type of program you want details.
 Give it a shot--you'll love it or you'll hate it.
 
 .. note::
 
   The Verbose mode prints the variables currently visible where the exception
   happened (shortening their strings if too long). This can potentially be
   very slow, if you happen to have a huge data structure whose string
   representation is complex to compute. Your computer may appear to freeze for
   a while with cpu usage at 100%. If this occurs, you can cancel the traceback
   with Ctrl-C (maybe hitting it more than once).
 
   If you encounter this kind of situation often, you may want to use the
   Verbose_novars mode instead of the regular Verbose, which avoids formatting
   variables (but otherwise includes the information and context given by
   Verbose).
 
 .. note::
 
   The verbose mode print all variables in the stack, which means it can
   potentially leak sensitive information like access keys, or unencrypted
   password.
 
 Installation instructions for VerboseTB::
 
     import sys,ultratb
     sys.excepthook = ultratb.VerboseTB()
 
 Note:  Much of the code in this module was lifted verbatim from the standard
 library module 'traceback.py' and Ka-Ping Yee's 'cgitb.py'.
 
 Color schemes
 -------------
 
 The colors are defined in the class TBTools through the use of the
 ColorSchemeTable class. Currently the following exist:
 
   - NoColor: allows all of this module to be used in any terminal (the color
     escapes are just dummy blank strings).
 
   - Linux: is meant to look good in a terminal like the Linux console (black
     or very dark background).
 
   - LightBG: similar to Linux but swaps dark/light colors to be more readable
     in light background terminals.
 
   - Neutral: a neutral color scheme that should be readable on both light and
     dark background
 
 You can implement other color schemes easily, the syntax is fairly
 self-explanatory. Please send back new schemes you develop to the author for
 possible inclusion in future releases.
 
 Inheritance diagram:
 
 .. inheritance-diagram:: IPython.core.ultratb
    :parts: 3
 """
 
 #*****************************************************************************
 # Copyright (C) 2001 Nathaniel Gray <[e-mail]>
 # Copyright (C) 2001-2004 Fernando Perez <[e-mail]>
 #
 # Distributed under the terms of the BSD License.  The full license is in
 # the file COPYING, distributed as part of this software.
 #*****************************************************************************
 
 
 import dis
 import inspect
 import keyword
 import linecache
 import os
 import pydoc
 import re
 import sys
 import time
 import tokenize
 import traceback
 
-try:  # Python 2
-    generate_tokens = tokenize.generate_tokens
-except AttributeError:  # Python 3
-    generate_tokens = tokenize.tokenize
+from tokenize import generate_tokens
 
 # For purposes of monkeypatching inspect to fix a bug in it.
 from inspect import getsourcefile, getfile, getmodule, \
     ismodule, isclass, ismethod, isfunction, istraceback, isframe, iscode
 
 # IPython's own modules
 from IPython import get_ipython
 from IPython.core import debugger
 from IPython.core.display_trap import DisplayTrap
 from IPython.core.excolors import exception_colors
 from IPython.utils import PyColorize
 from IPython.utils import path as util_path
 from IPython.utils import py3compat
 from IPython.utils.data import uniq_stable
 from IPython.utils.terminal import get_terminal_size
 
 from logging import info, error, debug
 
 from importlib.util import source_from_cache
 
 import IPython.utils.colorable as colorable
 
 # Globals
 # amount of space to put line numbers before verbose tracebacks
 INDENT_SIZE = 8
 
 # Default color scheme.  This is used, for example, by the traceback
 # formatter.  When running in an actual IPython instance, the user's rc.colors
 # value is used, but having a module global makes this functionality available
 # to users of ultratb who are NOT running inside ipython.
 DEFAULT_SCHEME = 'NoColor'
 
 
 # Number of frame above which we are likely to have a recursion and will
 # **attempt** to detect it.  Made modifiable mostly to speedup test suite
 # as detecting recursion is one of our slowest test
 _FRAME_RECURSION_LIMIT = 500
 
 # ---------------------------------------------------------------------------
 # Code begins
 
 # Utility functions
 def inspect_error():
     """Print a message about internal inspect errors.
 
     These are unfortunately quite common."""
 
     error('Internal Python error in the inspect module.\n'
           'Below is the traceback from this internal error.\n')
 
 
 # This function is a monkeypatch we apply to the Python inspect module. We have
 # now found when it's needed (see discussion on issue gh-1456), and we have a
 # test case (IPython.core.tests.test_ultratb.ChangedPyFileTest) that fails if
 # the monkeypatch is not applied. TK, Aug 2012.
 def findsource(object):
     """Return the entire source file and starting line number for an object.
 
     The argument may be a module, class, method, function, traceback, frame,
     or code object.  The source code is returned as a list of all the lines
     in the file and the line number indexes a line in that list.  An IOError
     is raised if the source code cannot be retrieved.
 
     FIXED version with which we monkeypatch the stdlib to work around a bug."""
 
     file = getsourcefile(object) or getfile(object)
     # If the object is a frame, then trying to get the globals dict from its
     # module won't work. Instead, the frame object itself has the globals
     # dictionary.
     globals_dict = None
     if inspect.isframe(object):
         # XXX: can this ever be false?
         globals_dict = object.f_globals
     else:
         module = getmodule(object, file)
         if module:
             globals_dict = module.__dict__
     lines = linecache.getlines(file, globals_dict)
     if not lines:
         raise IOError('could not get source code')
 
     if ismodule(object):
         return lines, 0
 
     if isclass(object):
         name = object.__name__
         pat = re.compile(r'^(\s*)class\s*' + name + r'\b')
         # make some effort to find the best matching class definition:
         # use the one with the least indentation, which is the one
         # that's most probably not inside a function definition.
         candidates = []
         for i, line in enumerate(lines):
             match = pat.match(line)
             if match:
                 # if it's at toplevel, it's already the best one
                 if line[0] == 'c':
                     return lines, i
                 # else add whitespace to candidate list
                 candidates.append((match.group(1), i))
         if candidates:
             # this will sort by whitespace, and by line number,
             # less whitespace first
             candidates.sort()
             return lines, candidates[0][1]
         else:
             raise IOError('could not find class definition')
 
     if ismethod(object):
         object = object.__func__
     if isfunction(object):
         object = object.__code__
     if istraceback(object):
         object = object.tb_frame
     if isframe(object):
         object = object.f_code
     if iscode(object):
         if not hasattr(object, 'co_firstlineno'):
             raise IOError('could not find function definition')
         pat = re.compile(r'^(\s*def\s)|(.*(?<!\w)lambda(:|\s))|^(\s*@)')
         pmatch = pat.match
         # fperez - fix: sometimes, co_firstlineno can give a number larger than
         # the length of lines, which causes an error.  Safeguard against that.
         lnum = min(object.co_firstlineno, len(lines)) - 1
         while lnum > 0:
             if pmatch(lines[lnum]):
                 break
             lnum -= 1
 
         return lines, lnum
     raise IOError('could not find code object')
 
 
 # This is a patched version of inspect.getargs that applies the (unmerged)
 # patch for http://bugs.python.org/issue14611 by Stefano Taschini.  This fixes
 # https://github.com/ipython/ipython/issues/8205 and
 # https://github.com/ipython/ipython/issues/8293
 def getargs(co):
     """Get information about the arguments accepted by a code object.
 
     Three things are returned: (args, varargs, varkw), where 'args' is
     a list of argument names (possibly containing nested lists), and
     'varargs' and 'varkw' are the names of the * and ** arguments or None."""
     if not iscode(co):
         raise TypeError('{!r} is not a code object'.format(co))
 
     nargs = co.co_argcount
     names = co.co_varnames
     args = list(names[:nargs])
     step = 0
 
     # The following acrobatics are for anonymous (tuple) arguments.
     for i in range(nargs):
         if args[i][:1] in ('', '.'):
             stack, remain, count = [], [], []
             while step < len(co.co_code):
                 op = ord(co.co_code[step])
                 step = step + 1
                 if op >= dis.HAVE_ARGUMENT:
                     opname = dis.opname[op]
                     value = ord(co.co_code[step]) + ord(co.co_code[step+1])*256
                     step = step + 2
                     if opname in ('UNPACK_TUPLE', 'UNPACK_SEQUENCE'):
                         remain.append(value)
                         count.append(value)
                     elif opname in ('STORE_FAST', 'STORE_DEREF'):
                         if op in dis.haslocal:
                             stack.append(co.co_varnames[value])
                         elif op in dis.hasfree:
                             stack.append((co.co_cellvars + co.co_freevars)[value])
                         # Special case for sublists of length 1: def foo((bar))
                         # doesn't generate the UNPACK_TUPLE bytecode, so if
                         # `remain` is empty here, we have such a sublist.
                         if not remain:
                             stack[0] = [stack[0]]
                             break
                         else:
                             remain[-1] = remain[-1] - 1
                             while remain[-1] == 0:
                                 remain.pop()
                                 size = count.pop()
                                 stack[-size:] = [stack[-size:]]
                                 if not remain:
                                     break
                                 remain[-1] = remain[-1] - 1
                             if not remain:
                                 break
             args[i] = stack[0]
 
     varargs = None
     if co.co_flags & inspect.CO_VARARGS:
         varargs = co.co_varnames[nargs]
         nargs = nargs + 1
     varkw = None
     if co.co_flags & inspect.CO_VARKEYWORDS:
         varkw = co.co_varnames[nargs]
     return inspect.Arguments(args, varargs, varkw)
 
 
 # Monkeypatch inspect to apply our bugfix.
 def with_patch_inspect(f):
     """
     Deprecated since IPython 6.0
     decorator for monkeypatching inspect.findsource
     """
 
     def wrapped(*args, **kwargs):
         save_findsource = inspect.findsource
         save_getargs = inspect.getargs
         inspect.findsource = findsource
         inspect.getargs = getargs
         try:
             return f(*args, **kwargs)
         finally:
             inspect.findsource = save_findsource
             inspect.getargs = save_getargs
 
     return wrapped
 
 
 def fix_frame_records_filenames(records):
     """Try to fix the filenames in each record from inspect.getinnerframes().
 
     Particularly, modules loaded from within zip files have useless filenames
     attached to their code object, and inspect.getinnerframes() just uses it.
     """
     fixed_records = []
     for frame, filename, line_no, func_name, lines, index in records:
         # Look inside the frame's globals dictionary for __file__,
         # which should be better. However, keep Cython filenames since
         # we prefer the source filenames over the compiled .so file.
         if not filename.endswith(('.pyx', '.pxd', '.pxi')):
             better_fn = frame.f_globals.get('__file__', None)
             if isinstance(better_fn, str):
                 # Check the type just in case someone did something weird with
                 # __file__. It might also be None if the error occurred during
                 # import.
                 filename = better_fn
         fixed_records.append((frame, filename, line_no, func_name, lines, index))
     return fixed_records
 
 
 @with_patch_inspect
 def _fixed_getinnerframes(etb, context=1, tb_offset=0):
     LNUM_POS, LINES_POS, INDEX_POS = 2, 4, 5
 
     records = fix_frame_records_filenames(inspect.getinnerframes(etb, context))
     # If the error is at the console, don't build any context, since it would
     # otherwise produce 5 blank lines printed out (there is no file at the
     # console)
     rec_check = records[tb_offset:]
     try:
         rname = rec_check[0][1]
         if rname == '<ipython console>' or rname.endswith('<string>'):
             return rec_check
     except IndexError:
         pass
 
     aux = traceback.extract_tb(etb)
     assert len(records) == len(aux)
     for i, (file, lnum, _, _) in enumerate(aux):
         maybeStart = lnum - 1 - context // 2
         start = max(maybeStart, 0)
         end = start + context
         lines = linecache.getlines(file)[start:end]
         buf = list(records[i])
         buf[LNUM_POS] = lnum
         buf[INDEX_POS] = lnum - 1 - start
         buf[LINES_POS] = lines
         records[i] = tuple(buf)
     return records[tb_offset:]
 
 # Helper function -- largely belongs to VerboseTB, but we need the same
 # functionality to produce a pseudo verbose TB for SyntaxErrors, so that they
 # can be recognized properly by ipython.el's py-traceback-line-re
 # (SyntaxErrors have to be treated specially because they have no traceback)
 
 
 def _format_traceback_lines(lnum, index, lines, Colors, lvals, _line_format):
     """
     Format tracebacks lines with pointing arrow, leading numbers...
 
     Parameters
     ==========
 
     lnum: int
     index: int
     lines: list[string]
     Colors:
         ColorScheme used.
     lvals: bytes
         Values of local variables, already colored, to inject just after the error line.
     _line_format: f (str) -> (str, bool)
         return (colorized version of str, failure to do so)
     """
     numbers_width = INDENT_SIZE - 1
     res = []
 
     for i,line in enumerate(lines, lnum-index):
         line = py3compat.cast_unicode(line)
 
         new_line, err = _line_format(line, 'str')
         if not err:
             line = new_line
 
         if i == lnum:
             # This is the line with the error
             pad = numbers_width - len(str(i))
             num = '%s%s' % (debugger.make_arrow(pad), str(lnum))
             line = '%s%s%s %s%s' % (Colors.linenoEm, num,
                                     Colors.line, line, Colors.Normal)
         else:
             num = '%*s' % (numbers_width, i)
             line = '%s%s%s %s' % (Colors.lineno, num,
                                   Colors.Normal, line)
 
         res.append(line)
         if lvals and i == lnum:
             res.append(lvals + '\n')
     return res
 
 def is_recursion_error(etype, value, records):
     try:
         # RecursionError is new in Python 3.5
         recursion_error_type = RecursionError
     except NameError:
         recursion_error_type = RuntimeError
 
     # The default recursion limit is 1000, but some of that will be taken up
     # by stack frames in IPython itself. >500 frames probably indicates
     # a recursion error.
     return (etype is recursion_error_type) \
            and "recursion" in str(value).lower() \
            and len(records) > _FRAME_RECURSION_LIMIT
 
 def find_recursion(etype, value, records):
     """Identify the repeating stack frames from a RecursionError traceback
 
     'records' is a list as returned by VerboseTB.get_records()
 
     Returns (last_unique, repeat_length)
     """
     # This involves a bit of guesswork - we want to show enough of the traceback
     # to indicate where the recursion is occurring. We guess that the innermost
     # quarter of the traceback (250 frames by default) is repeats, and find the
     # first frame (from in to out) that looks different.
     if not is_recursion_error(etype, value, records):
         return len(records), 0
 
     # Select filename, lineno, func_name to track frames with
     records = [r[1:4] for r in records]
     inner_frames = records[-(len(records)//4):]
     frames_repeated = set(inner_frames)
 
     last_seen_at = {}
     longest_repeat = 0
     i = len(records)
     for frame in reversed(records):
         i -= 1
         if frame not in frames_repeated:
             last_unique = i
             break
 
         if frame in last_seen_at:
             distance = last_seen_at[frame] - i
             longest_repeat = max(longest_repeat, distance)
 
         last_seen_at[frame] = i
     else:
         last_unique = 0 # The whole traceback was recursion
 
     return last_unique, longest_repeat
 
 #---------------------------------------------------------------------------
 # Module classes
 class TBTools(colorable.Colorable):
     """Basic tools used by all traceback printer classes."""
 
     # Number of frames to skip when reporting tracebacks
     tb_offset = 0
 
     def __init__(self, color_scheme='NoColor', call_pdb=False, ostream=None, parent=None, config=None):
         # Whether to call the interactive pdb debugger after printing
         # tracebacks or not
         super(TBTools, self).__init__(parent=parent, config=config)
         self.call_pdb = call_pdb
 
         # Output stream to write to.  Note that we store the original value in
         # a private attribute and then make the public ostream a property, so
         # that we can delay accessing sys.stdout until runtime.  The way
         # things are written now, the sys.stdout object is dynamically managed
         # so a reference to it should NEVER be stored statically.  This
         # property approach confines this detail to a single location, and all
         # subclasses can simply access self.ostream for writing.
         self._ostream = ostream
 
         # Create color table
         self.color_scheme_table = exception_colors()
 
         self.set_colors(color_scheme)
         self.old_scheme = color_scheme  # save initial value for toggles
 
         if call_pdb:
             self.pdb = debugger.Pdb()
         else:
             self.pdb = None
 
     def _get_ostream(self):
         """Output stream that exceptions are written to.
 
         Valid values are:
 
         - None: the default, which means that IPython will dynamically resolve
           to sys.stdout.  This ensures compatibility with most tools, including
           Windows (where plain stdout doesn't recognize ANSI escapes).
 
         - Any object with 'write' and 'flush' attributes.
         """
         return sys.stdout if self._ostream is None else self._ostream
 
     def _set_ostream(self, val):
         assert val is None or (hasattr(val, 'write') and hasattr(val, 'flush'))
         self._ostream = val
 
     ostream = property(_get_ostream, _set_ostream)
 
     def get_parts_of_chained_exception(self, evalue):
         def get_chained_exception(exception_value):
             cause = getattr(exception_value, '__cause__', None)
             if cause:
                 return cause
             if getattr(exception_value, '__suppress_context__', False):
                 return None
             return getattr(exception_value, '__context__', None)
 
         chained_evalue = get_chained_exception(evalue)
 
         if chained_evalue:
             return chained_evalue.__class__, chained_evalue, chained_evalue.__traceback__
 
     def prepare_chained_exception_message(self, cause):
         direct_cause = "\nThe above exception was the direct cause of the following exception:\n"
         exception_during_handling = "\nDuring handling of the above exception, another exception occurred:\n"
 
         if cause:
             message = [[direct_cause]]
         else:
             message = [[exception_during_handling]]
         return message
 
     def set_colors(self, *args, **kw):
         """Shorthand access to the color table scheme selector method."""
 
         # Set own color table
         self.color_scheme_table.set_active_scheme(*args, **kw)
         # for convenience, set Colors to the active scheme
         self.Colors = self.color_scheme_table.active_colors
         # Also set colors of debugger
         if hasattr(self, 'pdb') and self.pdb is not None:
             self.pdb.set_colors(*args, **kw)
 
     def color_toggle(self):
         """Toggle between the currently active color scheme and NoColor."""
 
         if self.color_scheme_table.active_scheme_name == 'NoColor':
             self.color_scheme_table.set_active_scheme(self.old_scheme)
             self.Colors = self.color_scheme_table.active_colors
         else:
             self.old_scheme = self.color_scheme_table.active_scheme_name
             self.color_scheme_table.set_active_scheme('NoColor')
             self.Colors = self.color_scheme_table.active_colors
 
     def stb2text(self, stb):
         """Convert a structured traceback (a list) to a string."""
         return '\n'.join(stb)
 
     def text(self, etype, value, tb, tb_offset=None, context=5):
         """Return formatted traceback.
 
         Subclasses may override this if they add extra arguments.
         """
         tb_list = self.structured_traceback(etype, value, tb,
                                             tb_offset, context)
         return self.stb2text(tb_list)
 
     def structured_traceback(self, etype, evalue, tb, tb_offset=None,
                              context=5, mode=None):
         """Return a list of traceback frames.
 
         Must be implemented by each class.
         """
         raise NotImplementedError()
 
 
 #---------------------------------------------------------------------------
 class ListTB(TBTools):
     """Print traceback information from a traceback list, with optional color.
 
     Calling requires 3 arguments: (etype, evalue, elist)
     as would be obtained by::
     
       etype, evalue, tb = sys.exc_info()
       if tb:
         elist = traceback.extract_tb(tb)
       else:
         elist = None
 
     It can thus be used by programs which need to process the traceback before
     printing (such as console replacements based on the code module from the
     standard library).
 
     Because they are meant to be called without a full traceback (only a
     list), instances of this class can't call the interactive pdb debugger."""
 
     def __init__(self, color_scheme='NoColor', call_pdb=False, ostream=None, parent=None, config=None):
         TBTools.__init__(self, color_scheme=color_scheme, call_pdb=call_pdb,
                          ostream=ostream, parent=parent,config=config)
 
     def __call__(self, etype, value, elist):
         self.ostream.flush()
         self.ostream.write(self.text(etype, value, elist))
         self.ostream.write('\n')
 
     def _extract_tb(self, tb):
         if tb:
             return traceback.extract_tb(tb)
         else:
             return None
 
     def structured_traceback(self, etype, evalue, etb=None, tb_offset=None,
                              context=5):
         """Return a color formatted string with the traceback info.
 
         Parameters
         ----------
         etype : exception type
           Type of the exception raised.
 
         evalue : object
           Data stored in the exception
 
         etb : object
           If list: List of frames, see class docstring for details.
           If Traceback: Traceback of the exception.
 
         tb_offset : int, optional
           Number of frames in the traceback to skip.  If not given, the
           instance evalue is used (set in constructor).
 
         context : int, optional
           Number of lines of context information to print.
 
         Returns
         -------
         String with formatted exception.
         """
         # This is a workaround to get chained_exc_ids in recursive calls
         # etb should not be a tuple if structured_traceback is not recursive
         if isinstance(etb, tuple):
             etb, chained_exc_ids = etb
         else:
             chained_exc_ids = set()
 
         if isinstance(etb, list):
             elist = etb
         elif etb is not None:
             elist = self._extract_tb(etb)
         else:
             elist = []
         tb_offset = self.tb_offset if tb_offset is None else tb_offset
         Colors = self.Colors
         out_list = []
         if elist:
 
             if tb_offset and len(elist) > tb_offset:
                 elist = elist[tb_offset:]
 
             out_list.append('Traceback %s(most recent call last)%s:' %
                             (Colors.normalEm, Colors.Normal) + '\n')
             out_list.extend(self._format_list(elist))
         # The exception info should be a single entry in the list.
         lines = ''.join(self._format_exception_only(etype, evalue))
         out_list.append(lines)
 
         exception = self.get_parts_of_chained_exception(evalue)
 
         if exception and not id(exception[1]) in chained_exc_ids:
             chained_exception_message = self.prepare_chained_exception_message(
                 evalue.__cause__)[0]
             etype, evalue, etb = exception
             # Trace exception to avoid infinite 'cause' loop
             chained_exc_ids.add(id(exception[1]))
             chained_exceptions_tb_offset = 0
             out_list = (
                 self.structured_traceback(
                     etype, evalue, (etb, chained_exc_ids),
                     chained_exceptions_tb_offset, context)
                 + chained_exception_message
                 + out_list)
 
         return out_list
 
     def _format_list(self, extracted_list):
         """Format a list of traceback entry tuples for printing.
 
         Given a list of tuples as returned by extract_tb() or
         extract_stack(), return a list of strings ready for printing.
         Each string in the resulting list corresponds to the item with the
         same index in the argument list.  Each string ends in a newline;
         the strings may contain internal newlines as well, for those items
         whose source text line is not None.
 
         Lifted almost verbatim from traceback.py
         """
 
         Colors = self.Colors
         list = []
         for filename, lineno, name, line in extracted_list[:-1]:
             item = '  File %s"%s"%s, line %s%d%s, in %s%s%s\n' % \
                    (Colors.filename, filename, Colors.Normal,
                     Colors.lineno, lineno, Colors.Normal,
                     Colors.name, name, Colors.Normal)
             if line:
                 item += '    %s\n' % line.strip()
             list.append(item)
         # Emphasize the last entry
         filename, lineno, name, line = extracted_list[-1]
         item = '%s  File %s"%s"%s, line %s%d%s, in %s%s%s%s\n' % \
                (Colors.normalEm,
                 Colors.filenameEm, filename, Colors.normalEm,
                 Colors.linenoEm, lineno, Colors.normalEm,
                 Colors.nameEm, name, Colors.normalEm,
                 Colors.Normal)
         if line:
             item += '%s    %s%s\n' % (Colors.line, line.strip(),
                                       Colors.Normal)
         list.append(item)
         return list
 
     def _format_exception_only(self, etype, value):
         """Format the exception part of a traceback.
 
         The arguments are the exception type and value such as given by
         sys.exc_info()[:2]. The return value is a list of strings, each ending
         in a newline.  Normally, the list contains a single string; however,
         for SyntaxError exceptions, it contains several lines that (when
         printed) display detailed information about where the syntax error
         occurred.  The message indicating which exception occurred is the
         always last string in the list.
 
         Also lifted nearly verbatim from traceback.py
         """
         have_filedata = False
         Colors = self.Colors
         list = []
         stype = py3compat.cast_unicode(Colors.excName + etype.__name__ + Colors.Normal)
         if value is None:
             # Not sure if this can still happen in Python 2.6 and above
             list.append(stype + '\n')
         else:
             if issubclass(etype, SyntaxError):
                 have_filedata = True
                 if not value.filename: value.filename = "<string>"
                 if value.lineno:
                     lineno = value.lineno
                     textline = linecache.getline(value.filename, value.lineno)
                 else:
                     lineno = 'unknown'
                     textline = ''
                 list.append('%s  File %s"%s"%s, line %s%s%s\n' % \
                             (Colors.normalEm,
                              Colors.filenameEm, py3compat.cast_unicode(value.filename), Colors.normalEm,
                              Colors.linenoEm, lineno, Colors.Normal  ))
                 if textline == '':
                     textline = py3compat.cast_unicode(value.text, "utf-8")
 
                 if textline is not None:
                     i = 0
                     while i < len(textline) and textline[i].isspace():
                         i += 1
                     list.append('%s    %s%s\n' % (Colors.line,
                                                   textline.strip(),
                                                   Colors.Normal))
                     if value.offset is not None:
                         s = '    '
                         for c in textline[i:value.offset - 1]:
                             if c.isspace():
                                 s += c
                             else:
                                 s += ' '
                         list.append('%s%s^%s\n' % (Colors.caret, s,
                                                    Colors.Normal))
 
             try:
                 s = value.msg
             except Exception:
                 s = self._some_str(value)
             if s:
                 list.append('%s%s:%s %s\n' % (stype, Colors.excName,
                                               Colors.Normal, s))
             else:
                 list.append('%s\n' % stype)
 
         # sync with user hooks
         if have_filedata:
             ipinst = get_ipython()
             if ipinst is not None:
                 ipinst.hooks.synchronize_with_editor(value.filename, value.lineno, 0)
 
         return list
 
     def get_exception_only(self, etype, value):
         """Only print the exception type and message, without a traceback.
 
         Parameters
         ----------
         etype : exception type
         value : exception value
         """
         return ListTB.structured_traceback(self, etype, value)
 
     def show_exception_only(self, etype, evalue):
         """Only print the exception type and message, without a traceback.
 
         Parameters
         ----------
         etype : exception type
         value : exception value
         """
         # This method needs to use __call__ from *this* class, not the one from
         # a subclass whose signature or behavior may be different
         ostream = self.ostream
         ostream.flush()
         ostream.write('\n'.join(self.get_exception_only(etype, evalue)))
         ostream.flush()
 
     def _some_str(self, value):
         # Lifted from traceback.py
         try:
             return py3compat.cast_unicode(str(value))
         except:
             return u'<unprintable %s object>' % type(value).__name__
 
 
 #----------------------------------------------------------------------------
 class VerboseTB(TBTools):
     """A port of Ka-Ping Yee's cgitb.py module that outputs color text instead
     of HTML.  Requires inspect and pydoc.  Crazy, man.
 
     Modified version which optionally strips the topmost entries from the
     traceback, to be used with alternate interpreters (because their own code
     would appear in the traceback)."""
 
     def __init__(self, color_scheme='Linux', call_pdb=False, ostream=None,
                  tb_offset=0, long_header=False, include_vars=True,
                  check_cache=None, debugger_cls = None,
                  parent=None, config=None):
         """Specify traceback offset, headers and color scheme.
 
         Define how many frames to drop from the tracebacks. Calling it with
         tb_offset=1 allows use of this handler in interpreters which will have
         their own code at the top of the traceback (VerboseTB will first
         remove that frame before printing the traceback info)."""
         TBTools.__init__(self, color_scheme=color_scheme, call_pdb=call_pdb,
                          ostream=ostream, parent=parent, config=config)
         self.tb_offset = tb_offset
         self.long_header = long_header
         self.include_vars = include_vars
         # By default we use linecache.checkcache, but the user can provide a
         # different check_cache implementation.  This is used by the IPython
         # kernel to provide tracebacks for interactive code that is cached,
         # by a compiler instance that flushes the linecache but preserves its
         # own code cache.
         if check_cache is None:
             check_cache = linecache.checkcache
         self.check_cache = check_cache
 
         self.debugger_cls = debugger_cls or debugger.Pdb
 
     def format_records(self, records, last_unique, recursion_repeat):
         """Format the stack frames of the traceback"""
         frames = []
         for r in records[:last_unique+recursion_repeat+1]:
             #print '*** record:',file,lnum,func,lines,index  # dbg
             frames.append(self.format_record(*r))
 
         if recursion_repeat:
             frames.append('... last %d frames repeated, from the frame below ...\n' % recursion_repeat)
             frames.append(self.format_record(*records[last_unique+recursion_repeat+1]))
 
         return frames
 
     def format_record(self, frame, file, lnum, func, lines, index):
         """Format a single stack frame"""
         Colors = self.Colors  # just a shorthand + quicker name lookup
         ColorsNormal = Colors.Normal  # used a lot
         col_scheme = self.color_scheme_table.active_scheme_name
         indent = ' ' * INDENT_SIZE
         em_normal = '%s\n%s%s' % (Colors.valEm, indent, ColorsNormal)
         undefined = '%sundefined%s' % (Colors.em, ColorsNormal)
         tpl_link = '%s%%s%s' % (Colors.filenameEm, ColorsNormal)
         tpl_call = 'in %s%%s%s%%s%s' % (Colors.vName, Colors.valEm,
                                         ColorsNormal)
         tpl_call_fail = 'in %s%%s%s(***failed resolving arguments***)%s' % \
                         (Colors.vName, Colors.valEm, ColorsNormal)
         tpl_local_var = '%s%%s%s' % (Colors.vName, ColorsNormal)
         tpl_global_var = '%sglobal%s %s%%s%s' % (Colors.em, ColorsNormal,
                                                  Colors.vName, ColorsNormal)
         tpl_name_val = '%%s %s= %%s%s' % (Colors.valEm, ColorsNormal)
 
         if not file:
             file = '?'
         elif file.startswith(str("<")) and file.endswith(str(">")):
             # Not a real filename, no problem...
             pass
         elif not os.path.isabs(file):
             # Try to make the filename absolute by trying all
             # sys.path entries (which is also what linecache does)
             for dirname in sys.path:
                 try:
                     fullname = os.path.join(dirname, file)
                     if os.path.isfile(fullname):
                         file = os.path.abspath(fullname)
                         break
                 except Exception:
                     # Just in case that sys.path contains very
                     # strange entries...
                     pass
 
         file = py3compat.cast_unicode(file, util_path.fs_encoding)
         link = tpl_link % util_path.compress_user(file)
         args, varargs, varkw, locals_ = inspect.getargvalues(frame)
 
         if func == '?':
             call = ''
         elif func == '<module>':
             call = tpl_call % (func, '')
         else:
             # Decide whether to include variable details or not
             var_repr = eqrepr if self.include_vars else nullrepr
             try:
                 call = tpl_call % (func, inspect.formatargvalues(args,
                                                                  varargs, varkw,
                                                                  locals_, formatvalue=var_repr))
             except KeyError:
                 # This happens in situations like errors inside generator
                 # expressions, where local variables are listed in the
                 # line, but can't be extracted from the frame.  I'm not
                 # 100% sure this isn't actually a bug in inspect itself,
                 # but since there's no info for us to compute with, the
                 # best we can do is report the failure and move on.  Here
                 # we must *not* call any traceback construction again,
                 # because that would mess up use of %debug later on.  So we
                 # simply report the failure and move on.  The only
                 # limitation will be that this frame won't have locals
                 # listed in the call signature.  Quite subtle problem...
                 # I can't think of a good way to validate this in a unit
                 # test, but running a script consisting of:
                 #  dict( (k,v.strip()) for (k,v) in range(10) )
                 # will illustrate the error, if this exception catch is
                 # disabled.
                 call = tpl_call_fail % func
 
         # Don't attempt to tokenize binary files.
         if file.endswith(('.so', '.pyd', '.dll')):
             return '%s %s\n' % (link, call)
 
         elif file.endswith(('.pyc', '.pyo')):
             # Look up the corresponding source file.
             try:
                 file = source_from_cache(file)
             except ValueError:
                 # Failed to get the source file for some reason
                 # E.g. https://github.com/ipython/ipython/issues/9486
                 return '%s %s\n' % (link, call)
 
         def linereader(file=file, lnum=[lnum], getline=linecache.getline):
             line = getline(file, lnum[0])
             lnum[0] += 1
             return line
 
         # Build the list of names on this line of code where the exception
         # occurred.
         try:
             names = []
             name_cont = False
 
             for token_type, token, start, end, line in generate_tokens(linereader):
                 # build composite names
                 if token_type == tokenize.NAME and token not in keyword.kwlist:
                     if name_cont:
                         # Continuation of a dotted name
                         try:
                             names[-1].append(token)
                         except IndexError:
                             names.append([token])
                         name_cont = False
                     else:
                         # Regular new names.  We append everything, the caller
                         # will be responsible for pruning the list later.  It's
                         # very tricky to try to prune as we go, b/c composite
                         # names can fool us.  The pruning at the end is easy
                         # to do (or the caller can print a list with repeated
                         # names if so desired.
                         names.append([token])
                 elif token == '.':
                     name_cont = True
                 elif token_type == tokenize.NEWLINE:
                     break
 
         except (IndexError, UnicodeDecodeError, SyntaxError):
             # signals exit of tokenizer
             # SyntaxError can occur if the file is not actually Python
             #  - see gh-6300
             pass
         except tokenize.TokenError as msg:
             # Tokenizing may fail for various reasons, many of which are
             # harmless. (A good example is when the line in question is the
             # close of a triple-quoted string, cf gh-6864). We don't want to
             # show this to users, but want make it available for debugging
             # purposes.
             _m = ("An unexpected error occurred while tokenizing input\n"
                   "The following traceback may be corrupted or invalid\n"
                   "The error message is: %s\n" % msg)
             debug(_m)
 
         # Join composite names (e.g. "dict.fromkeys")
         names = ['.'.join(n) for n in names]
         # prune names list of duplicates, but keep the right order
         unique_names = uniq_stable(names)
 
         # Start loop over vars
         lvals = ''
         lvals_list = []
         if self.include_vars:
             for name_full in unique_names:
                 name_base = name_full.split('.', 1)[0]
                 if name_base in frame.f_code.co_varnames:
                     if name_base in locals_:
                         try:
                             value = repr(eval(name_full, locals_))
                         except:
                             value = undefined
                     else:
                         value = undefined
                     name = tpl_local_var % name_full
                 else:
                     if name_base in frame.f_globals:
                         try:
                             value = repr(eval(name_full, frame.f_globals))
                         except:
                             value = undefined
                     else:
                         value = undefined
                     name = tpl_global_var % name_full
                 lvals_list.append(tpl_name_val % (name, value))
         if lvals_list:
             lvals = '%s%s' % (indent, em_normal.join(lvals_list))
 
         level = '%s %s\n' % (link, call)
 
         if index is None:
             return level
         else:
             _line_format = PyColorize.Parser(style=col_scheme, parent=self).format2
             return '%s%s' % (level, ''.join(
                 _format_traceback_lines(lnum, index, lines, Colors, lvals,
                                          _line_format)))
 
     def prepare_header(self, etype, long_version=False):
         colors = self.Colors  # just a shorthand + quicker name lookup
         colorsnormal = colors.Normal  # used a lot
         exc = '%s%s%s' % (colors.excName, etype, colorsnormal)
         width = min(75, get_terminal_size()[0])
         if long_version:
             # Header with the exception type, python version, and date
             pyver = 'Python ' + sys.version.split()[0] + ': ' + sys.executable
             date = time.ctime(time.time())
 
             head = '%s%s%s\n%s%s%s\n%s' % (colors.topline, '-' * width, colorsnormal,
                                            exc, ' ' * (width - len(str(etype)) - len(pyver)),
                                            pyver, date.rjust(width) )
             head += "\nA problem occurred executing Python code.  Here is the sequence of function" \
                     "\ncalls leading up to the error, with the most recent (innermost) call last."
         else:
             # Simplified header
             head = '%s%s' % (exc, 'Traceback (most recent call last)'. \
                              rjust(width - len(str(etype))) )
 
         return head
 
     def format_exception(self, etype, evalue):
         colors = self.Colors  # just a shorthand + quicker name lookup
         colorsnormal = colors.Normal  # used a lot
         # Get (safely) a string form of the exception info
         try:
             etype_str, evalue_str = map(str, (etype, evalue))
         except:
             # User exception is improperly defined.
             etype, evalue = str, sys.exc_info()[:2]
             etype_str, evalue_str = map(str, (etype, evalue))
         # ... and format it
         return ['%s%s%s: %s' % (colors.excName, etype_str,
                                 colorsnormal, py3compat.cast_unicode(evalue_str))]
 
     def format_exception_as_a_whole(self, etype, evalue, etb, number_of_lines_of_context, tb_offset):
         """Formats the header, traceback and exception message for a single exception.
 
         This may be called multiple times by Python 3 exception chaining
         (PEP 3134).
         """
         # some locals
         orig_etype = etype
         try:
             etype = etype.__name__
         except AttributeError:
             pass
 
         tb_offset = self.tb_offset if tb_offset is None else tb_offset
         head = self.prepare_header(etype, self.long_header)
         records = self.get_records(etb, number_of_lines_of_context, tb_offset)
 
         if records is None:
             return ""
 
         last_unique, recursion_repeat = find_recursion(orig_etype, evalue, records)
 
         frames = self.format_records(records, last_unique, recursion_repeat)
 
         formatted_exception = self.format_exception(etype, evalue)
         if records:
             filepath, lnum = records[-1][1:3]
             filepath = os.path.abspath(filepath)
             ipinst = get_ipython()
             if ipinst is not None:
                 ipinst.hooks.synchronize_with_editor(filepath, lnum, 0)
 
         return [[head] + frames + [''.join(formatted_exception[0])]]
 
     def get_records(self, etb, number_of_lines_of_context, tb_offset):
         try:
             # Try the default getinnerframes and Alex's: Alex's fixes some
             # problems, but it generates empty tracebacks for console errors
             # (5 blanks lines) where none should be returned.
             return _fixed_getinnerframes(etb, number_of_lines_of_context, tb_offset)
         except UnicodeDecodeError:
             # This can occur if a file's encoding magic comment is wrong.
             # I can't see a way to recover without duplicating a bunch of code
             # from the stdlib traceback module. --TK
             error('\nUnicodeDecodeError while processing traceback.\n')
             return None
         except:
             # FIXME: I've been getting many crash reports from python 2.3
             # users, traceable to inspect.py.  If I can find a small test-case
             # to reproduce this, I should either write a better workaround or
             # file a bug report against inspect (if that's the real problem).
             # So far, I haven't been able to find an isolated example to
             # reproduce the problem.
             inspect_error()
             traceback.print_exc(file=self.ostream)
             info('\nUnfortunately, your original traceback can not be constructed.\n')
             return None
 
     def structured_traceback(self, etype, evalue, etb, tb_offset=None,
                              number_of_lines_of_context=5):
         """Return a nice text document describing the traceback."""
 
         formatted_exception = self.format_exception_as_a_whole(etype, evalue, etb, number_of_lines_of_context,
                                                                tb_offset)
 
         colors = self.Colors  # just a shorthand + quicker name lookup
         colorsnormal = colors.Normal  # used a lot
         head = '%s%s%s' % (colors.topline, '-' * min(75, get_terminal_size()[0]), colorsnormal)
         structured_traceback_parts = [head]
         chained_exceptions_tb_offset = 0
         lines_of_context = 3
         formatted_exceptions = formatted_exception
         exception = self.get_parts_of_chained_exception(evalue)
         if exception:
             formatted_exceptions += self.prepare_chained_exception_message(evalue.__cause__)
             etype, evalue, etb = exception
         else:
             evalue = None
         chained_exc_ids = set()
         while evalue:
             formatted_exceptions += self.format_exception_as_a_whole(etype, evalue, etb, lines_of_context,
                                                                      chained_exceptions_tb_offset)
             exception = self.get_parts_of_chained_exception(evalue)
 
             if exception and not id(exception[1]) in chained_exc_ids:
                 chained_exc_ids.add(id(exception[1])) # trace exception to avoid infinite 'cause' loop
                 formatted_exceptions += self.prepare_chained_exception_message(evalue.__cause__)
                 etype, evalue, etb = exception
             else:
                 evalue = None
 
         # we want to see exceptions in a reversed order:
         # the first exception should be on top
         for formatted_exception in reversed(formatted_exceptions):
             structured_traceback_parts += formatted_exception
 
         return structured_traceback_parts
 
     def debugger(self, force=False):
         """Call up the pdb debugger if desired, always clean up the tb
         reference.
 
         Keywords:
 
           - force(False): by default, this routine checks the instance call_pdb
             flag and does not actually invoke the debugger if the flag is false.
             The 'force' option forces the debugger to activate even if the flag
             is false.
 
         If the call_pdb flag is set, the pdb interactive debugger is
         invoked. In all cases, the self.tb reference to the current traceback
         is deleted to prevent lingering references which hamper memory
         management.
 
         Note that each call to pdb() does an 'import readline', so if your app
         requires a special setup for the readline completers, you'll have to
         fix that by hand after invoking the exception handler."""
 
         if force or self.call_pdb:
             if self.pdb is None:
                 self.pdb = self.debugger_cls()
             # the system displayhook may have changed, restore the original
             # for pdb
             display_trap = DisplayTrap(hook=sys.__displayhook__)
             with display_trap:
                 self.pdb.reset()
                 # Find the right frame so we don't pop up inside ipython itself
                 if hasattr(self, 'tb') and self.tb is not None:
                     etb = self.tb
                 else:
                     etb = self.tb = sys.last_traceback
                 while self.tb is not None and self.tb.tb_next is not None:
                     self.tb = self.tb.tb_next
                 if etb and etb.tb_next:
                     etb = etb.tb_next
                 self.pdb.botframe = etb.tb_frame
                 self.pdb.interaction(None, etb)
 
         if hasattr(self, 'tb'):
             del self.tb
 
     def handler(self, info=None):
         (etype, evalue, etb) = info or sys.exc_info()
         self.tb = etb
         ostream = self.ostream
         ostream.flush()
         ostream.write(self.text(etype, evalue, etb))
         ostream.write('\n')
         ostream.flush()
 
     # Changed so an instance can just be called as VerboseTB_inst() and print
     # out the right info on its own.
     def __call__(self, etype=None, evalue=None, etb=None):
         """This hook can replace sys.excepthook (for Python 2.1 or higher)."""
         if etb is None:
             self.handler()
         else:
             self.handler((etype, evalue, etb))
         try:
             self.debugger()
         except KeyboardInterrupt:
             print("\nKeyboardInterrupt")
 
 
 #----------------------------------------------------------------------------
 class FormattedTB(VerboseTB, ListTB):
     """Subclass ListTB but allow calling with a traceback.
 
     It can thus be used as a sys.excepthook for Python > 2.1.
 
     Also adds 'Context' and 'Verbose' modes, not available in ListTB.
 
     Allows a tb_offset to be specified. This is useful for situations where
     one needs to remove a number of topmost frames from the traceback (such as
     occurs with python programs that themselves execute other python code,
     like Python shells).  """
 
     def __init__(self, mode='Plain', color_scheme='Linux', call_pdb=False,
                  ostream=None,
                  tb_offset=0, long_header=False, include_vars=False,
                  check_cache=None, debugger_cls=None,
                  parent=None, config=None):
 
         # NEVER change the order of this list. Put new modes at the end:
         self.valid_modes = ['Plain', 'Context', 'Verbose', 'Minimal']
         self.verbose_modes = self.valid_modes[1:3]
 
         VerboseTB.__init__(self, color_scheme=color_scheme, call_pdb=call_pdb,
                            ostream=ostream, tb_offset=tb_offset,
                            long_header=long_header, include_vars=include_vars,
                            check_cache=check_cache, debugger_cls=debugger_cls,
                            parent=parent, config=config)
 
         # Different types of tracebacks are joined with different separators to
         # form a single string.  They are taken from this dict
         self._join_chars = dict(Plain='', Context='\n', Verbose='\n',
                                 Minimal='')
         # set_mode also sets the tb_join_char attribute
         self.set_mode(mode)
 
     def structured_traceback(self, etype, value, tb, tb_offset=None, number_of_lines_of_context=5):
         tb_offset = self.tb_offset if tb_offset is None else tb_offset
         mode = self.mode
         if mode in self.verbose_modes:
             # Verbose modes need a full traceback
             return VerboseTB.structured_traceback(
                 self, etype, value, tb, tb_offset, number_of_lines_of_context
             )
         elif mode == 'Minimal':
             return ListTB.get_exception_only(self, etype, value)
         else:
             # We must check the source cache because otherwise we can print
             # out-of-date source code.
             self.check_cache()
             # Now we can extract and format the exception
             return ListTB.structured_traceback(
                 self, etype, value, tb, tb_offset, number_of_lines_of_context
             )
 
     def stb2text(self, stb):
         """Convert a structured traceback (a list) to a string."""
         return self.tb_join_char.join(stb)
 
 
     def set_mode(self, mode=None):
         """Switch to the desired mode.
 
         If mode is not specified, cycles through the available modes."""
 
         if not mode:
             new_idx = (self.valid_modes.index(self.mode) + 1 ) % \
                       len(self.valid_modes)
             self.mode = self.valid_modes[new_idx]
         elif mode not in self.valid_modes:
             raise ValueError('Unrecognized mode in FormattedTB: <' + mode + '>\n'
                                                                             'Valid modes: ' + str(self.valid_modes))
         else:
             self.mode = mode
         # include variable details only in 'Verbose' mode
         self.include_vars = (self.mode == self.valid_modes[2])
         # Set the join character for generating text tracebacks
         self.tb_join_char = self._join_chars[self.mode]
 
     # some convenient shortcuts
     def plain(self):
         self.set_mode(self.valid_modes[0])
 
     def context(self):
         self.set_mode(self.valid_modes[1])
 
     def verbose(self):
         self.set_mode(self.valid_modes[2])
 
     def minimal(self):
         self.set_mode(self.valid_modes[3])
 
 
 #----------------------------------------------------------------------------
 class AutoFormattedTB(FormattedTB):
     """A traceback printer which can be called on the fly.
 
     It will find out about exceptions by itself.
 
     A brief example::
 
         AutoTB = AutoFormattedTB(mode = 'Verbose',color_scheme='Linux')
         try:
           ...
         except:
           AutoTB()  # or AutoTB(out=logfile) where logfile is an open file object
     """
 
     def __call__(self, etype=None, evalue=None, etb=None,
                  out=None, tb_offset=None):
         """Print out a formatted exception traceback.
 
         Optional arguments:
           - out: an open file-like object to direct output to.
 
           - tb_offset: the number of frames to skip over in the stack, on a
           per-call basis (this overrides temporarily the instance's tb_offset
           given at initialization time.  """
 
         if out is None:
             out = self.ostream
         out.flush()
         out.write(self.text(etype, evalue, etb, tb_offset))
         out.write('\n')
         out.flush()
         # FIXME: we should remove the auto pdb behavior from here and leave
         # that to the clients.
         try:
             self.debugger()
         except KeyboardInterrupt:
             print("\nKeyboardInterrupt")
 
     def structured_traceback(self, etype=None, value=None, tb=None,
                              tb_offset=None, number_of_lines_of_context=5):
         if etype is None:
             etype, value, tb = sys.exc_info()
         if isinstance(tb, tuple):
             # tb is a tuple if this is a chained exception.
             self.tb = tb[0]
         else:
             self.tb = tb
         return FormattedTB.structured_traceback(
             self, etype, value, tb, tb_offset, number_of_lines_of_context)
 
 
 #---------------------------------------------------------------------------
 
 # A simple class to preserve Nathan's original functionality.
 class ColorTB(FormattedTB):
     """Shorthand to initialize a FormattedTB in Linux colors mode."""
 
     def __init__(self, color_scheme='Linux', call_pdb=0, **kwargs):
         FormattedTB.__init__(self, color_scheme=color_scheme,
                              call_pdb=call_pdb, **kwargs)
 
 
 class SyntaxTB(ListTB):
     """Extension which holds some state: the last exception value"""
 
     def __init__(self, color_scheme='NoColor', parent=None, config=None):
         ListTB.__init__(self, color_scheme, parent=parent, config=config)
         self.last_syntax_error = None
 
     def __call__(self, etype, value, elist):
         self.last_syntax_error = value
 
         ListTB.__call__(self, etype, value, elist)
 
     def structured_traceback(self, etype, value, elist, tb_offset=None,
                              context=5):
         # If the source file has been edited, the line in the syntax error can
         # be wrong (retrieved from an outdated cache). This replaces it with
         # the current value.
         if isinstance(value, SyntaxError) \
                 and isinstance(value.filename, str) \
                 and isinstance(value.lineno, int):
             linecache.checkcache(value.filename)
             newtext = linecache.getline(value.filename, value.lineno)
             if newtext:
                 value.text = newtext
         self.last_syntax_error = value
         return super(SyntaxTB, self).structured_traceback(etype, value, elist,
                                                           tb_offset=tb_offset, context=context)
 
     def clear_err_state(self):
         """Return the current error state and clear it"""
         e = self.last_syntax_error
         self.last_syntax_error = None
         return e
 
     def stb2text(self, stb):
         """Convert a structured traceback (a list) to a string."""
         return ''.join(stb)
 
 
 # some internal-use functions
 def text_repr(value):
     """Hopefully pretty robust repr equivalent."""
     # this is pretty horrible but should always return *something*
     try:
         return pydoc.text.repr(value)
     except KeyboardInterrupt:
         raise
     except:
         try:
             return repr(value)
         except KeyboardInterrupt:
             raise
         except:
             try:
                 # all still in an except block so we catch
                 # getattr raising
                 name = getattr(value, '__name__', None)
                 if name:
                     # ick, recursion
                     return text_repr(name)
                 klass = getattr(value, '__class__', None)
                 if klass:
                     return '%s instance' % text_repr(klass)
             except KeyboardInterrupt:
                 raise
             except:
                 return 'UNRECOVERABLE REPR FAILURE'
 
 
 def eqrepr(value, repr=text_repr):
     return '=%s' % repr(value)
 
 
 def nullrepr(value, repr=text_repr):
     return ''

IPython/lib/deepreload.py

@@ -1,347 +1,341
 # -*- coding: utf-8 -*-
 """
 Provides a reload() function that acts recursively.
 
 Python's normal :func:`python:reload` function only reloads the module that it's
 passed. The :func:`reload` function in this module also reloads everything
 imported from that module, which is useful when you're changing files deep
 inside a package.
 
-To use this as your default reload function, type this for Python 2::
-
-    import __builtin__
-    from IPython.lib import deepreload
-    __builtin__.reload = deepreload.reload
-
-Or this for Python 3::
+To use this as your default reload function, type this::
 
     import builtins
     from IPython.lib import deepreload
     builtins.reload = deepreload.reload
 
 A reference to the original :func:`python:reload` is stored in this module as
 :data:`original_reload`, so you can restore it later.
 
 This code is almost entirely based on knee.py, which is a Python
 re-implementation of hierarchical module import.
 """
 #*****************************************************************************
 #       Copyright (C) 2001 Nathaniel Gray <[e-mail]>
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #*****************************************************************************
 
 import builtins as builtin_mod
 from contextlib import contextmanager
 import imp
 import sys
 
 from types import ModuleType
 from warnings import warn
 import types
 
 original_import = builtin_mod.__import__
 
 @contextmanager
 def replace_import_hook(new_import):
     saved_import = builtin_mod.__import__
     builtin_mod.__import__ = new_import
     try:
         yield
     finally:
         builtin_mod.__import__ = saved_import
 
 def get_parent(globals, level):
     """
     parent, name = get_parent(globals, level)
 
     Return the package that an import is being performed in.  If globals comes
     from the module foo.bar.bat (not itself a package), this returns the
     sys.modules entry for foo.bar.  If globals is from a package's __init__.py,
     the package's entry in sys.modules is returned.
 
     If globals doesn't come from a package or a module in a package, or a
     corresponding entry is not found in sys.modules, None is returned.
     """
     orig_level = level
 
     if not level or not isinstance(globals, dict):
         return None, ''
 
     pkgname = globals.get('__package__', None)
 
     if pkgname is not None:
         # __package__ is set, so use it
         if not hasattr(pkgname, 'rindex'):
             raise ValueError('__package__ set to non-string')
         if len(pkgname) == 0:
             if level > 0:
                 raise ValueError('Attempted relative import in non-package')
             return None, ''
         name = pkgname
     else:
         # __package__ not set, so figure it out and set it
         if '__name__' not in globals:
             return None, ''
         modname = globals['__name__']
 
         if '__path__' in globals:
             # __path__ is set, so modname is already the package name
             globals['__package__'] = name = modname
         else:
             # Normal module, so work out the package name if any
             lastdot = modname.rfind('.')
             if lastdot < 0 < level:
                 raise ValueError("Attempted relative import in non-package")
             if lastdot < 0:
                 globals['__package__'] = None
                 return None, ''
             globals['__package__'] = name = modname[:lastdot]
 
     dot = len(name)
     for x in range(level, 1, -1):
         try:
             dot = name.rindex('.', 0, dot)
         except ValueError:
             raise ValueError("attempted relative import beyond top-level "
                              "package")
     name = name[:dot]
 
     try:
         parent = sys.modules[name]
     except:
         if orig_level < 1:
             warn("Parent module '%.200s' not found while handling absolute "
                  "import" % name)
             parent = None
         else:
             raise SystemError("Parent module '%.200s' not loaded, cannot "
                               "perform relative import" % name)
 
     # We expect, but can't guarantee, if parent != None, that:
     # - parent.__name__ == name
     # - parent.__dict__ is globals
     # If this is violated...  Who cares?
     return parent, name
 
 def load_next(mod, altmod, name, buf):
     """
     mod, name, buf = load_next(mod, altmod, name, buf)
 
     altmod is either None or same as mod
     """
 
     if len(name) == 0:
         # completely empty module name should only happen in
         # 'from . import' (or '__import__("")')
         return mod, None, buf
 
     dot = name.find('.')
     if dot == 0:
         raise ValueError('Empty module name')
 
     if dot < 0:
         subname = name
         next = None
     else:
         subname = name[:dot]
         next = name[dot+1:]
 
     if buf != '':
         buf += '.'
     buf += subname
 
     result = import_submodule(mod, subname, buf)
     if result is None and mod != altmod:
         result = import_submodule(altmod, subname, subname)
         if result is not None:
             buf = subname
 
     if result is None:
         raise ImportError("No module named %.200s" % name)
 
     return result, next, buf
 
 
 # Need to keep track of what we've already reloaded to prevent cyclic evil
 found_now = {}
 
 def import_submodule(mod, subname, fullname):
     """m = import_submodule(mod, subname, fullname)"""
     # Require:
     # if mod == None: subname == fullname
     # else: mod.__name__ + "." + subname == fullname
 
     global found_now
     if fullname in found_now and fullname in sys.modules:
         m = sys.modules[fullname]
     else:
         print('Reloading', fullname)
         found_now[fullname] = 1
         oldm = sys.modules.get(fullname, None)
 
         if mod is None:
             path = None
         elif hasattr(mod, '__path__'):
             path = mod.__path__
         else:
             return None
 
         try:
             # This appears to be necessary on Python 3, because imp.find_module()
             # tries to import standard libraries (like io) itself, and we don't
             # want them to be processed by our deep_import_hook.
             with replace_import_hook(original_import):
                 fp, filename, stuff = imp.find_module(subname, path)
         except ImportError:
             return None
 
         try:
             m = imp.load_module(fullname, fp, filename, stuff)
         except:
             # load_module probably removed name from modules because of
             # the error.  Put back the original module object.
             if oldm:
                 sys.modules[fullname] = oldm
             raise
         finally:
             if fp: fp.close()
 
         add_submodule(mod, m, fullname, subname)
 
     return m
 
 def add_submodule(mod, submod, fullname, subname):
     """mod.{subname} = submod"""
     if mod is None:
         return #Nothing to do here.
 
     if submod is None:
         submod = sys.modules[fullname]
 
     setattr(mod, subname, submod)
 
     return
 
 def ensure_fromlist(mod, fromlist, buf, recursive):
     """Handle 'from module import a, b, c' imports."""
     if not hasattr(mod, '__path__'):
         return
     for item in fromlist:
         if not hasattr(item, 'rindex'):
             raise TypeError("Item in ``from list'' not a string")
         if item == '*':
             if recursive:
                 continue # avoid endless recursion
             try:
                 all = mod.__all__
             except AttributeError:
                 pass
             else:
                 ret = ensure_fromlist(mod, all, buf, 1)
                 if not ret:
                     return 0
         elif not hasattr(mod, item):
             import_submodule(mod, item, buf + '.' + item)
 
 def deep_import_hook(name, globals=None, locals=None, fromlist=None, level=-1):
     """Replacement for __import__()"""
     parent, buf = get_parent(globals, level)
 
     head, name, buf = load_next(parent, None if level < 0 else parent, name, buf)
 
     tail = head
     while name:
         tail, name, buf = load_next(tail, tail, name, buf)
 
     # If tail is None, both get_parent and load_next found
     # an empty module name: someone called __import__("") or
     # doctored faulty bytecode
     if tail is None:
         raise ValueError('Empty module name')
 
     if not fromlist:
         return head
 
     ensure_fromlist(tail, fromlist, buf, 0)
     return tail
 
 modules_reloading = {}
 
 def deep_reload_hook(m):
     """Replacement for reload()."""
     # Hardcode this one  as it would raise a NotImplementedError from the
     # bowels of Python and screw up the import machinery after.
     # unlike other imports the `exclude` list already in place is not enough.
 
     if m is types:
         return m
     if not isinstance(m, ModuleType):
         raise TypeError("reload() argument must be module")
 
     name = m.__name__
 
     if name not in sys.modules:
         raise ImportError("reload(): module %.200s not in sys.modules" % name)
 
     global modules_reloading
     try:
         return modules_reloading[name]
     except:
         modules_reloading[name] = m
 
     dot = name.rfind('.')
     if dot < 0:
         subname = name
         path = None
     else:
         try:
             parent = sys.modules[name[:dot]]
         except KeyError:
             modules_reloading.clear()
             raise ImportError("reload(): parent %.200s not in sys.modules" % name[:dot])
         subname = name[dot+1:]
         path = getattr(parent, "__path__", None)
 
     try:
         # This appears to be necessary on Python 3, because imp.find_module()
         # tries to import standard libraries (like io) itself, and we don't
         # want them to be processed by our deep_import_hook.
         with replace_import_hook(original_import):
             fp, filename, stuff  = imp.find_module(subname, path)
     finally:
         modules_reloading.clear()
 
     try:
         newm = imp.load_module(name, fp, filename, stuff)
     except:
          # load_module probably removed name from modules because of
          # the error.  Put back the original module object.
         sys.modules[name] = m
         raise
     finally:
         if fp: fp.close()
 
     modules_reloading.clear()
     return newm
 
 # Save the original hooks
 original_reload = imp.reload
 
 # Replacement for reload()
 def reload(module, exclude=('sys', 'os.path', 'builtins', '__main__',
                             'numpy', 'numpy._globals')):
     """Recursively reload all modules used in the given module.  Optionally
     takes a list of modules to exclude from reloading.  The default exclude
     list contains sys, __main__, and __builtin__, to prevent, e.g., resetting
     display, exception, and io hooks.
     """
     global found_now
     for i in exclude:
         found_now[i] = 1
     try:
         with replace_import_hook(deep_import_hook):
             return deep_reload_hook(module)
     finally:
         found_now = {}

IPython/lib/lexers.py

@@ -1,532 +1,532
 # -*- coding: utf-8 -*-
 """
 Defines a variety of Pygments lexers for highlighting IPython code.
 
 This includes:
 
     IPythonLexer, IPython3Lexer
         Lexers for pure IPython (python + magic/shell commands)
 
     IPythonPartialTracebackLexer, IPythonTracebackLexer
         Supports 2.x and 3.x via keyword `python3`.  The partial traceback
         lexer reads everything but the Python code appearing in a traceback.
         The full lexer combines the partial lexer with an IPython lexer.
 
     IPythonConsoleLexer
         A lexer for IPython console sessions, with support for tracebacks.
 
     IPyLexer
         A friendly lexer which examines the first line of text and from it,
         decides whether to use an IPython lexer or an IPython console lexer.
         This is probably the only lexer that needs to be explicitly added
         to Pygments.
 
 """
 #-----------------------------------------------------------------------------
 # Copyright (c) 2013, the IPython Development Team.
 #
 # Distributed under the terms of the Modified BSD License.
 #
 # The full license is in the file COPYING.txt, distributed with this software.
 #-----------------------------------------------------------------------------
 
 # Standard library
 import re
 
 # Third party
 from pygments.lexers import (
     BashLexer, HtmlLexer, JavascriptLexer, RubyLexer, PerlLexer, PythonLexer,
     Python3Lexer, TexLexer)
 from pygments.lexer import (
     Lexer, DelegatingLexer, RegexLexer, do_insertions, bygroups, using,
 )
 from pygments.token import (
     Generic, Keyword, Literal, Name, Operator, Other, Text, Error,
 )
 from pygments.util import get_bool_opt
 
 # Local
 
 line_re = re.compile('.*?\n')
 
 __all__ = ['build_ipy_lexer', 'IPython3Lexer', 'IPythonLexer',
            'IPythonPartialTracebackLexer', 'IPythonTracebackLexer',
            'IPythonConsoleLexer', 'IPyLexer']
 
 
 def build_ipy_lexer(python3):
     """Builds IPython lexers depending on the value of `python3`.
 
     The lexer inherits from an appropriate Python lexer and then adds
     information about IPython specific keywords (i.e. magic commands,
     shell commands, etc.)
 
     Parameters
     ----------
     python3 : bool
         If `True`, then build an IPython lexer from a Python 3 lexer.
 
     """
     # It would be nice to have a single IPython lexer class which takes
     # a boolean `python3`.  But since there are two Python lexer classes,
     # we will also have two IPython lexer classes.
     if python3:
         PyLexer = Python3Lexer
         name = 'IPython3'
         aliases = ['ipython3']
         doc = """IPython3 Lexer"""
     else:
         PyLexer = PythonLexer
         name = 'IPython'
         aliases = ['ipython2', 'ipython']
         doc = """IPython Lexer"""
 
     ipython_tokens = [
        (r'(?s)(\s*)(%%capture)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(PyLexer))),
         (r'(?s)(\s*)(%%debug)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(PyLexer))),
         (r'(?is)(\s*)(%%html)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(HtmlLexer))),
         (r'(?s)(\s*)(%%javascript)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(JavascriptLexer))),
         (r'(?s)(\s*)(%%js)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(JavascriptLexer))),
         (r'(?s)(\s*)(%%latex)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(TexLexer))),
         (r'(?s)(\s*)(%%perl)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(PerlLexer))),
         (r'(?s)(\s*)(%%prun)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(PyLexer))),
         (r'(?s)(\s*)(%%pypy)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(PyLexer))),
         (r'(?s)(\s*)(%%python)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(PyLexer))),
         (r'(?s)(\s*)(%%python2)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(PythonLexer))),
         (r'(?s)(\s*)(%%python3)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(Python3Lexer))),
         (r'(?s)(\s*)(%%ruby)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(RubyLexer))),
         (r'(?s)(\s*)(%%time)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(PyLexer))),
         (r'(?s)(\s*)(%%timeit)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(PyLexer))),
         (r'(?s)(\s*)(%%writefile)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(PyLexer))),
         (r'(?s)(\s*)(%%file)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(PyLexer))),
         (r"(?s)(\s*)(%%)(\w+)(.*)", bygroups(Text, Operator, Keyword, Text)),
         (r'(?s)(^\s*)(%%!)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(BashLexer))),
         (r"(%%?)(\w+)(\?\??)$",  bygroups(Operator, Keyword, Operator)),
         (r"\b(\?\??)(\s*)$",  bygroups(Operator, Text)),
         (r'(%)(sx|sc|system)(.*)(\n)', bygroups(Operator, Keyword,
                                                 using(BashLexer), Text)),
         (r'(%)(\w+)(.*\n)', bygroups(Operator, Keyword, Text)),
         (r'^(!!)(.+)(\n)', bygroups(Operator, using(BashLexer), Text)),
         (r'(!)(?!=)(.+)(\n)', bygroups(Operator, using(BashLexer), Text)),
         (r'^(\s*)(\?\??)(\s*%{0,2}[\w\.\*]*)', bygroups(Text, Operator, Text)),
         (r'(\s*%{0,2}[\w\.\*]*)(\?\??)(\s*)$', bygroups(Text, Operator, Text)),
     ]
 
     tokens = PyLexer.tokens.copy()
     tokens['root'] = ipython_tokens + tokens['root']
 
     attrs = {'name': name, 'aliases': aliases, 'filenames': [],
              '__doc__': doc, 'tokens': tokens}
 
     return type(name, (PyLexer,), attrs)
 
 
 IPython3Lexer = build_ipy_lexer(python3=True)
 IPythonLexer = build_ipy_lexer(python3=False)
 
 
 class IPythonPartialTracebackLexer(RegexLexer):
     """
     Partial lexer for IPython tracebacks.
 
-    Handles all the non-python output. This works for both Python 2.x and 3.x.
+    Handles all the non-python output.
 
     """
     name = 'IPython Partial Traceback'
 
     tokens = {
         'root': [
             # Tracebacks for syntax errors have a different style.
             # For both types of tracebacks, we mark the first line with
             # Generic.Traceback.  For syntax errors, we mark the filename
             # as we mark the filenames for non-syntax tracebacks.
             #
             # These two regexps define how IPythonConsoleLexer finds a
             # traceback.
             #
             ## Non-syntax traceback
             (r'^(\^C)?(-+\n)', bygroups(Error, Generic.Traceback)),
             ## Syntax traceback
             (r'^(  File)(.*)(, line )(\d+\n)',
              bygroups(Generic.Traceback, Name.Namespace,
                       Generic.Traceback, Literal.Number.Integer)),
 
             # (Exception Identifier)(Whitespace)(Traceback Message)
             (r'(?u)(^[^\d\W]\w*)(\s*)(Traceback.*?\n)',
              bygroups(Name.Exception, Generic.Whitespace, Text)),
             # (Module/Filename)(Text)(Callee)(Function Signature)
             # Better options for callee and function signature?
             (r'(.*)( in )(.*)(\(.*\)\n)',
              bygroups(Name.Namespace, Text, Name.Entity, Name.Tag)),
             # Regular line: (Whitespace)(Line Number)(Python Code)
             (r'(\s*?)(\d+)(.*?\n)',
              bygroups(Generic.Whitespace, Literal.Number.Integer, Other)),
             # Emphasized line: (Arrow)(Line Number)(Python Code)
             # Using Exception token so arrow color matches the Exception.
             (r'(-*>?\s?)(\d+)(.*?\n)',
              bygroups(Name.Exception, Literal.Number.Integer, Other)),
             # (Exception Identifier)(Message)
             (r'(?u)(^[^\d\W]\w*)(:.*?\n)',
              bygroups(Name.Exception, Text)),
             # Tag everything else as Other, will be handled later.
             (r'.*\n', Other),
         ],
     }
 
 
 class IPythonTracebackLexer(DelegatingLexer):
     """
     IPython traceback lexer.
 
     For doctests, the tracebacks can be snipped as much as desired with the
     exception to the lines that designate a traceback. For non-syntax error
     tracebacks, this is the line of hyphens. For syntax error tracebacks,
     this is the line which lists the File and line number.
 
     """
     # The lexer inherits from DelegatingLexer.  The "root" lexer is an
     # appropriate IPython lexer, which depends on the value of the boolean
     # `python3`.  First, we parse with the partial IPython traceback lexer.
     # Then, any code marked with the "Other" token is delegated to the root
     # lexer.
     #
     name = 'IPython Traceback'
     aliases = ['ipythontb']
 
     def __init__(self, **options):
         self.python3 = get_bool_opt(options, 'python3', False)
         if self.python3:
             self.aliases = ['ipython3tb']
         else:
             self.aliases = ['ipython2tb', 'ipythontb']
 
         if self.python3:
             IPyLexer = IPython3Lexer
         else:
             IPyLexer = IPythonLexer
 
         DelegatingLexer.__init__(self, IPyLexer,
                                  IPythonPartialTracebackLexer, **options)
 
 class IPythonConsoleLexer(Lexer):
     """
     An IPython console lexer for IPython code-blocks and doctests, such as:
 
     .. code-block:: rst
 
         .. code-block:: ipythonconsole
 
             In [1]: a = 'foo'
 
             In [2]: a
             Out[2]: 'foo'
 
             In [3]: print a
             foo
 
             In [4]: 1 / 0
 
 
     Support is also provided for IPython exceptions:
 
     .. code-block:: rst
 
         .. code-block:: ipythonconsole
 
             In [1]: raise Exception
 
             ---------------------------------------------------------------------------
             Exception                                 Traceback (most recent call last)
             <ipython-input-1-fca2ab0ca76b> in <module>
             ----> 1 raise Exception
 
             Exception:
 
     """
     name = 'IPython console session'
     aliases = ['ipythonconsole']
     mimetypes = ['text/x-ipython-console']
 
     # The regexps used to determine what is input and what is output.
     # The default prompts for IPython are:
     #
     #    in           = 'In [#]: '
     #    continuation = '   .D.: '
     #    template     = 'Out[#]: '
     #
     # Where '#' is the 'prompt number' or 'execution count' and 'D' 
     # D is a number of dots  matching the width of the execution count 
     #
     in1_regex = r'In \[[0-9]+\]: '
     in2_regex = r'   \.\.+\.: '
     out_regex = r'Out\[[0-9]+\]: '
 
     #: The regex to determine when a traceback starts.
     ipytb_start = re.compile(r'^(\^C)?(-+\n)|^(  File)(.*)(, line )(\d+\n)')
 
     def __init__(self, **options):
         """Initialize the IPython console lexer.
 
         Parameters
         ----------
         python3 : bool
             If `True`, then the console inputs are parsed using a Python 3
             lexer. Otherwise, they are parsed using a Python 2 lexer.
         in1_regex : RegexObject
             The compiled regular expression used to detect the start
             of inputs. Although the IPython configuration setting may have a
             trailing whitespace, do not include it in the regex. If `None`,
             then the default input prompt is assumed.
         in2_regex : RegexObject
             The compiled regular expression used to detect the continuation
             of inputs. Although the IPython configuration setting may have a
             trailing whitespace, do not include it in the regex. If `None`,
             then the default input prompt is assumed.
         out_regex : RegexObject
             The compiled regular expression used to detect outputs. If `None`,
             then the default output prompt is assumed.
 
         """
         self.python3 = get_bool_opt(options, 'python3', False)
         if self.python3:
             self.aliases = ['ipython3console']
         else:
             self.aliases = ['ipython2console', 'ipythonconsole']
 
         in1_regex = options.get('in1_regex', self.in1_regex)
         in2_regex = options.get('in2_regex', self.in2_regex)
         out_regex = options.get('out_regex', self.out_regex)
 
         # So that we can work with input and output prompts which have been
         # rstrip'd (possibly by editors) we also need rstrip'd variants. If
         # we do not do this, then such prompts will be tagged as 'output'.
         # The reason can't just use the rstrip'd variants instead is because
         # we want any whitespace associated with the prompt to be inserted
         # with the token. This allows formatted code to be modified so as hide
         # the appearance of prompts, with the whitespace included. One example
         # use of this is in copybutton.js from the standard lib Python docs.
         in1_regex_rstrip = in1_regex.rstrip() + '\n'
         in2_regex_rstrip = in2_regex.rstrip() + '\n'
         out_regex_rstrip = out_regex.rstrip() + '\n'
 
         # Compile and save them all.
         attrs = ['in1_regex', 'in2_regex', 'out_regex',
                  'in1_regex_rstrip', 'in2_regex_rstrip', 'out_regex_rstrip']
         for attr in attrs:
             self.__setattr__(attr, re.compile(locals()[attr]))
 
         Lexer.__init__(self, **options)
 
         if self.python3:
             pylexer = IPython3Lexer
             tblexer = IPythonTracebackLexer
         else:
             pylexer = IPythonLexer
             tblexer = IPythonTracebackLexer
 
         self.pylexer = pylexer(**options)
         self.tblexer = tblexer(**options)
 
         self.reset()
 
     def reset(self):
         self.mode = 'output'
         self.index = 0
         self.buffer = u''
         self.insertions = []
 
     def buffered_tokens(self):
         """
         Generator of unprocessed tokens after doing insertions and before
         changing to a new state.
 
         """
         if self.mode == 'output':
             tokens = [(0, Generic.Output, self.buffer)]
         elif self.mode == 'input':
             tokens = self.pylexer.get_tokens_unprocessed(self.buffer)
         else: # traceback
             tokens = self.tblexer.get_tokens_unprocessed(self.buffer)
 
         for i, t, v in do_insertions(self.insertions, tokens):
             # All token indexes are relative to the buffer.
             yield self.index + i, t, v
 
         # Clear it all
         self.index += len(self.buffer)
         self.buffer = u''
         self.insertions = []
 
     def get_mci(self, line):
         """
         Parses the line and returns a 3-tuple: (mode, code, insertion).
 
         `mode` is the next mode (or state) of the lexer, and is always equal
         to 'input', 'output', or 'tb'.
 
         `code` is a portion of the line that should be added to the buffer
         corresponding to the next mode and eventually lexed by another lexer.
         For example, `code` could be Python code if `mode` were 'input'.
 
         `insertion` is a 3-tuple (index, token, text) representing an
         unprocessed "token" that will be inserted into the stream of tokens
         that are created from the buffer once we change modes. This is usually
         the input or output prompt.
 
         In general, the next mode depends on current mode and on the contents
         of `line`.
 
         """
         # To reduce the number of regex match checks, we have multiple
         # 'if' blocks instead of 'if-elif' blocks.
 
         # Check for possible end of input
         in2_match = self.in2_regex.match(line)
         in2_match_rstrip = self.in2_regex_rstrip.match(line)
         if (in2_match and in2_match.group().rstrip() == line.rstrip()) or \
            in2_match_rstrip:
             end_input = True
         else:
             end_input = False
         if end_input and self.mode != 'tb':
             # Only look for an end of input when not in tb mode.
             # An ellipsis could appear within the traceback.
             mode = 'output'
             code = u''
             insertion = (0, Generic.Prompt, line)
             return mode, code, insertion
 
         # Check for output prompt
         out_match = self.out_regex.match(line)
         out_match_rstrip = self.out_regex_rstrip.match(line)
         if out_match or out_match_rstrip:
             mode = 'output'
             if out_match:
                 idx = out_match.end()
             else:
                 idx = out_match_rstrip.end()
             code = line[idx:]
             # Use the 'heading' token for output.  We cannot use Generic.Error
             # since it would conflict with exceptions.
             insertion = (0, Generic.Heading, line[:idx])
             return mode, code, insertion
 
 
         # Check for input or continuation prompt (non stripped version)
         in1_match = self.in1_regex.match(line)
         if in1_match or (in2_match and self.mode != 'tb'):
             # New input or when not in tb, continued input.
             # We do not check for continued input when in tb since it is
             # allowable to replace a long stack with an ellipsis.
             mode = 'input'
             if in1_match:
                 idx = in1_match.end()
             else: # in2_match
                 idx = in2_match.end()
             code = line[idx:]
             insertion = (0, Generic.Prompt, line[:idx])
             return mode, code, insertion
 
         # Check for input or continuation prompt (stripped version)
         in1_match_rstrip = self.in1_regex_rstrip.match(line)
         if in1_match_rstrip or (in2_match_rstrip and self.mode != 'tb'):
             # New input or when not in tb, continued input.
             # We do not check for continued input when in tb since it is
             # allowable to replace a long stack with an ellipsis.
             mode = 'input'
             if in1_match_rstrip:
                 idx = in1_match_rstrip.end()
             else: # in2_match
                 idx = in2_match_rstrip.end()
             code = line[idx:]
             insertion = (0, Generic.Prompt, line[:idx])
             return mode, code, insertion
 
         # Check for traceback
         if self.ipytb_start.match(line):
             mode = 'tb'
             code = line
             insertion = None
             return mode, code, insertion
 
         # All other stuff...
         if self.mode in ('input', 'output'):
             # We assume all other text is output. Multiline input that
             # does not use the continuation marker cannot be detected.
             # For example, the 3 in the following is clearly output:
             #
             #    In [1]: print 3
             #    3
             #
             # But the following second line is part of the input:
             #
             #    In [2]: while True:
             #        print True
             #
             # In both cases, the 2nd line will be 'output'.
             #
             mode = 'output'
         else:
             mode = 'tb'
 
         code = line
         insertion = None
 
         return mode, code, insertion
 
     def get_tokens_unprocessed(self, text):
         self.reset()
         for match in line_re.finditer(text):
             line = match.group()
             mode, code, insertion = self.get_mci(line)
 
             if mode != self.mode:
                 # Yield buffered tokens before transitioning to new mode.
                 for token in self.buffered_tokens():
                     yield token
                 self.mode = mode
 
             if insertion:
                 self.insertions.append((len(self.buffer), [insertion]))
             self.buffer += code
 
         for token in self.buffered_tokens():
             yield token
 
 class IPyLexer(Lexer):
     r"""
     Primary lexer for all IPython-like code.
 
     This is a simple helper lexer.  If the first line of the text begins with
     "In \[[0-9]+\]:", then the entire text is parsed with an IPython console
     lexer. If not, then the entire text is parsed with an IPython lexer.
 
     The goal is to reduce the number of lexers that are registered
     with Pygments.
 
     """
     name = 'IPy session'
     aliases = ['ipy']
 
     def __init__(self, **options):
         self.python3 = get_bool_opt(options, 'python3', False)
         if self.python3:
             self.aliases = ['ipy3']
         else:
             self.aliases = ['ipy2', 'ipy']
 
         Lexer.__init__(self, **options)
 
         self.IPythonLexer = IPythonLexer(**options)
         self.IPythonConsoleLexer = IPythonConsoleLexer(**options)
 
     def get_tokens_unprocessed(self, text):
         # Search for the input prompt anywhere...this allows code blocks to
         # begin with comments as well.
         if re.match(r'.*(In \[[0-9]+\]:)', text.strip(), re.DOTALL):
             lex = self.IPythonConsoleLexer
         else:
             lex = self.IPythonLexer
         for token in lex.get_tokens_unprocessed(text):
             yield token
 

IPython/lib/pretty.py

@@ -1,871 +1,856
 # -*- coding: utf-8 -*-
 """
 Python advanced pretty printer.  This pretty printer is intended to
 replace the old `pprint` python module which does not allow developers
 to provide their own pretty print callbacks.
 
 This module is based on ruby's `prettyprint.rb` library by `Tanaka Akira`.
 
 
 Example Usage
 -------------
 
 To directly print the representation of an object use `pprint`::
 
     from pretty import pprint
     pprint(complex_object)
 
 To get a string of the output use `pretty`::
 
     from pretty import pretty
     string = pretty(complex_object)
 
 
 Extending
 ---------
 
 The pretty library allows developers to add pretty printing rules for their
 own objects.  This process is straightforward.  All you have to do is to
 add a `_repr_pretty_` method to your object and call the methods on the
 pretty printer passed::
 
     class MyObject(object):
 
         def _repr_pretty_(self, p, cycle):
             ...
 
 Here is an example implementation of a `_repr_pretty_` method for a list
 subclass::
 
     class MyList(list):
 
         def _repr_pretty_(self, p, cycle):
             if cycle:
                 p.text('MyList(...)')
             else:
                 with p.group(8, 'MyList([', '])'):
                     for idx, item in enumerate(self):
                         if idx:
                             p.text(',')
                             p.breakable()
                         p.pretty(item)
 
 The `cycle` parameter is `True` if pretty detected a cycle.  You *have* to
 react to that or the result is an infinite loop.  `p.text()` just adds
 non breaking text to the output, `p.breakable()` either adds a whitespace
 or breaks here.  If you pass it an argument it's used instead of the
 default space.  `p.pretty` prettyprints another object using the pretty print
 method.
 
 The first parameter to the `group` function specifies the extra indentation
 of the next line.  In this example the next item will either be on the same
 line (if the items are short enough) or aligned with the right edge of the
 opening bracket of `MyList`.
 
 If you just want to indent something you can use the group function
 without open / close parameters.  You can also use this code::
 
     with p.indent(2):
         ...
 
 Inheritance diagram:
 
 .. inheritance-diagram:: IPython.lib.pretty
    :parts: 3
 
 :copyright: 2007 by Armin Ronacher.
             Portions (c) 2009 by Robert Kern.
 :license: BSD License.
 """
 
 from contextlib import contextmanager
 import datetime
 import os
 import re
 import sys
 import types
 from collections import deque
 from inspect import signature
 from io import StringIO
 from warnings import warn
 
 from IPython.utils.decorators import undoc
 from IPython.utils.py3compat import PYPY
 
 __all__ = ['pretty', 'pprint', 'PrettyPrinter', 'RepresentationPrinter',
     'for_type', 'for_type_by_name']
 
 
 MAX_SEQ_LENGTH = 1000
 _re_pattern_type = type(re.compile(''))
 
 def _safe_getattr(obj, attr, default=None):
     """Safe version of getattr.
 
     Same as getattr, but will return ``default`` on any Exception,
     rather than raising.
     """
     try:
         return getattr(obj, attr, default)
     except Exception:
         return default
 
 @undoc
 class CUnicodeIO(StringIO):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         warn(("CUnicodeIO is deprecated since IPython 6.0. "
               "Please use io.StringIO instead."),
              DeprecationWarning, stacklevel=2)
 
 def _sorted_for_pprint(items):
     """
     Sort the given items for pretty printing. Since some predictable
     sorting is better than no sorting at all, we sort on the string
     representation if normal sorting fails.
     """
     items = list(items)
     try:
         return sorted(items)
     except Exception:
         try:
             return sorted(items, key=str)
         except Exception:
             return items
 
 def pretty(obj, verbose=False, max_width=79, newline='\n', max_seq_length=MAX_SEQ_LENGTH):
     """
     Pretty print the object's representation.
     """
     stream = StringIO()
     printer = RepresentationPrinter(stream, verbose, max_width, newline, max_seq_length=max_seq_length)
     printer.pretty(obj)
     printer.flush()
     return stream.getvalue()
 
 
 def pprint(obj, verbose=False, max_width=79, newline='\n', max_seq_length=MAX_SEQ_LENGTH):
     """
     Like `pretty` but print to stdout.
     """
     printer = RepresentationPrinter(sys.stdout, verbose, max_width, newline, max_seq_length=max_seq_length)
     printer.pretty(obj)
     printer.flush()
     sys.stdout.write(newline)
     sys.stdout.flush()
 
 class _PrettyPrinterBase(object):
 
     @contextmanager
     def indent(self, indent):
         """with statement support for indenting/dedenting."""
         self.indentation += indent
         try:
             yield
         finally:
             self.indentation -= indent
 
     @contextmanager
     def group(self, indent=0, open='', close=''):
         """like begin_group / end_group but for the with statement."""
         self.begin_group(indent, open)
         try:
             yield
         finally:
             self.end_group(indent, close)
 
 class PrettyPrinter(_PrettyPrinterBase):
     """
     Baseclass for the `RepresentationPrinter` prettyprinter that is used to
     generate pretty reprs of objects.  Contrary to the `RepresentationPrinter`
     this printer knows nothing about the default pprinters or the `_repr_pretty_`
     callback method.
     """
 
     def __init__(self, output, max_width=79, newline='\n', max_seq_length=MAX_SEQ_LENGTH):
         self.output = output
         self.max_width = max_width
         self.newline = newline
         self.max_seq_length = max_seq_length
         self.output_width = 0
         self.buffer_width = 0
         self.buffer = deque()
 
         root_group = Group(0)
         self.group_stack = [root_group]
         self.group_queue = GroupQueue(root_group)
         self.indentation = 0
 
     def _break_one_group(self, group):
         while group.breakables:
             x = self.buffer.popleft()
             self.output_width = x.output(self.output, self.output_width)
             self.buffer_width -= x.width
         while self.buffer and isinstance(self.buffer[0], Text):
             x = self.buffer.popleft()
             self.output_width = x.output(self.output, self.output_width)
             self.buffer_width -= x.width
 
     def _break_outer_groups(self):
         while self.max_width < self.output_width + self.buffer_width:
             group = self.group_queue.deq()
             if not group:
                 return
             self._break_one_group(group)
 
     def text(self, obj):
         """Add literal text to the output."""
         width = len(obj)
         if self.buffer:
             text = self.buffer[-1]
             if not isinstance(text, Text):
                 text = Text()
                 self.buffer.append(text)
             text.add(obj, width)
             self.buffer_width += width
             self._break_outer_groups()
         else:
             self.output.write(obj)
             self.output_width += width
 
     def breakable(self, sep=' '):
         """
         Add a breakable separator to the output.  This does not mean that it
         will automatically break here.  If no breaking on this position takes
         place the `sep` is inserted which default to one space.
         """
         width = len(sep)
         group = self.group_stack[-1]
         if group.want_break:
             self.flush()
             self.output.write(self.newline)
             self.output.write(' ' * self.indentation)
             self.output_width = self.indentation
             self.buffer_width = 0
         else:
             self.buffer.append(Breakable(sep, width, self))
             self.buffer_width += width
             self._break_outer_groups()
 
     def break_(self):
         """
         Explicitly insert a newline into the output, maintaining correct indentation.
         """
         group = self.group_queue.deq()
         if group:
             self._break_one_group(group)
         self.flush()
         self.output.write(self.newline)
         self.output.write(' ' * self.indentation)
         self.output_width = self.indentation
         self.buffer_width = 0
 
 
     def begin_group(self, indent=0, open=''):
         """
-        Begin a group.  If you want support for python < 2.5 which doesn't has
-        the with statement this is the preferred way:
-
-            p.begin_group(1, '{')
-            ...
-            p.end_group(1, '}')
-
-        The python 2.5 expression would be this:
-
-            with p.group(1, '{', '}'):
-                ...
-
+        Begin a group.
         The first parameter specifies the indentation for the next line (usually
         the width of the opening text), the second the opening text.  All
         parameters are optional.
         """
         if open:
             self.text(open)
         group = Group(self.group_stack[-1].depth + 1)
         self.group_stack.append(group)
         self.group_queue.enq(group)
         self.indentation += indent
 
     def _enumerate(self, seq):
         """like enumerate, but with an upper limit on the number of items"""
         for idx, x in enumerate(seq):
             if self.max_seq_length and idx >= self.max_seq_length:
                 self.text(',')
                 self.breakable()
                 self.text('...')
                 return
             yield idx, x
 
     def end_group(self, dedent=0, close=''):
         """End a group. See `begin_group` for more details."""
         self.indentation -= dedent
         group = self.group_stack.pop()
         if not group.breakables:
             self.group_queue.remove(group)
         if close:
             self.text(close)
 
     def flush(self):
         """Flush data that is left in the buffer."""
         for data in self.buffer:
             self.output_width += data.output(self.output, self.output_width)
         self.buffer.clear()
         self.buffer_width = 0
 
 
 def _get_mro(obj_class):
     """ Get a reasonable method resolution order of a class and its superclasses
     for both old-style and new-style classes.
     """
     if not hasattr(obj_class, '__mro__'):
         # Old-style class. Mix in object to make a fake new-style class.
         try:
             obj_class = type(obj_class.__name__, (obj_class, object), {})
         except TypeError:
             # Old-style extension type that does not descend from object.
             # FIXME: try to construct a more thorough MRO.
             mro = [obj_class]
         else:
             mro = obj_class.__mro__[1:-1]
     else:
         mro = obj_class.__mro__
     return mro
 
 
 class RepresentationPrinter(PrettyPrinter):
     """
     Special pretty printer that has a `pretty` method that calls the pretty
     printer for a python object.
 
     This class stores processing data on `self` so you must *never* use
     this class in a threaded environment.  Always lock it or reinstanciate
     it.
 
     Instances also have a verbose flag callbacks can access to control their
     output.  For example the default instance repr prints all attributes and
     methods that are not prefixed by an underscore if the printer is in
     verbose mode.
     """
 
     def __init__(self, output, verbose=False, max_width=79, newline='\n',
         singleton_pprinters=None, type_pprinters=None, deferred_pprinters=None,
         max_seq_length=MAX_SEQ_LENGTH):
 
         PrettyPrinter.__init__(self, output, max_width, newline, max_seq_length=max_seq_length)
         self.verbose = verbose
         self.stack = []
         if singleton_pprinters is None:
             singleton_pprinters = _singleton_pprinters.copy()
         self.singleton_pprinters = singleton_pprinters
         if type_pprinters is None:
             type_pprinters = _type_pprinters.copy()
         self.type_pprinters = type_pprinters
         if deferred_pprinters is None:
             deferred_pprinters = _deferred_type_pprinters.copy()
         self.deferred_pprinters = deferred_pprinters
 
     def pretty(self, obj):
         """Pretty print the given object."""
         obj_id = id(obj)
         cycle = obj_id in self.stack
         self.stack.append(obj_id)
         self.begin_group()
         try:
             obj_class = _safe_getattr(obj, '__class__', None) or type(obj)
             # First try to find registered singleton printers for the type.
             try:
                 printer = self.singleton_pprinters[obj_id]
             except (TypeError, KeyError):
                 pass
             else:
                 return printer(obj, self, cycle)
             # Next walk the mro and check for either:
             #   1) a registered printer
             #   2) a _repr_pretty_ method
             for cls in _get_mro(obj_class):
                 if cls in self.type_pprinters:
                     # printer registered in self.type_pprinters
                     return self.type_pprinters[cls](obj, self, cycle)
                 else:
                     # deferred printer
                     printer = self._in_deferred_types(cls)
                     if printer is not None:
                         return printer(obj, self, cycle)
                     else:
                         # Finally look for special method names.
                         # Some objects automatically create any requested
                         # attribute. Try to ignore most of them by checking for
                         # callability.
                         if '_repr_pretty_' in cls.__dict__:
                             meth = cls._repr_pretty_
                             if callable(meth):
                                 return meth(obj, self, cycle)
                         if cls is not object \
                                 and callable(cls.__dict__.get('__repr__')):
                             return _repr_pprint(obj, self, cycle)
 
             return _default_pprint(obj, self, cycle)
         finally:
             self.end_group()
             self.stack.pop()
 
     def _in_deferred_types(self, cls):
         """
         Check if the given class is specified in the deferred type registry.
 
         Returns the printer from the registry if it exists, and None if the
         class is not in the registry. Successful matches will be moved to the
         regular type registry for future use.
         """
         mod = _safe_getattr(cls, '__module__', None)
         name = _safe_getattr(cls, '__name__', None)
         key = (mod, name)
         printer = None
         if key in self.deferred_pprinters:
             # Move the printer over to the regular registry.
             printer = self.deferred_pprinters.pop(key)
             self.type_pprinters[cls] = printer
         return printer
 
 
 class Printable(object):
 
     def output(self, stream, output_width):
         return output_width
 
 
 class Text(Printable):
 
     def __init__(self):
         self.objs = []
         self.width = 0
 
     def output(self, stream, output_width):
         for obj in self.objs:
             stream.write(obj)
         return output_width + self.width
 
     def add(self, obj, width):
         self.objs.append(obj)
         self.width += width
 
 
 class Breakable(Printable):
 
     def __init__(self, seq, width, pretty):
         self.obj = seq
         self.width = width
         self.pretty = pretty
         self.indentation = pretty.indentation
         self.group = pretty.group_stack[-1]
         self.group.breakables.append(self)
 
     def output(self, stream, output_width):
         self.group.breakables.popleft()
         if self.group.want_break:
             stream.write(self.pretty.newline)
             stream.write(' ' * self.indentation)
             return self.indentation
         if not self.group.breakables:
             self.pretty.group_queue.remove(self.group)
         stream.write(self.obj)
         return output_width + self.width
 
 
 class Group(Printable):
 
     def __init__(self, depth):
         self.depth = depth
         self.breakables = deque()
         self.want_break = False
 
 
 class GroupQueue(object):
 
     def __init__(self, *groups):
         self.queue = []
         for group in groups:
             self.enq(group)
 
     def enq(self, group):
         depth = group.depth
         while depth > len(self.queue) - 1:
             self.queue.append([])
         self.queue[depth].append(group)
 
     def deq(self):
         for stack in self.queue:
             for idx, group in enumerate(reversed(stack)):
                 if group.breakables:
                     del stack[idx]
                     group.want_break = True
                     return group
             for group in stack:
                 group.want_break = True
             del stack[:]
 
     def remove(self, group):
         try:
             self.queue[group.depth].remove(group)
         except ValueError:
             pass
 
 
 def _default_pprint(obj, p, cycle):
     """
     The default print function.  Used if an object does not provide one and
     it's none of the builtin objects.
     """
     klass = _safe_getattr(obj, '__class__', None) or type(obj)
     if _safe_getattr(klass, '__repr__', None) is not object.__repr__:
         # A user-provided repr. Find newlines and replace them with p.break_()
         _repr_pprint(obj, p, cycle)
         return
     p.begin_group(1, '<')
     p.pretty(klass)
     p.text(' at 0x%x' % id(obj))
     if cycle:
         p.text(' ...')
     elif p.verbose:
         first = True
         for key in dir(obj):
             if not key.startswith('_'):
                 try:
                     value = getattr(obj, key)
                 except AttributeError:
                     continue
                 if isinstance(value, types.MethodType):
                     continue
                 if not first:
                     p.text(',')
                 p.breakable()
                 p.text(key)
                 p.text('=')
                 step = len(key) + 1
                 p.indentation += step
                 p.pretty(value)
                 p.indentation -= step
                 first = False
     p.end_group(1, '>')
 
 
 def _seq_pprinter_factory(start, end):
     """
     Factory that returns a pprint function useful for sequences.  Used by
     the default pprint for tuples, dicts, and lists.
     """
     def inner(obj, p, cycle):
         if cycle:
             return p.text(start + '...' + end)
         step = len(start)
         p.begin_group(step, start)
         for idx, x in p._enumerate(obj):
             if idx:
                 p.text(',')
                 p.breakable()
             p.pretty(x)
         if len(obj) == 1 and type(obj) is tuple:
             # Special case for 1-item tuples.
             p.text(',')
         p.end_group(step, end)
     return inner
 
 
 def _set_pprinter_factory(start, end):
     """
     Factory that returns a pprint function useful for sets and frozensets.
     """
     def inner(obj, p, cycle):
         if cycle:
             return p.text(start + '...' + end)
         if len(obj) == 0:
             # Special case.
             p.text(type(obj).__name__ + '()')
         else:
             step = len(start)
             p.begin_group(step, start)
             # Like dictionary keys, we will try to sort the items if there aren't too many
             if not (p.max_seq_length and len(obj) >= p.max_seq_length):
                 items = _sorted_for_pprint(obj)
             else:
                 items = obj
             for idx, x in p._enumerate(items):
                 if idx:
                     p.text(',')
                     p.breakable()
                 p.pretty(x)
             p.end_group(step, end)
     return inner
 
 
 def _dict_pprinter_factory(start, end):
     """
     Factory that returns a pprint function used by the default pprint of
     dicts and dict proxies.
     """
     def inner(obj, p, cycle):
         if cycle:
             return p.text('{...}')
         step = len(start)
         p.begin_group(step, start)
         keys = obj.keys()
         for idx, key in p._enumerate(keys):
             if idx:
                 p.text(',')
                 p.breakable()
             p.pretty(key)
             p.text(': ')
             p.pretty(obj[key])
         p.end_group(step, end)
     return inner
 
 
 def _super_pprint(obj, p, cycle):
     """The pprint for the super type."""
     p.begin_group(8, '<super: ')
     p.pretty(obj.__thisclass__)
     p.text(',')
     p.breakable()
     if PYPY: # In PyPy, super() objects don't have __self__ attributes
         dself = obj.__repr__.__self__
         p.pretty(None if dself is obj else dself)
     else:
         p.pretty(obj.__self__)
     p.end_group(8, '>')
 
 
 def _re_pattern_pprint(obj, p, cycle):
     """The pprint function for regular expression patterns."""
     p.text('re.compile(')
     pattern = repr(obj.pattern)
     if pattern[:1] in 'uU':
         pattern = pattern[1:]
         prefix = 'ur'
     else:
         prefix = 'r'
     pattern = prefix + pattern.replace('\\\\', '\\')
     p.text(pattern)
     if obj.flags:
         p.text(',')
         p.breakable()
         done_one = False
         for flag in ('TEMPLATE', 'IGNORECASE', 'LOCALE', 'MULTILINE', 'DOTALL',
             'UNICODE', 'VERBOSE', 'DEBUG'):
             if obj.flags & getattr(re, flag):
                 if done_one:
                     p.text('|')
                 p.text('re.' + flag)
                 done_one = True
     p.text(')')
 
 
 def _type_pprint(obj, p, cycle):
     """The pprint for classes and types."""
     # Heap allocated types might not have the module attribute,
     # and others may set it to None.
 
     # Checks for a __repr__ override in the metaclass. Can't compare the
     # type(obj).__repr__ directly because in PyPy the representation function
     # inherited from type isn't the same type.__repr__
     if [m for m in _get_mro(type(obj)) if "__repr__" in vars(m)][:1] != [type]:
         _repr_pprint(obj, p, cycle)
         return
 
     mod = _safe_getattr(obj, '__module__', None)
     try:
         name = obj.__qualname__
         if not isinstance(name, str):
             # This can happen if the type implements __qualname__ as a property
             # or other descriptor in Python 2.
             raise Exception("Try __name__")
     except Exception:
         name = obj.__name__
         if not isinstance(name, str):
             name = '<unknown type>'
 
     if mod in (None, '__builtin__', 'builtins', 'exceptions'):
         p.text(name)
     else:
         p.text(mod + '.' + name)
 
 
 def _repr_pprint(obj, p, cycle):
     """A pprint that just redirects to the normal repr function."""
     # Find newlines and replace them with p.break_()
     output = repr(obj)
     lines = output.splitlines()
     with p.group():
         for idx, output_line in enumerate(lines):
             if idx:
                 p.break_()
             p.text(output_line)
 
 
 def _function_pprint(obj, p, cycle):
     """Base pprint for all functions and builtin functions."""
     name = _safe_getattr(obj, '__qualname__', obj.__name__)
     mod = obj.__module__
     if mod and mod not in ('__builtin__', 'builtins', 'exceptions'):
         name = mod + '.' + name
     try:
        func_def = name + str(signature(obj))
     except ValueError:
        func_def = name
     p.text('<function %s>' % func_def)
 
 
 def _exception_pprint(obj, p, cycle):
     """Base pprint for all exceptions."""
     name = getattr(obj.__class__, '__qualname__', obj.__class__.__name__)
     if obj.__class__.__module__ not in ('exceptions', 'builtins'):
         name = '%s.%s' % (obj.__class__.__module__, name)
     step = len(name) + 1
     p.begin_group(step, name + '(')
     for idx, arg in enumerate(getattr(obj, 'args', ())):
         if idx:
             p.text(',')
             p.breakable()
         p.pretty(arg)
     p.end_group(step, ')')
 
 
 #: the exception base
 try:
     _exception_base = BaseException
 except NameError:
     _exception_base = Exception
 
 
 #: printers for builtin types
 _type_pprinters = {
     int:                        _repr_pprint,
     float:                      _repr_pprint,
     str:                        _repr_pprint,
     tuple:                      _seq_pprinter_factory('(', ')'),
     list:                       _seq_pprinter_factory('[', ']'),
     dict:                       _dict_pprinter_factory('{', '}'),
     set:                        _set_pprinter_factory('{', '}'),
     frozenset:                  _set_pprinter_factory('frozenset({', '})'),
     super:                      _super_pprint,
     _re_pattern_type:           _re_pattern_pprint,
     type:                       _type_pprint,
     types.FunctionType:         _function_pprint,
     types.BuiltinFunctionType:  _function_pprint,
     types.MethodType:           _repr_pprint,
     datetime.datetime:          _repr_pprint,
     datetime.timedelta:         _repr_pprint,
     _exception_base:            _exception_pprint
 }
 
 # render os.environ like a dict
 _env_type = type(os.environ)
 # future-proof in case os.environ becomes a plain dict?
 if _env_type is not dict:
     _type_pprinters[_env_type] = _dict_pprinter_factory('environ{', '}')
 
 try:
     # In PyPy, types.DictProxyType is dict, setting the dictproxy printer
     # using dict.setdefault avoids overwriting the dict printer
     _type_pprinters.setdefault(types.DictProxyType,
                                _dict_pprinter_factory('dict_proxy({', '})'))
     _type_pprinters[types.ClassType] = _type_pprint
     _type_pprinters[types.SliceType] = _repr_pprint
 except AttributeError: # Python 3
     _type_pprinters[types.MappingProxyType] = \
         _dict_pprinter_factory('mappingproxy({', '})')
     _type_pprinters[slice] = _repr_pprint
 
-try:
-    _type_pprinters[long] = _repr_pprint
-    _type_pprinters[unicode] = _repr_pprint
-except NameError:
     _type_pprinters[range] = _repr_pprint
     _type_pprinters[bytes] = _repr_pprint
 
 #: printers for types specified by name
 _deferred_type_pprinters = {
 }
 
 def for_type(typ, func):
     """
     Add a pretty printer for a given type.
     """
     oldfunc = _type_pprinters.get(typ, None)
     if func is not None:
         # To support easy restoration of old pprinters, we need to ignore Nones.
         _type_pprinters[typ] = func
     return oldfunc
 
 def for_type_by_name(type_module, type_name, func):
     """
     Add a pretty printer for a type specified by the module and name of a type
     rather than the type object itself.
     """
     key = (type_module, type_name)
     oldfunc = _deferred_type_pprinters.get(key, None)
     if func is not None:
         # To support easy restoration of old pprinters, we need to ignore Nones.
         _deferred_type_pprinters[key] = func
     return oldfunc
 
 
 #: printers for the default singletons
 _singleton_pprinters = dict.fromkeys(map(id, [None, True, False, Ellipsis,
                                       NotImplemented]), _repr_pprint)
 
 
 def _defaultdict_pprint(obj, p, cycle):
     name = obj.__class__.__name__
     with p.group(len(name) + 1, name + '(', ')'):
         if cycle:
             p.text('...')
         else:
             p.pretty(obj.default_factory)
             p.text(',')
             p.breakable()
             p.pretty(dict(obj))
 
 def _ordereddict_pprint(obj, p, cycle):
     name = obj.__class__.__name__
     with p.group(len(name) + 1, name + '(', ')'):
         if cycle:
             p.text('...')
         elif len(obj):
             p.pretty(list(obj.items()))
 
 def _deque_pprint(obj, p, cycle):
     name = obj.__class__.__name__
     with p.group(len(name) + 1, name + '(', ')'):
         if cycle:
             p.text('...')
         else:
             p.pretty(list(obj))
 
 
 def _counter_pprint(obj, p, cycle):
     name = obj.__class__.__name__
     with p.group(len(name) + 1, name + '(', ')'):
         if cycle:
             p.text('...')
         elif len(obj):
             p.pretty(dict(obj))
 
 for_type_by_name('collections', 'defaultdict', _defaultdict_pprint)
 for_type_by_name('collections', 'OrderedDict', _ordereddict_pprint)
 for_type_by_name('collections', 'deque', _deque_pprint)
 for_type_by_name('collections', 'Counter', _counter_pprint)
 
 if __name__ == '__main__':
     from random import randrange
     class Foo(object):
         def __init__(self):
             self.foo = 1
             self.bar = re.compile(r'\s+')
             self.blub = dict.fromkeys(range(30), randrange(1, 40))
             self.hehe = 23424.234234
             self.list = ["blub", "blah", self]
 
         def get_foo(self):
             print("foo")
 
     pprint(Foo(), verbose=True)

IPython/lib/tests/test_display.py

@@ -1,270 +1,266
 """Tests for IPython.lib.display.
 
 """
 #-----------------------------------------------------------------------------
 # Copyright (c) 2012, the IPython Development Team.
 #
 # Distributed under the terms of the Modified BSD License.
 #
 # The full license is in the file COPYING.txt, distributed with this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 from tempfile import NamedTemporaryFile, mkdtemp
 from os.path import split, join as pjoin, dirname
-import sys
-try:
     import pathlib
-except ImportError:
-    pass
 from unittest import TestCase, mock
 import struct
 import wave
 from io import BytesIO
 
 # Third-party imports
 import nose.tools as nt
 
 try:
     import numpy
 except ImportError:
     pass
 
 # Our own imports
 from IPython.lib import display
 
 from IPython.testing.decorators import skipif_not_numpy
 
 #-----------------------------------------------------------------------------
 # Classes and functions
 #-----------------------------------------------------------------------------
 
 #--------------------------
 # FileLink tests
 #--------------------------
 
 def test_instantiation_FileLink():
     """FileLink: Test class can be instantiated"""
     fl = display.FileLink('example.txt')
     # TODO: remove if when only Python >= 3.6 is supported
     fl = display.FileLink(pathlib.PurePath('example.txt'))
 
 def test_warning_on_non_existent_path_FileLink():
     """FileLink: Calling _repr_html_ on non-existent files returns a warning
     """
     fl = display.FileLink('example.txt')
     nt.assert_true(fl._repr_html_().startswith('Path (<tt>example.txt</tt>)'))
 
 def test_existing_path_FileLink():
     """FileLink: Calling _repr_html_ functions as expected on existing filepath
     """
     tf = NamedTemporaryFile()
     fl = display.FileLink(tf.name)
     actual = fl._repr_html_()
     expected = "<a href='%s' target='_blank'>%s</a><br>" % (tf.name,tf.name)
     nt.assert_equal(actual,expected)
 
 def test_existing_path_FileLink_repr():
     """FileLink: Calling repr() functions as expected on existing filepath
     """
     tf = NamedTemporaryFile()
     fl = display.FileLink(tf.name)
     actual = repr(fl)
     expected = tf.name
     nt.assert_equal(actual,expected)
 
 def test_error_on_directory_to_FileLink():
     """FileLink: Raises error when passed directory
     """
     td = mkdtemp()
     nt.assert_raises(ValueError,display.FileLink,td)
 
 #--------------------------
 # FileLinks tests
 #--------------------------
 
 def test_instantiation_FileLinks():
     """FileLinks: Test class can be instantiated
     """
     fls = display.FileLinks('example')
 
 def test_warning_on_non_existent_path_FileLinks():
     """FileLinks: Calling _repr_html_ on non-existent files returns a warning
     """
     fls = display.FileLinks('example')
     nt.assert_true(fls._repr_html_().startswith('Path (<tt>example</tt>)'))
 
 def test_existing_path_FileLinks():
     """FileLinks: Calling _repr_html_ functions as expected on existing dir
     """
     td = mkdtemp()
     tf1 = NamedTemporaryFile(dir=td)
     tf2 = NamedTemporaryFile(dir=td)
     fl = display.FileLinks(td)
     actual = fl._repr_html_()
     actual = actual.split('\n')
     actual.sort()
     # the links should always have forward slashes, even on windows, so replace
     # backslashes with forward slashes here
     expected = ["%s/<br>" % td,
                 "&nbsp;&nbsp;<a href='%s' target='_blank'>%s</a><br>" %\
                  (tf2.name.replace("\\","/"),split(tf2.name)[1]),
                 "&nbsp;&nbsp;<a href='%s' target='_blank'>%s</a><br>" %\
                  (tf1.name.replace("\\","/"),split(tf1.name)[1])]
     expected.sort()
     # We compare the sorted list of links here as that's more reliable
     nt.assert_equal(actual,expected)
 
 def test_existing_path_FileLinks_alt_formatter():
     """FileLinks: Calling _repr_html_ functions as expected w/ an alt formatter
     """
     td = mkdtemp()
     tf1 = NamedTemporaryFile(dir=td)
     tf2 = NamedTemporaryFile(dir=td)
     def fake_formatter(dirname,fnames,included_suffixes):
         return ["hello","world"]
     fl = display.FileLinks(td,notebook_display_formatter=fake_formatter)
     actual = fl._repr_html_()
     actual = actual.split('\n')
     actual.sort()
     expected = ["hello","world"]
     expected.sort()
     # We compare the sorted list of links here as that's more reliable
     nt.assert_equal(actual,expected)
 
 def test_existing_path_FileLinks_repr():
     """FileLinks: Calling repr() functions as expected on existing directory """
     td = mkdtemp()
     tf1 = NamedTemporaryFile(dir=td)
     tf2 = NamedTemporaryFile(dir=td)
     fl = display.FileLinks(td)
     actual = repr(fl)
     actual = actual.split('\n')
     actual.sort()
     expected = ['%s/' % td, '  %s' % split(tf1.name)[1],'  %s' % split(tf2.name)[1]]
     expected.sort()
     # We compare the sorted list of links here as that's more reliable
     nt.assert_equal(actual,expected)
     
 def test_existing_path_FileLinks_repr_alt_formatter():
     """FileLinks: Calling repr() functions as expected w/ alt formatter
     """
     td = mkdtemp()
     tf1 = NamedTemporaryFile(dir=td)
     tf2 = NamedTemporaryFile(dir=td)
     def fake_formatter(dirname,fnames,included_suffixes):
         return ["hello","world"]
     fl = display.FileLinks(td,terminal_display_formatter=fake_formatter)
     actual = repr(fl)
     actual = actual.split('\n')
     actual.sort()
     expected = ["hello","world"]
     expected.sort()
     # We compare the sorted list of links here as that's more reliable
     nt.assert_equal(actual,expected)
     
 def test_error_on_file_to_FileLinks():
     """FileLinks: Raises error when passed file
     """
     td = mkdtemp()
     tf1 = NamedTemporaryFile(dir=td)
     nt.assert_raises(ValueError,display.FileLinks,tf1.name)
 
 def test_recursive_FileLinks():
     """FileLinks: Does not recurse when recursive=False
     """
     td = mkdtemp()
     tf = NamedTemporaryFile(dir=td)
     subtd = mkdtemp(dir=td)
     subtf = NamedTemporaryFile(dir=subtd)
     fl = display.FileLinks(td)
     actual = str(fl)
     actual = actual.split('\n')
     nt.assert_equal(len(actual), 4, actual)
     fl = display.FileLinks(td, recursive=False)
     actual = str(fl)
     actual = actual.split('\n')
     nt.assert_equal(len(actual), 2, actual)
 
 def test_audio_from_file():
     path = pjoin(dirname(__file__), 'test.wav')
     display.Audio(filename=path)
 
 class TestAudioDataWithNumpy(TestCase):
 
     @skipif_not_numpy
     def test_audio_from_numpy_array(self):
         test_tone = get_test_tone()
         audio = display.Audio(test_tone, rate=44100)
         nt.assert_equal(len(read_wav(audio.data)), len(test_tone))
 
     @skipif_not_numpy
     def test_audio_from_list(self):
         test_tone = get_test_tone()
         audio = display.Audio(list(test_tone), rate=44100)
         nt.assert_equal(len(read_wav(audio.data)), len(test_tone))
 
     @skipif_not_numpy
     def test_audio_from_numpy_array_without_rate_raises(self):
         nt.assert_raises(ValueError, display.Audio, get_test_tone())
 
     @skipif_not_numpy
     def test_audio_data_normalization(self):
         expected_max_value = numpy.iinfo(numpy.int16).max
         for scale in [1, 0.5, 2]:
             audio = display.Audio(get_test_tone(scale), rate=44100)
             actual_max_value = numpy.max(numpy.abs(read_wav(audio.data)))
             nt.assert_equal(actual_max_value, expected_max_value)
 
     @skipif_not_numpy
     def test_audio_data_without_normalization(self):
         max_int16 = numpy.iinfo(numpy.int16).max
         for scale in [1, 0.5, 0.2]:
             test_tone = get_test_tone(scale)
             test_tone_max_abs = numpy.max(numpy.abs(test_tone))
             expected_max_value = int(max_int16 * test_tone_max_abs)
             audio = display.Audio(test_tone, rate=44100, normalize=False)
             actual_max_value = numpy.max(numpy.abs(read_wav(audio.data)))
             nt.assert_equal(actual_max_value, expected_max_value)
 
     def test_audio_data_without_normalization_raises_for_invalid_data(self):
         nt.assert_raises(
             ValueError,
             lambda: display.Audio([1.001], rate=44100, normalize=False))
         nt.assert_raises(
             ValueError,
             lambda: display.Audio([-1.001], rate=44100, normalize=False))
 
 def simulate_numpy_not_installed():
     try:
         import numpy
         return mock.patch('numpy.array', mock.MagicMock(side_effect=ImportError))
     except ModuleNotFoundError:
         return lambda x:x
 
 @simulate_numpy_not_installed()
 class TestAudioDataWithoutNumpy(TestAudioDataWithNumpy):
     # All tests from `TestAudioDataWithNumpy` are inherited.
 
     @skipif_not_numpy
     def test_audio_raises_for_nested_list(self):
         stereo_signal = [list(get_test_tone())] * 2
         nt.assert_raises(
             TypeError,
             lambda: display.Audio(stereo_signal, rate=44100))
 
 @skipif_not_numpy
 def get_test_tone(scale=1):
     return numpy.sin(2 * numpy.pi * 440 * numpy.linspace(0, 1, 44100)) * scale
 
 def read_wav(data):
     with wave.open(BytesIO(data)) as wave_file:
         wave_data = wave_file.readframes(wave_file.getnframes())
         num_samples = wave_file.getnframes() * wave_file.getnchannels()
         return struct.unpack('<%sh' % num_samples, wave_data)
 
 def test_code_from_file():
     c = display.Code(filename=__file__)
     assert c._repr_html_().startswith('<style>')

IPython/utils/encoding.py

@@ -1,71 +1,71
 # coding: utf-8
 """
 Utilities for dealing with text encodings
 """
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2008-2012  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 import sys
 import locale
 import warnings
 
 # to deal with the possibility of sys.std* not being a stream at all
 def get_stream_enc(stream, default=None):
     """Return the given stream's encoding or a default.
 
     There are cases where ``sys.std*`` might not actually be a stream, so
     check for the encoding attribute prior to returning it, and return
     a default if it doesn't exist or evaluates as False. ``default``
     is None if not provided.
     """
     if not hasattr(stream, 'encoding') or not stream.encoding:
         return default
     else:
         return stream.encoding
 
 # Less conservative replacement for sys.getdefaultencoding, that will try
 # to match the environment.
 # Defined here as central function, so if we find better choices, we
 # won't need to make changes all over IPython.
 def getdefaultencoding(prefer_stream=True):
     """Return IPython's guess for the default encoding for bytes as text.
     
     If prefer_stream is True (default), asks for stdin.encoding first,
     to match the calling Terminal, but that is often None for subprocesses.
     
     Then fall back on locale.getpreferredencoding(),
     which should be a sensible platform default (that respects LANG environment),
     and finally to sys.getdefaultencoding() which is the most conservative option,
-    and usually ASCII on Python 2 or UTF8 on Python 3.
+    and usually UTF8 as of Python 3.
     """
     enc = None
     if prefer_stream:
         enc = get_stream_enc(sys.stdin)
     if not enc or enc=='ascii':
         try:
             # There are reports of getpreferredencoding raising errors
             # in some cases, which may well be fixed, but let's be conservative here.
             enc = locale.getpreferredencoding()
         except Exception:
             pass
     enc = enc or sys.getdefaultencoding()
     # On windows `cp0` can be returned to indicate that there is no code page.
     # Since cp0 is an invalid encoding return instead cp1252 which is the
     # Western European default.
     if enc == 'cp0':
         warnings.warn(
             "Invalid code page cp0 detected - using cp1252 instead."
             "If cp1252 is incorrect please ensure a valid code page "
             "is defined for the process.", RuntimeWarning)
         return 'cp1252'
     return enc
 
 DEFAULT_ENCODING = getdefaultencoding()

IPython/utils/openpy.py

@@ -1,105 +1,103
 """
 Tools to open .py files as Unicode, using the encoding specified within the file,
 as per PEP 263.
 
 Much of the code is taken from the tokenize module in Python 3.2.
 """
 
 import io
 from io import TextIOWrapper, BytesIO
 import re
 from tokenize import open, detect_encoding
 
 cookie_re = re.compile(r"coding[:=]\s*([-\w.]+)", re.UNICODE)
 cookie_comment_re = re.compile(r"^\s*#.*coding[:=]\s*([-\w.]+)", re.UNICODE)
 
 def source_to_unicode(txt, errors='replace', skip_encoding_cookie=True):
     """Converts a bytes string with python source code to unicode.
 
     Unicode strings are passed through unchanged. Byte strings are checked
     for the python source file encoding cookie to determine encoding.
     txt can be either a bytes buffer or a string containing the source
     code.
     """
     if isinstance(txt, str):
         return txt
     if isinstance(txt, bytes):
         buffer = BytesIO(txt)
     else:
         buffer = txt
     try:
         encoding, _ = detect_encoding(buffer.readline)
     except SyntaxError:
         encoding = "ascii"
     buffer.seek(0)
     with TextIOWrapper(buffer, encoding, errors=errors, line_buffering=True) as text:
         text.mode = 'r'
         if skip_encoding_cookie:
             return u"".join(strip_encoding_cookie(text))
         else:
             return text.read()
 
 def strip_encoding_cookie(filelike):
     """Generator to pull lines from a text-mode file, skipping the encoding
     cookie if it is found in the first two lines.
     """
     it = iter(filelike)
     try:
         first = next(it)
         if not cookie_comment_re.match(first):
             yield first
         second = next(it)
         if not cookie_comment_re.match(second):
             yield second
     except StopIteration:
         return
     
     for line in it:
         yield line
 
 def read_py_file(filename, skip_encoding_cookie=True):
     """Read a Python file, using the encoding declared inside the file.
     
     Parameters
     ----------
     filename : str
       The path to the file to read.
     skip_encoding_cookie : bool
       If True (the default), and the encoding declaration is found in the first
-      two lines, that line will be excluded from the output - compiling a
-      unicode string with an encoding declaration is a SyntaxError in Python 2.
+      two lines, that line will be excluded from the output.
     
     Returns
     -------
     A unicode string containing the contents of the file.
     """
     with open(filename) as f:   # the open function defined in this module.
         if skip_encoding_cookie:
             return "".join(strip_encoding_cookie(f))
         else:
             return f.read()
 
 def read_py_url(url, errors='replace', skip_encoding_cookie=True):
     """Read a Python file from a URL, using the encoding declared inside the file.
     
     Parameters
     ----------
     url : str
       The URL from which to fetch the file.
     errors : str
       How to handle decoding errors in the file. Options are the same as for
       bytes.decode(), but here 'replace' is the default.
     skip_encoding_cookie : bool
       If True (the default), and the encoding declaration is found in the first
-      two lines, that line will be excluded from the output - compiling a
-      unicode string with an encoding declaration is a SyntaxError in Python 2.
+      two lines, that line will be excluded from the output.
     
     Returns
     -------
     A unicode string containing the contents of the file.
     """
     # Deferred import for faster start
     from urllib.request import urlopen 
     response = urlopen(url)
     buffer = io.BytesIO(response.read())
     return source_to_unicode(buffer, errors, skip_encoding_cookie)

examples/IPython Kernel/ipython-get-history.py

@@ -1,39 +1,38
 #!/usr/bin/env python
 """Extract a session from the IPython input history.
 
 Usage:
   ipython-get-history.py sessionnumber [outputfile]
 
 If outputfile is not given, the relevant history is written to stdout. If
 outputfile has a .py extension, the translated history (without IPython's
 special syntax) will be extracted.
 
 Example:
   ./ipython-get-history.py 57 record.ipy
 
 
 This script is a simple demonstration of HistoryAccessor. It should be possible
 to build much more flexible and powerful tools to browse and pull from the
 history database.
 """
 import sys
 
 from IPython.core.history import HistoryAccessor
 
 session_number = int(sys.argv[1])
 if len(sys.argv) > 2:
     dest = open(sys.argv[2], "w")
     raw = not sys.argv[2].endswith('.py')
 else:
     dest = sys.stdout
     raw = True
 
 with dest:
     dest.write("# coding: utf-8\n")
 
     # Profiles other than 'default' can be specified here with a profile= argument:
     hist = HistoryAccessor()
 
     for session, lineno, cell in hist.get_range(session=session_number, raw=raw):
-      cell = cell.encode('utf-8')  # This line is only needed on Python 2.
       dest.write(cell + '\n')