Merge pull request #10135 from srinivasreddy/dead_code

remove python2 specific code

IPython/core/oinspect.py

@@ -1,987 +1,980 b''
 # -*- coding: utf-8 -*-
 """Tools for inspecting Python objects.
 
 Uses syntax highlighting for presenting the various information elements.
 
 Similar in spirit to the inspect module, but all calls take a name argument to
 reference the name under which an object is being read.
 """
 
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
 __all__ = ['Inspector','InspectColors']
 
 # stdlib modules
 import inspect
 from inspect import signature
 import linecache
 import warnings
 import os
 from textwrap import dedent
 import types
 import io as stdlib_io
 from itertools import zip_longest 
 
 # IPython's own
 from IPython.core import page
 from IPython.lib.pretty import pretty
 from IPython.testing.skipdoctest import skip_doctest
 from IPython.utils import PyColorize
 from IPython.utils import openpy
 from IPython.utils import py3compat
 from IPython.utils.dir2 import safe_hasattr
 from IPython.utils.path import compress_user
 from IPython.utils.text import indent
 from IPython.utils.wildcard import list_namespace
 from IPython.utils.coloransi import TermColors, ColorScheme, ColorSchemeTable
-from IPython.utils.py3compat import cast_unicode, PY3
+from IPython.utils.py3compat import cast_unicode
 from IPython.utils.colorable import Colorable
 from IPython.utils.decorators import undoc
 
 from pygments import highlight
 from pygments.lexers import PythonLexer
 from pygments.formatters import HtmlFormatter
 
 def pylight(code):
     return highlight(code, PythonLexer(), HtmlFormatter(noclasses=True))
 
 # builtin docstrings to ignore
 _func_call_docstring = types.FunctionType.__call__.__doc__
 _object_init_docstring = object.__init__.__doc__
 _builtin_type_docstrings = {
     inspect.getdoc(t) for t in (types.ModuleType, types.MethodType,
                                 types.FunctionType, property)
 }
 
 _builtin_func_type = type(all)
 _builtin_meth_type = type(str.upper)  # Bound methods have the same type as builtin functions
 #****************************************************************************
 # Builtin color schemes
 
 Colors = TermColors  # just a shorthand
 
 InspectColors = PyColorize.ANSICodeColors
 
 #****************************************************************************
 # Auxiliary functions and objects
 
 # See the messaging spec for the definition of all these fields.  This list
 # effectively defines the order of display
 info_fields = ['type_name', 'base_class', 'string_form', 'namespace',
                'length', 'file', 'definition', 'docstring', 'source',
                'init_definition', 'class_docstring', 'init_docstring',
                'call_def', 'call_docstring',
                # These won't be printed but will be used to determine how to
                # format the object
                'ismagic', 'isalias', 'isclass', 'argspec', 'found', 'name'
                ]
 
 
 def object_info(**kw):
     """Make an object info dict with all fields present."""
     infodict = dict(zip_longest(info_fields, [None]))
     infodict.update(kw)
     return infodict
 
 
 def get_encoding(obj):
     """Get encoding for python source file defining obj
 
     Returns None if obj is not defined in a sourcefile.
     """
     ofile = find_file(obj)
     # run contents of file through pager starting at line where the object
     # is defined, as long as the file isn't binary and is actually on the
     # filesystem.
     if ofile is None:
         return None
     elif ofile.endswith(('.so', '.dll', '.pyd')):
         return None
     elif not os.path.isfile(ofile):
         return None
     else:
         # Print only text files, not extension binaries.  Note that
         # getsourcelines returns lineno with 1-offset and page() uses
         # 0-offset, so we must adjust.
         with stdlib_io.open(ofile, 'rb') as buffer:   # Tweaked to use io.open for Python 2
             encoding, lines = openpy.detect_encoding(buffer.readline)
         return encoding
 
 def getdoc(obj):
     """Stable wrapper around inspect.getdoc.
 
     This can't crash because of attribute problems.
 
     It also attempts to call a getdoc() method on the given object.  This
     allows objects which provide their docstrings via non-standard mechanisms
     (like Pyro proxies) to still be inspected by ipython's ? system.
     """
     # Allow objects to offer customized documentation via a getdoc method:
     try:
         ds = obj.getdoc()
     except Exception:
         pass
     else:
         # if we get extra info, we add it to the normal docstring.
         if isinstance(ds, str):
             return inspect.cleandoc(ds)
     try:
         docstr = inspect.getdoc(obj)
         encoding = get_encoding(obj)
         return py3compat.cast_unicode(docstr, encoding=encoding)
     except Exception:
         # Harden against an inspect failure, which can occur with
         # extensions modules.
         raise
         return None
 
 
 def getsource(obj, oname=''):
     """Wrapper around inspect.getsource.
 
     This can be modified by other projects to provide customized source
     extraction.
 
     Parameters
     ----------
     obj : object
         an object whose source code we will attempt to extract
     oname : str
         (optional) a name under which the object is known
 
     Returns
     -------
     src : unicode or None
 
     """
 
     if isinstance(obj, property):
         sources = []
         for attrname in ['fget', 'fset', 'fdel']:
             fn = getattr(obj, attrname)
             if fn is not None:
                 encoding = get_encoding(fn)
                 oname_prefix = ('%s.' % oname) if oname else ''
                 sources.append(cast_unicode(
                     ''.join(('# ', oname_prefix, attrname)),
                     encoding=encoding))
                 if inspect.isfunction(fn):
                     sources.append(dedent(getsource(fn)))
                 else:
                     # Default str/repr only prints function name,
                     # pretty.pretty prints module name too.
                     sources.append(cast_unicode(
                         '%s%s = %s\n' % (
                             oname_prefix, attrname, pretty(fn)),
                         encoding=encoding))
         if sources:
             return '\n'.join(sources)
         else:
             return None
 
     else:
         # Get source for non-property objects.
 
         obj = _get_wrapped(obj)
 
         try:
             src = inspect.getsource(obj)
         except TypeError:
             # The object itself provided no meaningful source, try looking for
             # its class definition instead.
             if hasattr(obj, '__class__'):
                 try:
                     src = inspect.getsource(obj.__class__)
                 except TypeError:
                     return None
 
         encoding = get_encoding(obj)
         return cast_unicode(src, encoding=encoding)
 
 
 def is_simple_callable(obj):
     """True if obj is a function ()"""
     return (inspect.isfunction(obj) or inspect.ismethod(obj) or \
             isinstance(obj, _builtin_func_type) or isinstance(obj, _builtin_meth_type))
 
 
 def getargspec(obj):
     """Wrapper around :func:`inspect.getfullargspec` on Python 3, and
     :func:inspect.getargspec` on Python 2.
     
     In addition to functions and methods, this can also handle objects with a
     ``__call__`` attribute.
     """
     if safe_hasattr(obj, '__call__') and not is_simple_callable(obj):
         obj = obj.__call__
 
-    return inspect.getfullargspec(obj) if PY3 else inspect.getargspec(obj)
+    return inspect.getfullargspec(obj)
 
 
 def format_argspec(argspec):
     """Format argspect, convenience wrapper around inspect's.
 
     This takes a dict instead of ordered arguments and calls
     inspect.format_argspec with the arguments in the necessary order.
     """
     return inspect.formatargspec(argspec['args'], argspec['varargs'],
                                  argspec['varkw'], argspec['defaults'])
 
 @undoc
 def call_tip(oinfo, format_call=True):
     """DEPRECATED. Extract call tip data from an oinfo dict.
     """
     warnings.warn('`call_tip` function is deprecated as of IPython 6.0'
                   'and will be removed in future versions.', DeprecationWarning, stacklevel=2)
     # Get call definition
     argspec = oinfo.get('argspec')
     if argspec is None:
         call_line = None
     else:
         # Callable objects will have 'self' as their first argument, prune
         # it out if it's there for clarity (since users do *not* pass an
         # extra first argument explicitly).
         try:
             has_self = argspec['args'][0] == 'self'
         except (KeyError, IndexError):
             pass
         else:
             if has_self:
                 argspec['args'] = argspec['args'][1:]
 
         call_line = oinfo['name']+format_argspec(argspec)
 
     # Now get docstring.
     # The priority is: call docstring, constructor docstring, main one.
     doc = oinfo.get('call_docstring')
     if doc is None:
         doc = oinfo.get('init_docstring')
     if doc is None:
         doc = oinfo.get('docstring','')
 
     return call_line, doc
 
 
 def _get_wrapped(obj):
     """Get the original object if wrapped in one or more @decorators
 
     Some objects automatically construct similar objects on any unrecognised
     attribute access (e.g. unittest.mock.call). To protect against infinite loops,
     this will arbitrarily cut off after 100 levels of obj.__wrapped__
     attribute access. --TK, Jan 2016
     """
     orig_obj = obj
     i = 0
     while safe_hasattr(obj, '__wrapped__'):
         obj = obj.__wrapped__
         i += 1
         if i > 100:
             # __wrapped__ is probably a lie, so return the thing we started with
             return orig_obj
     return obj
 
 def find_file(obj):
     """Find the absolute path to the file where an object was defined.
 
     This is essentially a robust wrapper around `inspect.getabsfile`.
 
     Returns None if no file can be found.
 
     Parameters
     ----------
     obj : any Python object
 
     Returns
     -------
     fname : str
       The absolute path to the file where the object was defined.
     """
     obj = _get_wrapped(obj)
 
     fname = None
     try:
         fname = inspect.getabsfile(obj)
     except TypeError:
         # For an instance, the file that matters is where its class was
         # declared.
         if hasattr(obj, '__class__'):
             try:
                 fname = inspect.getabsfile(obj.__class__)
             except TypeError:
                 # Can happen for builtins
                 pass
     except:
         pass
     return cast_unicode(fname)
 
 
 def find_source_lines(obj):
     """Find the line number in a file where an object was defined.
 
     This is essentially a robust wrapper around `inspect.getsourcelines`.
 
     Returns None if no file can be found.
 
     Parameters
     ----------
     obj : any Python object
 
     Returns
     -------
     lineno : int
       The line number where the object definition starts.
     """
     obj = _get_wrapped(obj)
     
     try:
         try:
             lineno = inspect.getsourcelines(obj)[1]
         except TypeError:
             # For instances, try the class object like getsource() does
             if hasattr(obj, '__class__'):
                 lineno = inspect.getsourcelines(obj.__class__)[1]
             else:
                 lineno = None
     except:
         return None
 
     return lineno
 
 class Inspector(Colorable):
 
     def __init__(self, color_table=InspectColors,
                  code_color_table=PyColorize.ANSICodeColors,
                  scheme='NoColor',
                  str_detail_level=0,
                  parent=None, config=None):
         super(Inspector, self).__init__(parent=parent, config=config)
         self.color_table = color_table
         self.parser = PyColorize.Parser(out='str', parent=self, style=scheme)
         self.format = self.parser.format
         self.str_detail_level = str_detail_level
         self.set_active_scheme(scheme)
 
     def _getdef(self,obj,oname=''):
         """Return the call signature for any callable object.
 
         If any exception is generated, None is returned instead and the
         exception is suppressed."""
         try:
             hdef = oname + str(signature(obj))
             return cast_unicode(hdef)
         except:
             return None
 
     def __head(self,h):
         """Return a header string with proper colors."""
         return '%s%s%s' % (self.color_table.active_colors.header,h,
                            self.color_table.active_colors.normal)
 
     def set_active_scheme(self, scheme):
         self.color_table.set_active_scheme(scheme)
         self.parser.color_table.set_active_scheme(scheme)
 
     def noinfo(self, msg, oname):
         """Generic message when no information is found."""
         print('No %s found' % msg, end=' ')
         if oname:
             print('for %s' % oname)
         else:
             print()
 
     def pdef(self, obj, oname=''):
         """Print the call signature for any callable object.
 
         If the object is a class, print the constructor information."""
 
         if not callable(obj):
             print('Object is not callable.')
             return
 
         header = ''
 
         if inspect.isclass(obj):
             header = self.__head('Class constructor information:\n')
-        elif (not py3compat.PY3) and type(obj) is types.InstanceType:
-            obj = obj.__call__
+
 
         output = self._getdef(obj,oname)
         if output is None:
             self.noinfo('definition header',oname)
         else:
             print(header,self.format(output), end=' ')
 
     # In Python 3, all classes are new-style, so they all have __init__.
     @skip_doctest
     def pdoc(self, obj, oname='', formatter=None):
         """Print the docstring for any object.
 
         Optional:
         -formatter: a function to run the docstring through for specially
         formatted docstrings.
 
         Examples
         --------
 
         In [1]: class NoInit:
            ...:     pass
 
         In [2]: class NoDoc:
            ...:     def __init__(self):
            ...:         pass
 
         In [3]: %pdoc NoDoc
         No documentation found for NoDoc
 
         In [4]: %pdoc NoInit
         No documentation found for NoInit
 
         In [5]: obj = NoInit()
 
         In [6]: %pdoc obj
         No documentation found for obj
 
         In [5]: obj2 = NoDoc()
 
         In [6]: %pdoc obj2
         No documentation found for obj2
         """
 
         head = self.__head  # For convenience
         lines = []
         ds = getdoc(obj)
         if formatter:
             ds = formatter(ds).get('plain/text', ds)
         if ds:
             lines.append(head("Class docstring:"))
             lines.append(indent(ds))
         if inspect.isclass(obj) and hasattr(obj, '__init__'):
             init_ds = getdoc(obj.__init__)
             if init_ds is not None:
                 lines.append(head("Init docstring:"))
                 lines.append(indent(init_ds))
         elif hasattr(obj,'__call__'):
             call_ds = getdoc(obj.__call__)
             if call_ds:
                 lines.append(head("Call docstring:"))
                 lines.append(indent(call_ds))
 
         if not lines:
             self.noinfo('documentation',oname)
         else:
             page.page('\n'.join(lines))
 
     def psource(self, obj, oname=''):
         """Print the source code for an object."""
 
         # Flush the source cache because inspect can return out-of-date source
         linecache.checkcache()
         try:
             src = getsource(obj, oname=oname)
         except Exception:
             src = None
 
         if src is None:
             self.noinfo('source', oname)
         else:
             page.page(self.format(src))
 
     def pfile(self, obj, oname=''):
         """Show the whole file where an object was defined."""
 
         lineno = find_source_lines(obj)
         if lineno is None:
             self.noinfo('file', oname)
             return
 
         ofile = find_file(obj)
         # run contents of file through pager starting at line where the object
         # is defined, as long as the file isn't binary and is actually on the
         # filesystem.
         if ofile.endswith(('.so', '.dll', '.pyd')):
             print('File %r is binary, not printing.' % ofile)
         elif not os.path.isfile(ofile):
             print('File %r does not exist, not printing.' % ofile)
         else:
             # Print only text files, not extension binaries.  Note that
             # getsourcelines returns lineno with 1-offset and page() uses
             # 0-offset, so we must adjust.
             page.page(self.format(openpy.read_py_file(ofile, skip_encoding_cookie=False)), lineno - 1)
 
     def _format_fields(self, fields, title_width=0):
         """Formats a list of fields for display.
 
         Parameters
         ----------
         fields : list
           A list of 2-tuples: (field_title, field_content)
         title_width : int
           How many characters to pad titles to. Default to longest title.
         """
         out = []
         header = self.__head
         if title_width == 0:
             title_width = max(len(title) + 2 for title, _ in fields)
         for title, content in fields:
             if len(content.splitlines()) > 1:
                 title = header(title + ':') + '\n'
             else:
                 title = header((title + ':').ljust(title_width))
             out.append(cast_unicode(title) + cast_unicode(content))
         return "\n".join(out)
 
     def _mime_format(self, text, formatter=None):
         """Return a mime bundle representation of the input text.
 
         - if `formatter` is None, the returned mime bundle has
            a `text/plain` field, with the input text.
            a `text/html` field with a `<pre>` tag containing the input text.
 
         - if `formatter` is not None, it must be a callable transforming the
           input text into a mime bundle. Default values for `text/plain` and
           `text/html` representations are the ones described above.
 
         Note:
 
         Formatters returning strings are supported but this behavior is deprecated.
 
         """
         text = cast_unicode(text)
         defaults = {
             'text/plain': text,
             'text/html': '<pre>' + text + '</pre>'
         }
 
         if formatter is None:
             return defaults
         else:
             formatted = formatter(text)
 
             if not isinstance(formatted, dict):
                 # Handle the deprecated behavior of a formatter returning
                 # a string instead of a mime bundle.
                 return {
                     'text/plain': formatted,
                     'text/html': '<pre>' + formatted + '</pre>'
                 }
 
             else:
                 return dict(defaults, **formatted)
 
 
     def format_mime(self, bundle):
 
         text_plain = bundle['text/plain']
 
         text = ''
         heads, bodies = list(zip(*text_plain))
         _len = max(len(h) for h in heads)
 
         for head, body in zip(heads, bodies):
             body = body.strip('\n')
             delim = '\n' if '\n' in body else ' '
             text += self.__head(head+':') + (_len - len(head))*' ' +delim + body +'\n'
 
         bundle['text/plain'] = text
         return bundle
 
     def _get_info(self, obj, oname='', formatter=None, info=None, detail_level=0):
         """Retrieve an info dict and format it."""
 
         info = self._info(obj, oname=oname, info=info, detail_level=detail_level)
 
         _mime = {
             'text/plain': [],
             'text/html': '',
         }
 
         def append_field(bundle, title, key, formatter=None):
             field = info[key]
             if field is not None:
                 formatted_field = self._mime_format(field, formatter)
                 bundle['text/plain'].append((title, formatted_field['text/plain']))
                 bundle['text/html'] += '<h1>' + title + '</h1>\n' + formatted_field['text/html'] + '\n'
 
         def code_formatter(text):
             return {
                 'text/plain': self.format(text),
                 'text/html': pylight(text)
             }
 
         if info['isalias']:
             append_field(_mime, 'Repr', 'string_form')
 
         elif info['ismagic']:
             if detail_level > 0:
                 append_field(_mime, 'Source', 'source', code_formatter)
             else:
                 append_field(_mime, 'Docstring', 'docstring', formatter)
             append_field(_mime, 'File', 'file')
 
         elif info['isclass'] or is_simple_callable(obj):
             # Functions, methods, classes
             append_field(_mime, 'Signature', 'definition', code_formatter)
             append_field(_mime, 'Init signature', 'init_definition', code_formatter)
             if detail_level > 0 and info['source']:
                 append_field(_mime, 'Source', 'source', code_formatter)
             else:
                 append_field(_mime, 'Docstring', 'docstring', formatter)
                 append_field(_mime, 'Init docstring', 'init_docstring', formatter)
 
             append_field(_mime, 'File', 'file')
             append_field(_mime, 'Type', 'type_name')
 
         else:
             # General Python objects
             append_field(_mime, 'Signature', 'definition', code_formatter)
             append_field(_mime, 'Call signature', 'call_def', code_formatter)
-            
             append_field(_mime, 'Type', 'type_name')
-
-            # Base class for old-style instances
-            if (not py3compat.PY3) and isinstance(obj, types.InstanceType) and info['base_class']:
-                append_field(_mime, 'Base Class', 'base_class')
-
             append_field(_mime, 'String form', 'string_form')
 
             # Namespace
             if info['namespace'] != 'Interactive':
                 append_field(_mime, 'Namespace', 'namespace')
 
             append_field(_mime, 'Length', 'length')
             append_field(_mime, 'File', 'file')
             
             # Source or docstring, depending on detail level and whether
             # source found.
             if detail_level > 0:
                 append_field(_mime, 'Source', 'source', code_formatter)
             else:
                 append_field(_mime, 'Docstring', 'docstring', formatter)
 
             append_field(_mime, 'Class docstring', 'class_docstring', formatter)
             append_field(_mime, 'Init docstring', 'init_docstring', formatter)
             append_field(_mime, 'Call docstring', 'call_docstring', formatter)
             
 
         return self.format_mime(_mime)
 
     def pinfo(self, obj, oname='', formatter=None, info=None, detail_level=0, enable_html_pager=True):
         """Show detailed information about an object.
 
         Optional arguments:
 
         - oname: name of the variable pointing to the object.
 
         - formatter: callable (optional)
               A special formatter for docstrings.
 
               The formatter is a callable that takes a string as an input
               and returns either a formatted string or a mime type bundle
               in the form of a dictionnary.
 
               Although the support of custom formatter returning a string
               instead of a mime type bundle is deprecated.
 
         - info: a structure with some information fields which may have been
           precomputed already.
 
         - detail_level: if set to 1, more information is given.
         """
         info = self._get_info(obj, oname, formatter, info, detail_level)
         if not enable_html_pager:
             del info['text/html']
         page.page(info)
 
     def info(self, obj, oname='', formatter=None, info=None, detail_level=0):
         """DEPRECATED. Compute a dict with detailed information about an object.
         """
         if formatter is not None:
             warnings.warn('The `formatter` keyword argument to `Inspector.info`'
                      'is deprecated as of IPython 5.0 and will have no effects.',
                       DeprecationWarning, stacklevel=2)
         return self._info(obj, oname=oname, info=info, detail_level=detail_level)
 
     def _info(self, obj, oname='', info=None, detail_level=0):
         """Compute a dict with detailed information about an object.
 
         Optional arguments:
 
         - oname: name of the variable pointing to the object.
 
         - info: a structure with some information fields which may have been
           precomputed already.
 
         - detail_level: if set to 1, more information is given.
         """
 
         obj_type = type(obj)
 
         if info is None:
             ismagic = 0
             isalias = 0
             ospace = ''
         else:
             ismagic = info.ismagic
             isalias = info.isalias
             ospace = info.namespace
 
         # Get docstring, special-casing aliases:
         if isalias:
             if not callable(obj):
                 try:
                     ds = "Alias to the system command:\n  %s" % obj[1]
                 except:
                     ds = "Alias: " + str(obj)
             else:
                 ds = "Alias to " + str(obj)
                 if obj.__doc__:
                     ds += "\nDocstring:\n" + obj.__doc__
         else:
             ds = getdoc(obj)
             if ds is None:
                 ds = '<no docstring>'
 
         # store output in a dict, we initialize it here and fill it as we go
         out = dict(name=oname, found=True, isalias=isalias, ismagic=ismagic)
 
         string_max = 200 # max size of strings to show (snipped if longer)
         shalf = int((string_max - 5) / 2)
 
         if ismagic:
             obj_type_name = 'Magic function'
         elif isalias:
             obj_type_name = 'System alias'
         else:
             obj_type_name = obj_type.__name__
         out['type_name'] = obj_type_name
 
         try:
             bclass = obj.__class__
             out['base_class'] = str(bclass)
         except: pass
 
         # String form, but snip if too long in ? form (full in ??)
         if detail_level >= self.str_detail_level:
             try:
                 ostr = str(obj)
                 str_head = 'string_form'
                 if not detail_level and len(ostr)>string_max:
                     ostr = ostr[:shalf] + ' <...> ' + ostr[-shalf:]
                     ostr = ("\n" + " " * len(str_head.expandtabs())).\
                             join(q.strip() for q in ostr.split("\n"))
                 out[str_head] = ostr
             except:
                 pass
 
         if ospace:
             out['namespace'] = ospace
 
         # Length (for strings and lists)
         try:
             out['length'] = str(len(obj))
         except: pass
 
         # Filename where object was defined
         binary_file = False
         fname = find_file(obj)
         if fname is None:
             # if anything goes wrong, we don't want to show source, so it's as
             # if the file was binary
             binary_file = True
         else:
             if fname.endswith(('.so', '.dll', '.pyd')):
                 binary_file = True
             elif fname.endswith('<string>'):
                 fname = 'Dynamically generated function. No source code available.'
             out['file'] = compress_user(fname)
 
         # Original source code for a callable, class or property.
         if detail_level:
             # Flush the source cache because inspect can return out-of-date
             # source
             linecache.checkcache()
             try:
                 if isinstance(obj, property) or not binary_file:
                     src = getsource(obj, oname)
                     if src is not None:
                         src = src.rstrip()
                     out['source'] = src
 
             except Exception:
                 pass
 
         # Add docstring only if no source is to be shown (avoid repetitions).
         if ds and out.get('source', None) is None:
             out['docstring'] = ds
 
         # Constructor docstring for classes
         if inspect.isclass(obj):
             out['isclass'] = True
 
             # get the init signature:
             try:
                 init_def = self._getdef(obj, oname)
             except AttributeError:
                 init_def = None
 
             # get the __init__ docstring
             try:
                 obj_init = obj.__init__
             except AttributeError:
                 init_ds = None
             else:
                 if init_def is None:
                     # Get signature from init if top-level sig failed.
                     # Can happen for built-in types (list, etc.).
                     try:
                         init_def = self._getdef(obj_init, oname)
                     except AttributeError:
                         pass
                 init_ds = getdoc(obj_init)
                 # Skip Python's auto-generated docstrings
                 if init_ds == _object_init_docstring:
                     init_ds = None
 
             if init_def:
                 out['init_definition'] = init_def
 
             if init_ds:
                 out['init_docstring'] = init_ds
 
         # and class docstring for instances:
         else:
             # reconstruct the function definition and print it:
             defln = self._getdef(obj, oname)
             if defln:
                 out['definition'] = defln
 
             # First, check whether the instance docstring is identical to the
             # class one, and print it separately if they don't coincide.  In
             # most cases they will, but it's nice to print all the info for
             # objects which use instance-customized docstrings.
             if ds:
                 try:
                     cls = getattr(obj,'__class__')
                 except:
                     class_ds = None
                 else:
                     class_ds = getdoc(cls)
                 # Skip Python's auto-generated docstrings
                 if class_ds in _builtin_type_docstrings:
                     class_ds = None
                 if class_ds and ds != class_ds:
                     out['class_docstring'] = class_ds
 
             # Next, try to show constructor docstrings
             try:
                 init_ds = getdoc(obj.__init__)
                 # Skip Python's auto-generated docstrings
                 if init_ds == _object_init_docstring:
                     init_ds = None
             except AttributeError:
                 init_ds = None
             if init_ds:
                 out['init_docstring'] = init_ds
 
             # Call form docstring for callable instances
             if safe_hasattr(obj, '__call__') and not is_simple_callable(obj):
                 call_def = self._getdef(obj.__call__, oname)
                 if call_def and (call_def != out.get('definition')):
                     # it may never be the case that call def and definition differ,
                     # but don't include the same signature twice
                     out['call_def'] = call_def
                 call_ds = getdoc(obj.__call__)
                 # Skip Python's auto-generated docstrings
                 if call_ds == _func_call_docstring:
                     call_ds = None
                 if call_ds:
                     out['call_docstring'] = call_ds
 
         # Compute the object's argspec as a callable.  The key is to decide
         # whether to pull it from the object itself, from its __init__ or
         # from its __call__ method.
 
         if inspect.isclass(obj):
             # Old-style classes need not have an __init__
             callable_obj = getattr(obj, "__init__", None)
         elif callable(obj):
             callable_obj = obj
         else:
             callable_obj = None
 
         if callable_obj is not None:
             try:
                 argspec = getargspec(callable_obj)
             except (TypeError, AttributeError):
                 # For extensions/builtins we can't retrieve the argspec
                 pass
             else:
                 # named tuples' _asdict() method returns an OrderedDict, but we
                 # we want a normal
                 out['argspec'] = argspec_dict = dict(argspec._asdict())
                 # We called this varkw before argspec became a named tuple.
                 # With getfullargspec it's also called varkw.
                 if 'varkw' not in argspec_dict:
                     argspec_dict['varkw'] = argspec_dict.pop('keywords')
 
         return object_info(**out)
 
     def psearch(self,pattern,ns_table,ns_search=[],
                 ignore_case=False,show_all=False):
         """Search namespaces with wildcards for objects.
 
         Arguments:
 
         - pattern: string containing shell-like wildcards to use in namespace
           searches and optionally a type specification to narrow the search to
           objects of that type.
 
         - ns_table: dict of name->namespaces for search.
 
         Optional arguments:
 
           - ns_search: list of namespace names to include in search.
 
           - ignore_case(False): make the search case-insensitive.
 
           - show_all(False): show all names, including those starting with
             underscores.
         """
         #print 'ps pattern:<%r>' % pattern # dbg
 
         # defaults
         type_pattern = 'all'
         filter = ''
 
         cmds = pattern.split()
         len_cmds  =  len(cmds)
         if len_cmds == 1:
             # Only filter pattern given
             filter = cmds[0]
         elif len_cmds == 2:
             # Both filter and type specified
             filter,type_pattern = cmds
         else:
             raise ValueError('invalid argument string for psearch: <%s>' %
                              pattern)
 
         # filter search namespaces
         for name in ns_search:
             if name not in ns_table:
                 raise ValueError('invalid namespace <%s>. Valid names: %s' %
                                  (name,ns_table.keys()))
 
         #print 'type_pattern:',type_pattern # dbg
         search_result, namespaces_seen = set(), set()
         for ns_name in ns_search:
             ns = ns_table[ns_name]
             # Normally, locals and globals are the same, so we just check one.
             if id(ns) in namespaces_seen:
                 continue
             namespaces_seen.add(id(ns))
             tmp_res = list_namespace(ns, type_pattern, filter,
                                     ignore_case=ignore_case, show_all=show_all)
             search_result.update(tmp_res)
 
         page.page('\n'.join(sorted(search_result)))

IPython/core/tests/test_compilerop.py

@@ -1,74 +1,74 b''
 # coding: utf-8
 """Tests for the compilerop module.
 """
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2010-2011 The IPython Development Team.
 #
 #  Distributed under the terms of the BSD License.
 #
 #  The full license is in the file COPYING.txt, distributed with this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 # Stdlib imports
 import linecache
 import sys
 
 # Third-party imports
 import nose.tools as nt
 
 # Our own imports
 from IPython.core import compilerop
 from IPython.utils import py3compat
 
 #-----------------------------------------------------------------------------
 # Test functions
 #-----------------------------------------------------------------------------
 
 def test_code_name():
     code = 'x=1'
     name = compilerop.code_name(code)
     nt.assert_true(name.startswith('<ipython-input-0'))
 
 
 def test_code_name2():
     code = 'x=1'
     name = compilerop.code_name(code, 9)
     nt.assert_true(name.startswith('<ipython-input-9'))
 
 
 def test_cache():
     """Test the compiler correctly compiles and caches inputs
     """
     cp = compilerop.CachingCompiler()
     ncache = len(linecache.cache)
     cp.cache('x=1')
     nt.assert_true(len(linecache.cache) > ncache)
 
 def setUp():
     # Check we're in a proper Python 2 environment (some imports, such
     # as GTK, can change the default encoding, which can hide bugs.)
-    nt.assert_equal(sys.getdefaultencoding(), "utf-8" if py3compat.PY3 else "ascii")
+    nt.assert_equal(sys.getdefaultencoding(), "utf-8")
 
 def test_cache_unicode():
     cp = compilerop.CachingCompiler()
     ncache = len(linecache.cache)
     cp.cache(u"t = 'žćčšđ'")
     nt.assert_true(len(linecache.cache) > ncache)
 
 def test_compiler_check_cache():
     """Test the compiler properly manages the cache.
     """
     # Rather simple-minded tests that just exercise the API
     cp = compilerop.CachingCompiler()
     cp.cache('x=1', 99)
     # Ensure now that after clearing the cache, our entries survive
     linecache.checkcache()
     for k in linecache.cache:
         if k.startswith('<ipython-input-99'):
             break
     else:
         raise AssertionError('Entry for input-99 missing from linecache')

IPython/core/tests/test_history.py

@@ -1,211 +1,211 b''
 # coding: utf-8
 """Tests for the IPython tab-completion machinery.
 """
 #-----------------------------------------------------------------------------
 # Module imports
 #-----------------------------------------------------------------------------
 
 # stdlib
 import io
 import os
 import sys
 import tempfile
 from datetime import datetime
 
 # third party
 import nose.tools as nt
 
 # our own packages
 from traitlets.config.loader import Config
 from IPython.utils.tempdir import TemporaryDirectory
 from IPython.core.history import HistoryManager, extract_hist_ranges
 from IPython.utils import py3compat
 
 def setUp():
-    nt.assert_equal(sys.getdefaultencoding(), "utf-8" if py3compat.PY3 else "ascii")
+    nt.assert_equal(sys.getdefaultencoding(), "utf-8")
 
 def test_history():
     ip = get_ipython()
     with TemporaryDirectory() as tmpdir:
         hist_manager_ori = ip.history_manager
         hist_file = os.path.join(tmpdir, 'history.sqlite')
         try:
             ip.history_manager = HistoryManager(shell=ip, hist_file=hist_file)
             hist = [u'a=1', u'def f():\n    test = 1\n    return test', u"b='€Æ¾÷ß'"]
             for i, h in enumerate(hist, start=1):
                 ip.history_manager.store_inputs(i, h)
 
             ip.history_manager.db_log_output = True
             # Doesn't match the input, but we'll just check it's stored.
             ip.history_manager.output_hist_reprs[3] = "spam"
             ip.history_manager.store_output(3)
 
             nt.assert_equal(ip.history_manager.input_hist_raw, [''] + hist)
             
             # Detailed tests for _get_range_session
             grs = ip.history_manager._get_range_session
             nt.assert_equal(list(grs(start=2,stop=-1)), list(zip([0], [2], hist[1:-1])))
             nt.assert_equal(list(grs(start=-2)), list(zip([0,0], [2,3], hist[-2:])))
             nt.assert_equal(list(grs(output=True)), list(zip([0,0,0], [1,2,3], zip(hist, [None,None,'spam']))))
 
             # Check whether specifying a range beyond the end of the current
             # session results in an error (gh-804)
             ip.magic('%hist 2-500')
             
             # Check that we can write non-ascii characters to a file
             ip.magic("%%hist -f %s" % os.path.join(tmpdir, "test1"))
             ip.magic("%%hist -pf %s" % os.path.join(tmpdir, "test2"))
             ip.magic("%%hist -nf %s" % os.path.join(tmpdir, "test3"))
             ip.magic("%%save %s 1-10" % os.path.join(tmpdir, "test4"))
 
             # New session
             ip.history_manager.reset()
             newcmds = [u"z=5",
                        u"class X(object):\n    pass",
                        u"k='p'",
                        u"z=5"]
             for i, cmd in enumerate(newcmds, start=1):
                 ip.history_manager.store_inputs(i, cmd)
             gothist = ip.history_manager.get_range(start=1, stop=4)
             nt.assert_equal(list(gothist), list(zip([0,0,0],[1,2,3], newcmds)))
             # Previous session:
             gothist = ip.history_manager.get_range(-1, 1, 4)
             nt.assert_equal(list(gothist), list(zip([1,1,1],[1,2,3], hist)))
 
             newhist = [(2, i, c) for (i, c) in enumerate(newcmds, 1)]
 
             # Check get_hist_tail
             gothist = ip.history_manager.get_tail(5, output=True,
                                                     include_latest=True)
             expected = [(1, 3, (hist[-1], "spam"))] \
                 + [(s, n, (c, None)) for (s, n, c) in newhist]
             nt.assert_equal(list(gothist), expected)
 
             gothist = ip.history_manager.get_tail(2)
             expected = newhist[-3:-1]
             nt.assert_equal(list(gothist), expected)
 
             # Check get_hist_search
             gothist = ip.history_manager.search("*test*")
             nt.assert_equal(list(gothist), [(1,2,hist[1])] )
 
             gothist = ip.history_manager.search("*=*")
             nt.assert_equal(list(gothist),
                             [(1, 1, hist[0]),
                              (1, 2, hist[1]),
                              (1, 3, hist[2]),
                              newhist[0],
                              newhist[2],
                              newhist[3]])
 
             gothist = ip.history_manager.search("*=*", n=4)
             nt.assert_equal(list(gothist),
                             [(1, 3, hist[2]),
                              newhist[0],
                              newhist[2],
                              newhist[3]])
 
             gothist = ip.history_manager.search("*=*", unique=True)
             nt.assert_equal(list(gothist),
                             [(1, 1, hist[0]),
                              (1, 2, hist[1]),
                              (1, 3, hist[2]),
                              newhist[2],
                              newhist[3]])
 
             gothist = ip.history_manager.search("*=*", unique=True, n=3)
             nt.assert_equal(list(gothist),
                             [(1, 3, hist[2]),
                              newhist[2],
                              newhist[3]])
 
             gothist = ip.history_manager.search("b*", output=True)
             nt.assert_equal(list(gothist), [(1,3,(hist[2],"spam"))] )
 
             # Cross testing: check that magic %save can get previous session.
             testfilename = os.path.realpath(os.path.join(tmpdir, "test.py"))
             ip.magic("save " + testfilename + " ~1/1-3")
             with io.open(testfilename, encoding='utf-8') as testfile:
                 nt.assert_equal(testfile.read(),
                                         u"# coding: utf-8\n" + u"\n".join(hist)+u"\n")
 
             # Duplicate line numbers - check that it doesn't crash, and
             # gets a new session
             ip.history_manager.store_inputs(1, "rogue")
             ip.history_manager.writeout_cache()
             nt.assert_equal(ip.history_manager.session_number, 3)
         finally:
             # Ensure saving thread is shut down before we try to clean up the files
             ip.history_manager.save_thread.stop()
             # Forcibly close database rather than relying on garbage collection
             ip.history_manager.db.close()
             # Restore history manager
             ip.history_manager = hist_manager_ori
 
 
 def test_extract_hist_ranges():
     instr = "1 2/3 ~4/5-6 ~4/7-~4/9 ~9/2-~7/5 ~10/"
     expected = [(0, 1, 2),  # 0 == current session
                 (2, 3, 4),
                 (-4, 5, 7),
                 (-4, 7, 10),
                 (-9, 2, None),  # None == to end
                 (-8, 1, None),
                 (-7, 1, 6),
                 (-10, 1, None)]
     actual = list(extract_hist_ranges(instr))
     nt.assert_equal(actual, expected)
 
 def test_magic_rerun():
     """Simple test for %rerun (no args -> rerun last line)"""
     ip = get_ipython()
     ip.run_cell("a = 10", store_history=True)
     ip.run_cell("a += 1", store_history=True)
     nt.assert_equal(ip.user_ns["a"], 11)
     ip.run_cell("%rerun", store_history=True)
     nt.assert_equal(ip.user_ns["a"], 12)
 
 def test_timestamp_type():
     ip = get_ipython()
     info = ip.history_manager.get_session_info()
     nt.assert_true(isinstance(info[1], datetime))
 
 def test_hist_file_config():
     cfg = Config()
     tfile = tempfile.NamedTemporaryFile(delete=False)
     cfg.HistoryManager.hist_file = tfile.name
     try:
         hm = HistoryManager(shell=get_ipython(), config=cfg)
         nt.assert_equal(hm.hist_file, cfg.HistoryManager.hist_file)
     finally:
         try:
             os.remove(tfile.name)
         except OSError:
             # same catch as in testing.tools.TempFileMixin
             # On Windows, even though we close the file, we still can't
             # delete it.  I have no clue why
             pass
 
 def test_histmanager_disabled():
     """Ensure that disabling the history manager doesn't create a database."""
     cfg = Config()
     cfg.HistoryAccessor.enabled = False
 
     ip = get_ipython()
     with TemporaryDirectory() as tmpdir:
         hist_manager_ori = ip.history_manager
         hist_file = os.path.join(tmpdir, 'history.sqlite')
         cfg.HistoryManager.hist_file = hist_file
         try:
             ip.history_manager = HistoryManager(shell=ip, config=cfg)
             hist = [u'a=1', u'def f():\n    test = 1\n    return test', u"b='€Æ¾÷ß'"]
             for i, h in enumerate(hist, start=1):
                 ip.history_manager.store_inputs(i, h)
             nt.assert_equal(ip.history_manager.input_hist_raw, [''] + hist)
             ip.history_manager.reset()
             ip.history_manager.end_session()
         finally:
             ip.history_manager = hist_manager_ori
 
     # hist_file should not be created
     nt.assert_false(os.path.exists(hist_file))

IPython/core/tests/test_interactiveshell.py

@@ -1,904 +1,899 b''
 # -*- coding: utf-8 -*-
 """Tests for the key interactiveshell module.
 
 Historically the main classes in interactiveshell have been under-tested.  This
 module should grow as many single-method tests as possible to trap many of the
 recurring bugs we seem to encounter with high-level interaction.
 """
 
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
 import ast
 import os
 import signal
 import shutil
 import sys
 import tempfile
 import unittest
 from unittest import mock
+from io import StringIO
 
 from os.path import join
 
 import nose.tools as nt
 
 from IPython.core.error import InputRejected
 from IPython.core.inputtransformer import InputTransformer
 from IPython.testing.decorators import (
     skipif, skip_win32, onlyif_unicode_paths, onlyif_cmds_exist,
 )
 from IPython.testing import tools as tt
 from IPython.utils.process import find_cmd
 from IPython.utils import py3compat
-from IPython.utils.py3compat import PY3
-
-if PY3:
-    from io import StringIO
-else:
-    from StringIO import StringIO
 
 #-----------------------------------------------------------------------------
 # Globals
 #-----------------------------------------------------------------------------
 # This is used by every single test, no point repeating it ad nauseam
 ip = get_ipython()
 
 #-----------------------------------------------------------------------------
 # Tests
 #-----------------------------------------------------------------------------
 
 class DerivedInterrupt(KeyboardInterrupt):
     pass
 
 class InteractiveShellTestCase(unittest.TestCase):
     def test_naked_string_cells(self):
         """Test that cells with only naked strings are fully executed"""
         # First, single-line inputs
         ip.run_cell('"a"\n')
         self.assertEqual(ip.user_ns['_'], 'a')
         # And also multi-line cells
         ip.run_cell('"""a\nb"""\n')
         self.assertEqual(ip.user_ns['_'], 'a\nb')
 
     def test_run_empty_cell(self):
         """Just make sure we don't get a horrible error with a blank
         cell of input. Yes, I did overlook that."""
         old_xc = ip.execution_count
         res = ip.run_cell('')
         self.assertEqual(ip.execution_count, old_xc)
         self.assertEqual(res.execution_count, None)
 
     def test_run_cell_multiline(self):
         """Multi-block, multi-line cells must execute correctly.
         """
         src = '\n'.join(["x=1",
                          "y=2",
                          "if 1:",
                          "    x += 1",
                          "    y += 1",])
         res = ip.run_cell(src)
         self.assertEqual(ip.user_ns['x'], 2)
         self.assertEqual(ip.user_ns['y'], 3)
         self.assertEqual(res.success, True)
         self.assertEqual(res.result, None)
 
     def test_multiline_string_cells(self):
         "Code sprinkled with multiline strings should execute (GH-306)"
         ip.run_cell('tmp=0')
         self.assertEqual(ip.user_ns['tmp'], 0)
         res = ip.run_cell('tmp=1;"""a\nb"""\n')
         self.assertEqual(ip.user_ns['tmp'], 1)
         self.assertEqual(res.success, True)
         self.assertEqual(res.result, "a\nb")
 
     def test_dont_cache_with_semicolon(self):
         "Ending a line with semicolon should not cache the returned object (GH-307)"
         oldlen = len(ip.user_ns['Out'])
         for cell in ['1;', '1;1;']:
             res = ip.run_cell(cell, store_history=True)
             newlen = len(ip.user_ns['Out'])
             self.assertEqual(oldlen, newlen)
             self.assertIsNone(res.result)
         i = 0
         #also test the default caching behavior
         for cell in ['1', '1;1']:
             ip.run_cell(cell, store_history=True)
             newlen = len(ip.user_ns['Out'])
             i += 1
             self.assertEqual(oldlen+i, newlen)
 
     def test_syntax_error(self):
         res = ip.run_cell("raise = 3")
         self.assertIsInstance(res.error_before_exec, SyntaxError)
 
     def test_In_variable(self):
         "Verify that In variable grows with user input (GH-284)"
         oldlen = len(ip.user_ns['In'])
         ip.run_cell('1;', store_history=True)
         newlen = len(ip.user_ns['In'])
         self.assertEqual(oldlen+1, newlen)
         self.assertEqual(ip.user_ns['In'][-1],'1;')
         
     def test_magic_names_in_string(self):
         ip.run_cell('a = """\n%exit\n"""')
         self.assertEqual(ip.user_ns['a'], '\n%exit\n')
     
     def test_trailing_newline(self):
         """test that running !(command) does not raise a SyntaxError"""
         ip.run_cell('!(true)\n', False)
         ip.run_cell('!(true)\n\n\n', False)
     
     def test_gh_597(self):
         """Pretty-printing lists of objects with non-ascii reprs may cause
         problems."""
         class Spam(object):
           def __repr__(self):
             return "\xe9"*50
         import IPython.core.formatters
         f = IPython.core.formatters.PlainTextFormatter()
         f([Spam(),Spam()])
     
 
     def test_future_flags(self):
         """Check that future flags are used for parsing code (gh-777)"""
         ip.run_cell('from __future__ import barry_as_FLUFL')
         try:
             ip.run_cell('prfunc_return_val = 1 <> 2')
             assert 'prfunc_return_val' in ip.user_ns
         finally:
             # Reset compiler flags so we don't mess up other tests.
             ip.compile.reset_compiler_flags()
 
     def test_can_pickle(self):
         "Can we pickle objects defined interactively (GH-29)"
         ip = get_ipython()
         ip.reset()
         ip.run_cell(("class Mylist(list):\n"
                      "    def __init__(self,x=[]):\n"
                      "        list.__init__(self,x)"))
         ip.run_cell("w=Mylist([1,2,3])")
         
         from pickle import dumps
         
         # We need to swap in our main module - this is only necessary
         # inside the test framework, because IPython puts the interactive module
         # in place (but the test framework undoes this).
         _main = sys.modules['__main__']
         sys.modules['__main__'] = ip.user_module
         try:
             res = dumps(ip.user_ns["w"])
         finally:
             sys.modules['__main__'] = _main
         self.assertTrue(isinstance(res, bytes))
         
     def test_global_ns(self):
         "Code in functions must be able to access variables outside them."
         ip = get_ipython()
         ip.run_cell("a = 10")
         ip.run_cell(("def f(x):\n"
                      "    return x + a"))
         ip.run_cell("b = f(12)")
         self.assertEqual(ip.user_ns["b"], 22)
 
     def test_bad_custom_tb(self):
         """Check that InteractiveShell is protected from bad custom exception handlers"""
         ip.set_custom_exc((IOError,), lambda etype,value,tb: 1/0)
         self.assertEqual(ip.custom_exceptions, (IOError,))
         with tt.AssertPrints("Custom TB Handler failed", channel='stderr'):
             ip.run_cell(u'raise IOError("foo")')
         self.assertEqual(ip.custom_exceptions, ())
 
     def test_bad_custom_tb_return(self):
         """Check that InteractiveShell is protected from bad return types in custom exception handlers"""
         ip.set_custom_exc((NameError,),lambda etype,value,tb, tb_offset=None: 1)
         self.assertEqual(ip.custom_exceptions, (NameError,))
         with tt.AssertPrints("Custom TB Handler failed", channel='stderr'):
             ip.run_cell(u'a=abracadabra')
         self.assertEqual(ip.custom_exceptions, ())
 
     def test_drop_by_id(self):
         myvars = {"a":object(), "b":object(), "c": object()}
         ip.push(myvars, interactive=False)
         for name in myvars:
             assert name in ip.user_ns, name
             assert name in ip.user_ns_hidden, name
         ip.user_ns['b'] = 12
         ip.drop_by_id(myvars)
         for name in ["a", "c"]:
             assert name not in ip.user_ns, name
             assert name not in ip.user_ns_hidden, name
         assert ip.user_ns['b'] == 12
         ip.reset()
 
     def test_var_expand(self):
         ip.user_ns['f'] = u'Ca\xf1o'
         self.assertEqual(ip.var_expand(u'echo $f'), u'echo Ca\xf1o')
         self.assertEqual(ip.var_expand(u'echo {f}'), u'echo Ca\xf1o')
         self.assertEqual(ip.var_expand(u'echo {f[:-1]}'), u'echo Ca\xf1')
         self.assertEqual(ip.var_expand(u'echo {1*2}'), u'echo 2')
 
         ip.user_ns['f'] = b'Ca\xc3\xb1o'
         # This should not raise any exception:
         ip.var_expand(u'echo $f')
     
     def test_var_expand_local(self):
         """Test local variable expansion in !system and %magic calls"""
         # !system
         ip.run_cell('def test():\n'
                     '    lvar = "ttt"\n'
                     '    ret = !echo {lvar}\n'
                     '    return ret[0]\n')
         res = ip.user_ns['test']()
         nt.assert_in('ttt', res)
         
         # %magic
         ip.run_cell('def makemacro():\n'
                     '    macroname = "macro_var_expand_locals"\n'
                     '    %macro {macroname} codestr\n')
         ip.user_ns['codestr'] = "str(12)"
         ip.run_cell('makemacro()')
         nt.assert_in('macro_var_expand_locals', ip.user_ns)
     
     def test_var_expand_self(self):
         """Test variable expansion with the name 'self', which was failing.
         
         See https://github.com/ipython/ipython/issues/1878#issuecomment-7698218
         """
         ip.run_cell('class cTest:\n'
                     '  classvar="see me"\n'
                     '  def test(self):\n'
                     '    res = !echo Variable: {self.classvar}\n'
                     '    return res[0]\n')
         nt.assert_in('see me', ip.user_ns['cTest']().test())
     
     def test_bad_var_expand(self):
         """var_expand on invalid formats shouldn't raise"""
         # SyntaxError
         self.assertEqual(ip.var_expand(u"{'a':5}"), u"{'a':5}")
         # NameError
         self.assertEqual(ip.var_expand(u"{asdf}"), u"{asdf}")
         # ZeroDivisionError
         self.assertEqual(ip.var_expand(u"{1/0}"), u"{1/0}")
     
     def test_silent_postexec(self):
         """run_cell(silent=True) doesn't invoke pre/post_run_cell callbacks"""
         pre_explicit = mock.Mock()
         pre_always = mock.Mock()
         post_explicit = mock.Mock()
         post_always = mock.Mock()
         
         ip.events.register('pre_run_cell', pre_explicit)
         ip.events.register('pre_execute', pre_always)
         ip.events.register('post_run_cell', post_explicit)
         ip.events.register('post_execute', post_always)
         
         try:
             ip.run_cell("1", silent=True)
             assert pre_always.called
             assert not pre_explicit.called
             assert post_always.called
             assert not post_explicit.called
             # double-check that non-silent exec did what we expected
             # silent to avoid
             ip.run_cell("1")
             assert pre_explicit.called
             assert post_explicit.called
         finally:
             # remove post-exec
             ip.events.unregister('pre_run_cell', pre_explicit)
             ip.events.unregister('pre_execute', pre_always)
             ip.events.unregister('post_run_cell', post_explicit)
             ip.events.unregister('post_execute', post_always)
     
     def test_silent_noadvance(self):
         """run_cell(silent=True) doesn't advance execution_count"""
         ec = ip.execution_count
         # silent should force store_history=False
         ip.run_cell("1", store_history=True, silent=True)
         
         self.assertEqual(ec, ip.execution_count)
         # double-check that non-silent exec did what we expected
         # silent to avoid
         ip.run_cell("1", store_history=True)
         self.assertEqual(ec+1, ip.execution_count)
     
     def test_silent_nodisplayhook(self):
         """run_cell(silent=True) doesn't trigger displayhook"""
         d = dict(called=False)
         
         trap = ip.display_trap
         save_hook = trap.hook
         
         def failing_hook(*args, **kwargs):
             d['called'] = True
         
         try:
             trap.hook = failing_hook
             res = ip.run_cell("1", silent=True)
             self.assertFalse(d['called'])
             self.assertIsNone(res.result)
             # double-check that non-silent exec did what we expected
             # silent to avoid
             ip.run_cell("1")
             self.assertTrue(d['called'])
         finally:
             trap.hook = save_hook
 
     def test_ofind_line_magic(self):
         from IPython.core.magic import register_line_magic
         
         @register_line_magic
         def lmagic(line):
             "A line magic"
 
         # Get info on line magic
         lfind = ip._ofind('lmagic')
         info = dict(found=True, isalias=False, ismagic=True,
                     namespace = 'IPython internal', obj= lmagic.__wrapped__,
                     parent = None)
         nt.assert_equal(lfind, info)
         
     def test_ofind_cell_magic(self):
         from IPython.core.magic import register_cell_magic
         
         @register_cell_magic
         def cmagic(line, cell):
             "A cell magic"
 
         # Get info on cell magic
         find = ip._ofind('cmagic')
         info = dict(found=True, isalias=False, ismagic=True,
                     namespace = 'IPython internal', obj= cmagic.__wrapped__,
                     parent = None)
         nt.assert_equal(find, info)
 
     def test_ofind_property_with_error(self):
         class A(object):
             @property
             def foo(self):
                 raise NotImplementedError()
         a = A()
 
         found = ip._ofind('a.foo', [('locals', locals())])
         info = dict(found=True, isalias=False, ismagic=False,
                     namespace='locals', obj=A.foo, parent=a)
         nt.assert_equal(found, info)
 
     def test_ofind_multiple_attribute_lookups(self):
         class A(object):
             @property
             def foo(self):
                 raise NotImplementedError()
 
         a = A()
         a.a = A()
         a.a.a = A()
 
         found = ip._ofind('a.a.a.foo', [('locals', locals())])
         info = dict(found=True, isalias=False, ismagic=False,
                     namespace='locals', obj=A.foo, parent=a.a.a)
         nt.assert_equal(found, info)
 
     def test_ofind_slotted_attributes(self):
         class A(object):
             __slots__ = ['foo']
             def __init__(self):
                 self.foo = 'bar'
 
         a = A()
         found = ip._ofind('a.foo', [('locals', locals())])
         info = dict(found=True, isalias=False, ismagic=False,
                     namespace='locals', obj=a.foo, parent=a)
         nt.assert_equal(found, info)
 
         found = ip._ofind('a.bar', [('locals', locals())])
         info = dict(found=False, isalias=False, ismagic=False,
                     namespace=None, obj=None, parent=a)
         nt.assert_equal(found, info)
 
     def test_ofind_prefers_property_to_instance_level_attribute(self):
         class A(object):
             @property
             def foo(self):
                 return 'bar'
         a = A()
         a.__dict__['foo'] = 'baz'
         nt.assert_equal(a.foo, 'bar')
         found = ip._ofind('a.foo', [('locals', locals())])
         nt.assert_is(found['obj'], A.foo)
 
     def test_custom_syntaxerror_exception(self):
         called = []
         def my_handler(shell, etype, value, tb, tb_offset=None):
             called.append(etype)
             shell.showtraceback((etype, value, tb), tb_offset=tb_offset)
 
         ip.set_custom_exc((SyntaxError,), my_handler)
         try:
             ip.run_cell("1f")
             # Check that this was called, and only once.
             self.assertEqual(called, [SyntaxError])
         finally:
             # Reset the custom exception hook
             ip.set_custom_exc((), None)
 
     def test_custom_exception(self):
         called = []
         def my_handler(shell, etype, value, tb, tb_offset=None):
             called.append(etype)
             shell.showtraceback((etype, value, tb), tb_offset=tb_offset)
         
         ip.set_custom_exc((ValueError,), my_handler)
         try:
             res = ip.run_cell("raise ValueError('test')")
             # Check that this was called, and only once.
             self.assertEqual(called, [ValueError])
             # Check that the error is on the result object
             self.assertIsInstance(res.error_in_exec, ValueError)
         finally:
             # Reset the custom exception hook
             ip.set_custom_exc((), None)
     
     def test_mktempfile(self):
         filename = ip.mktempfile()
         # Check that we can open the file again on Windows
         with open(filename, 'w') as f:
             f.write('abc')
 
         filename = ip.mktempfile(data='blah')
         with open(filename, 'r') as f:
             self.assertEqual(f.read(), 'blah')
 
     def test_new_main_mod(self):
         # Smoketest to check that this accepts a unicode module name
         name = u'jiefmw'
         mod = ip.new_main_mod(u'%s.py' % name, name)
         self.assertEqual(mod.__name__, name)
 
     def test_get_exception_only(self):
         try:
             raise KeyboardInterrupt
         except KeyboardInterrupt:
             msg = ip.get_exception_only()
         self.assertEqual(msg, 'KeyboardInterrupt\n')
 
         try:
             raise DerivedInterrupt("foo")
         except KeyboardInterrupt:
             msg = ip.get_exception_only()
         self.assertEqual(msg, 'IPython.core.tests.test_interactiveshell.DerivedInterrupt: foo\n')
 
     def test_inspect_text(self):
         ip.run_cell('a = 5')
         text = ip.object_inspect_text('a')
         self.assertIsInstance(text, str)
 
 
 class TestSafeExecfileNonAsciiPath(unittest.TestCase):
 
     @onlyif_unicode_paths
     def setUp(self):
         self.BASETESTDIR = tempfile.mkdtemp()
         self.TESTDIR = join(self.BASETESTDIR, u"åäö")
         os.mkdir(self.TESTDIR)
         with open(join(self.TESTDIR, u"åäötestscript.py"), "w") as sfile:
             sfile.write("pass\n")
         self.oldpath = os.getcwd()
         os.chdir(self.TESTDIR)
         self.fname = u"åäötestscript.py"
 
     def tearDown(self):
         os.chdir(self.oldpath)
         shutil.rmtree(self.BASETESTDIR)
 
     @onlyif_unicode_paths
     def test_1(self):
         """Test safe_execfile with non-ascii path
         """
         ip.safe_execfile(self.fname, {}, raise_exceptions=True)
 
 class ExitCodeChecks(tt.TempFileMixin):
     def test_exit_code_ok(self):
         self.system('exit 0')
         self.assertEqual(ip.user_ns['_exit_code'], 0)
 
     def test_exit_code_error(self):
         self.system('exit 1')
         self.assertEqual(ip.user_ns['_exit_code'], 1)
     
     @skipif(not hasattr(signal, 'SIGALRM'))
     def test_exit_code_signal(self):
         self.mktmp("import signal, time\n"
                    "signal.setitimer(signal.ITIMER_REAL, 0.1)\n"
                    "time.sleep(1)\n")
         self.system("%s %s" % (sys.executable, self.fname))
         self.assertEqual(ip.user_ns['_exit_code'], -signal.SIGALRM)
     
     @onlyif_cmds_exist("csh")
     def test_exit_code_signal_csh(self):
         SHELL = os.environ.get('SHELL', None)
         os.environ['SHELL'] = find_cmd("csh")
         try:
             self.test_exit_code_signal()
         finally:
             if SHELL is not None:
                 os.environ['SHELL'] = SHELL
             else:
                 del os.environ['SHELL']
 
 class TestSystemRaw(unittest.TestCase, ExitCodeChecks):
     system = ip.system_raw
 
     @onlyif_unicode_paths
     def test_1(self):
         """Test system_raw with non-ascii cmd
         """
         cmd = u'''python -c "'åäö'"   '''
         ip.system_raw(cmd)
 
     @mock.patch('subprocess.call', side_effect=KeyboardInterrupt)
     @mock.patch('os.system', side_effect=KeyboardInterrupt)
     def test_control_c(self, *mocks):
         try:
             self.system("sleep 1 # wont happen")
         except KeyboardInterrupt:
             self.fail("system call should intercept "
                       "keyboard interrupt from subprocess.call")
         self.assertEqual(ip.user_ns['_exit_code'], -signal.SIGINT)
 
 # TODO: Exit codes are currently ignored on Windows.
 class TestSystemPipedExitCode(unittest.TestCase, ExitCodeChecks):
     system = ip.system_piped
 
     @skip_win32
     def test_exit_code_ok(self):
         ExitCodeChecks.test_exit_code_ok(self)
 
     @skip_win32
     def test_exit_code_error(self):
         ExitCodeChecks.test_exit_code_error(self)
 
     @skip_win32
     def test_exit_code_signal(self):
         ExitCodeChecks.test_exit_code_signal(self)
 
 class TestModules(unittest.TestCase, tt.TempFileMixin):
     def test_extraneous_loads(self):
         """Test we're not loading modules on startup that we shouldn't.
         """
         self.mktmp("import sys\n"
                    "print('numpy' in sys.modules)\n"
                    "print('ipyparallel' in sys.modules)\n"
                    "print('ipykernel' in sys.modules)\n"
                    )
         out = "False\nFalse\nFalse\n"
         tt.ipexec_validate(self.fname, out)
 
 class Negator(ast.NodeTransformer):
     """Negates all number literals in an AST."""
     def visit_Num(self, node):
         node.n = -node.n
         return node
 
 class TestAstTransform(unittest.TestCase):
     def setUp(self):
         self.negator = Negator()
         ip.ast_transformers.append(self.negator)
     
     def tearDown(self):
         ip.ast_transformers.remove(self.negator)
     
     def test_run_cell(self):
         with tt.AssertPrints('-34'):
             ip.run_cell('print (12 + 22)')
         
         # A named reference to a number shouldn't be transformed.
         ip.user_ns['n'] = 55
         with tt.AssertNotPrints('-55'):
             ip.run_cell('print (n)')
     
     def test_timeit(self):
         called = set()
         def f(x):
             called.add(x)
         ip.push({'f':f})
         
         with tt.AssertPrints("average of "):
             ip.run_line_magic("timeit", "-n1 f(1)")
         self.assertEqual(called, {-1})
         called.clear()
 
         with tt.AssertPrints("average of "):
             ip.run_cell_magic("timeit", "-n1 f(2)", "f(3)")
         self.assertEqual(called, {-2, -3})
     
     def test_time(self):
         called = []
         def f(x):
             called.append(x)
         ip.push({'f':f})
         
         # Test with an expression
         with tt.AssertPrints("Wall time: "):
             ip.run_line_magic("time", "f(5+9)")
         self.assertEqual(called, [-14])
         called[:] = []
         
         # Test with a statement (different code path)
         with tt.AssertPrints("Wall time: "):
             ip.run_line_magic("time", "a = f(-3 + -2)")
         self.assertEqual(called, [5])
     
     def test_macro(self):
         ip.push({'a':10})
         # The AST transformation makes this do a+=-1
         ip.define_macro("amacro", "a+=1\nprint(a)")
         
         with tt.AssertPrints("9"):
             ip.run_cell("amacro")
         with tt.AssertPrints("8"):
             ip.run_cell("amacro")
 
 class IntegerWrapper(ast.NodeTransformer):
     """Wraps all integers in a call to Integer()"""
     def visit_Num(self, node):
         if isinstance(node.n, int):
             return ast.Call(func=ast.Name(id='Integer', ctx=ast.Load()),
                             args=[node], keywords=[])
         return node
 
 class TestAstTransform2(unittest.TestCase):
     def setUp(self):
         self.intwrapper = IntegerWrapper()
         ip.ast_transformers.append(self.intwrapper)
         
         self.calls = []
         def Integer(*args):
             self.calls.append(args)
             return args
         ip.push({"Integer": Integer})
     
     def tearDown(self):
         ip.ast_transformers.remove(self.intwrapper)
         del ip.user_ns['Integer']
     
     def test_run_cell(self):
         ip.run_cell("n = 2")
         self.assertEqual(self.calls, [(2,)])
         
         # This shouldn't throw an error
         ip.run_cell("o = 2.0")
         self.assertEqual(ip.user_ns['o'], 2.0)
     
     def test_timeit(self):
         called = set()
         def f(x):
             called.add(x)
         ip.push({'f':f})
 
         with tt.AssertPrints("average of "):
             ip.run_line_magic("timeit", "-n1 f(1)")
         self.assertEqual(called, {(1,)})
         called.clear()
 
         with tt.AssertPrints("average of "):
             ip.run_cell_magic("timeit", "-n1 f(2)", "f(3)")
         self.assertEqual(called, {(2,), (3,)})
 
 class ErrorTransformer(ast.NodeTransformer):
     """Throws an error when it sees a number."""
     def visit_Num(self, node):
         raise ValueError("test")
 
 class TestAstTransformError(unittest.TestCase):
     def test_unregistering(self):
         err_transformer = ErrorTransformer()
         ip.ast_transformers.append(err_transformer)
         
         with tt.AssertPrints("unregister", channel='stderr'):
             ip.run_cell("1 + 2")
         
         # This should have been removed.
         nt.assert_not_in(err_transformer, ip.ast_transformers)
 
 
 class StringRejector(ast.NodeTransformer):
     """Throws an InputRejected when it sees a string literal.
 
     Used to verify that NodeTransformers can signal that a piece of code should
     not be executed by throwing an InputRejected.
     """
 
     def visit_Str(self, node):
         raise InputRejected("test")
 
 
 class TestAstTransformInputRejection(unittest.TestCase):
 
     def setUp(self):
         self.transformer = StringRejector()
         ip.ast_transformers.append(self.transformer)
 
     def tearDown(self):
         ip.ast_transformers.remove(self.transformer)
 
     def test_input_rejection(self):
         """Check that NodeTransformers can reject input."""
 
         expect_exception_tb = tt.AssertPrints("InputRejected: test")
         expect_no_cell_output = tt.AssertNotPrints("'unsafe'", suppress=False)
 
         # Run the same check twice to verify that the transformer is not
         # disabled after raising.
         with expect_exception_tb, expect_no_cell_output:
             ip.run_cell("'unsafe'")
 
         with expect_exception_tb, expect_no_cell_output:
             res = ip.run_cell("'unsafe'")
 
         self.assertIsInstance(res.error_before_exec, InputRejected)
 
 def test__IPYTHON__():
     # This shouldn't raise a NameError, that's all
     __IPYTHON__
 
 
 class DummyRepr(object):
     def __repr__(self):
         return "DummyRepr"
     
     def _repr_html_(self):
         return "<b>dummy</b>"
     
     def _repr_javascript_(self):
         return "console.log('hi');", {'key': 'value'}
     
 
 def test_user_variables():
     # enable all formatters
     ip.display_formatter.active_types = ip.display_formatter.format_types
     
     ip.user_ns['dummy'] = d = DummyRepr()
     keys = {'dummy', 'doesnotexist'}
     r = ip.user_expressions({ key:key for key in keys})
 
     nt.assert_equal(keys, set(r.keys()))
     dummy = r['dummy']
     nt.assert_equal({'status', 'data', 'metadata'}, set(dummy.keys()))
     nt.assert_equal(dummy['status'], 'ok')
     data = dummy['data']
     metadata = dummy['metadata']
     nt.assert_equal(data.get('text/html'), d._repr_html_())
     js, jsmd = d._repr_javascript_()
     nt.assert_equal(data.get('application/javascript'), js)
     nt.assert_equal(metadata.get('application/javascript'), jsmd)
     
     dne = r['doesnotexist']
     nt.assert_equal(dne['status'], 'error')
     nt.assert_equal(dne['ename'], 'NameError')
     
     # back to text only
     ip.display_formatter.active_types = ['text/plain']
     
 def test_user_expression():
     # enable all formatters
     ip.display_formatter.active_types = ip.display_formatter.format_types
     query = {
         'a' : '1 + 2',
         'b' : '1/0',
     }
     r = ip.user_expressions(query)
     import pprint
     pprint.pprint(r)
     nt.assert_equal(set(r.keys()), set(query.keys()))
     a = r['a']
     nt.assert_equal({'status', 'data', 'metadata'}, set(a.keys()))
     nt.assert_equal(a['status'], 'ok')
     data = a['data']
     metadata = a['metadata']
     nt.assert_equal(data.get('text/plain'), '3')
     
     b = r['b']
     nt.assert_equal(b['status'], 'error')
     nt.assert_equal(b['ename'], 'ZeroDivisionError')
     
     # back to text only
     ip.display_formatter.active_types = ['text/plain']
     
 
 
 
 
 class TestSyntaxErrorTransformer(unittest.TestCase):
     """Check that SyntaxError raised by an input transformer is handled by run_cell()"""
 
     class SyntaxErrorTransformer(InputTransformer):
 
         def push(self, line):
             pos = line.find('syntaxerror')
             if pos >= 0:
                 e = SyntaxError('input contains "syntaxerror"')
                 e.text = line
                 e.offset = pos + 1
                 raise e
             return line
 
         def reset(self):
             pass
 
     def setUp(self):
         self.transformer = TestSyntaxErrorTransformer.SyntaxErrorTransformer()
         ip.input_splitter.python_line_transforms.append(self.transformer)
         ip.input_transformer_manager.python_line_transforms.append(self.transformer)
 
     def tearDown(self):
         ip.input_splitter.python_line_transforms.remove(self.transformer)
         ip.input_transformer_manager.python_line_transforms.remove(self.transformer)
 
     def test_syntaxerror_input_transformer(self):
         with tt.AssertPrints('1234'):
             ip.run_cell('1234')
         with tt.AssertPrints('SyntaxError: invalid syntax'):
             ip.run_cell('1 2 3')   # plain python syntax error
         with tt.AssertPrints('SyntaxError: input contains "syntaxerror"'):
             ip.run_cell('2345  # syntaxerror')  # input transformer syntax error
         with tt.AssertPrints('3456'):
             ip.run_cell('3456')
 
 
 
 def test_warning_suppression():
     ip.run_cell("import warnings")
     try:
         with tt.AssertPrints("UserWarning: asdf", channel="stderr"):
             ip.run_cell("warnings.warn('asdf')")
         # Here's the real test -- if we run that again, we should get the
         # warning again. Traditionally, each warning was only issued once per
         # IPython session (approximately), even if the user typed in new and
         # different code that should have also triggered the warning, leading
         # to much confusion.
         with tt.AssertPrints("UserWarning: asdf", channel="stderr"):
             ip.run_cell("warnings.warn('asdf')")
     finally:
         ip.run_cell("del warnings")
 
 
 def test_deprecation_warning():
     ip.run_cell("""
 import warnings
 def wrn():
     warnings.warn(
         "I AM  A WARNING",
         DeprecationWarning
     )
         """)
     try:
         with tt.AssertPrints("I AM  A WARNING", channel="stderr"):
             ip.run_cell("wrn()")
     finally:
         ip.run_cell("del warnings")
         ip.run_cell("del wrn")
 
 
 class TestImportNoDeprecate(tt.TempFileMixin):
 
     def setup(self):
         """Make a valid python temp file."""
         self.mktmp("""
 import warnings
 def wrn():
     warnings.warn(
         "I AM  A WARNING",
         DeprecationWarning
     )
 """)
 
     def test_no_dep(self):
         """
         No deprecation warning should be raised from imported functions
         """
         ip.run_cell("from {} import wrn".format(self.fname))
 
         with tt.AssertNotPrints("I AM  A WARNING"):
             ip.run_cell("wrn()")
         ip.run_cell("del wrn")

IPython/core/tests/test_magic.py

@@ -1,987 +1,979 b''
 # -*- coding: utf-8 -*-
 """Tests for various magic functions.
 
 Needs to be run by nose (to make ipython session available).
 """
 
 import io
 import os
 import sys
 import warnings
 from unittest import TestCase
-
-try:
-    from importlib import invalidate_caches   # Required from Python 3.3
-except ImportError:
-    def invalidate_caches():
-        pass
+from importlib import invalidate_caches
+from io import StringIO
 
 import nose.tools as nt
 
 from IPython import get_ipython
 from IPython.core import magic
 from IPython.core.error import UsageError
 from IPython.core.magic import (Magics, magics_class, line_magic,
                                 cell_magic,
                                 register_line_magic, register_cell_magic)
 from IPython.core.magics import execution, script, code
 from IPython.testing import decorators as dec
 from IPython.testing import tools as tt
 from IPython.utils import py3compat
 from IPython.utils.io import capture_output
 from IPython.utils.tempdir import TemporaryDirectory
 from IPython.utils.process import find_cmd
 
-if py3compat.PY3:
-    from io import StringIO
-else:
-    from StringIO import StringIO
 
 
 _ip = get_ipython()
 
 @magic.magics_class
 class DummyMagics(magic.Magics): pass
 
 def test_extract_code_ranges():
     instr = "1 3 5-6 7-9 10:15 17: :10 10- -13 :"
     expected = [(0, 1), 
                 (2, 3),
                 (4, 6),
                 (6, 9),
                 (9, 14),
                 (16, None),
                 (None, 9),
                 (9, None),
                 (None, 13),
                 (None, None)]
     actual = list(code.extract_code_ranges(instr))
     nt.assert_equal(actual, expected)
 
 def test_extract_symbols():
     source = """import foo\na = 10\ndef b():\n    return 42\n\n\nclass A: pass\n\n\n"""
     symbols_args = ["a", "b", "A", "A,b", "A,a", "z"]
     expected = [([], ['a']),
                 (["def b():\n    return 42\n"], []),
                 (["class A: pass\n"], []),
                 (["class A: pass\n", "def b():\n    return 42\n"], []),
                 (["class A: pass\n"], ['a']),
                 ([], ['z'])]
     for symbols, exp in zip(symbols_args, expected):
         nt.assert_equal(code.extract_symbols(source, symbols), exp)
 
 
 def test_extract_symbols_raises_exception_with_non_python_code():
     source = ("=begin A Ruby program :)=end\n"
               "def hello\n"
               "puts 'Hello world'\n"
               "end")
     with nt.assert_raises(SyntaxError):
         code.extract_symbols(source, "hello")
 
 def test_config():
     """ test that config magic does not raise
     can happen if Configurable init is moved too early into
     Magics.__init__ as then a Config object will be registerd as a
     magic.
     """
     ## should not raise.
     _ip.magic('config')
 
 def test_rehashx():
     # clear up everything
     _ip.alias_manager.clear_aliases()
     del _ip.db['syscmdlist']
     
     _ip.magic('rehashx')
     # Practically ALL ipython development systems will have more than 10 aliases
 
     nt.assert_true(len(_ip.alias_manager.aliases) > 10)
     for name, cmd in _ip.alias_manager.aliases:
         # we must strip dots from alias names
         nt.assert_not_in('.', name)
 
     # rehashx must fill up syscmdlist
     scoms = _ip.db['syscmdlist']
     nt.assert_true(len(scoms) > 10)
 
 
 def test_magic_parse_options():
     """Test that we don't mangle paths when parsing magic options."""
     ip = get_ipython()
     path = 'c:\\x'
     m = DummyMagics(ip)
     opts = m.parse_options('-f %s' % path,'f:')[0]
     # argv splitting is os-dependent
     if os.name == 'posix':
         expected = 'c:x'
     else:
         expected = path
     nt.assert_equal(opts['f'], expected)
 
 def test_magic_parse_long_options():
     """Magic.parse_options can handle --foo=bar long options"""
     ip = get_ipython()
     m = DummyMagics(ip)
     opts, _ = m.parse_options('--foo --bar=bubble', 'a', 'foo', 'bar=')
     nt.assert_in('foo', opts)
     nt.assert_in('bar', opts)
     nt.assert_equal(opts['bar'], "bubble")
 
 
 @dec.skip_without('sqlite3')
 def doctest_hist_f():
     """Test %hist -f with temporary filename.
 
     In [9]: import tempfile
 
     In [10]: tfile = tempfile.mktemp('.py','tmp-ipython-')
 
     In [11]: %hist -nl -f $tfile 3
 
     In [13]: import os; os.unlink(tfile)
     """
 
 
 @dec.skip_without('sqlite3')
 def doctest_hist_r():
     """Test %hist -r
 
     XXX - This test is not recording the output correctly.  For some reason, in
     testing mode the raw history isn't getting populated.  No idea why.
     Disabling the output checking for now, though at least we do run it.
 
     In [1]: 'hist' in _ip.lsmagic()
     Out[1]: True
 
     In [2]: x=1
 
     In [3]: %hist -rl 2
     x=1 # random
     %hist -r 2
     """
 
 
 @dec.skip_without('sqlite3')
 def doctest_hist_op():
     """Test %hist -op
 
     In [1]: class b(float):
        ...:     pass
        ...: 
 
     In [2]: class s(object):
        ...:     def __str__(self):
        ...:         return 's'
        ...: 
 
     In [3]: 
 
     In [4]: class r(b):
        ...:     def __repr__(self):
        ...:         return 'r'
        ...: 
 
     In [5]: class sr(s,r): pass
        ...: 
 
     In [6]: 
 
     In [7]: bb=b()
 
     In [8]: ss=s()
 
     In [9]: rr=r()
 
     In [10]: ssrr=sr()
 
     In [11]: 4.5
     Out[11]: 4.5
 
     In [12]: str(ss)
     Out[12]: 's'
 
     In [13]: 
 
     In [14]: %hist -op
     >>> class b:
     ...     pass
     ... 
     >>> class s(b):
     ...     def __str__(self):
     ...         return 's'
     ... 
     >>> 
     >>> class r(b):
     ...     def __repr__(self):
     ...         return 'r'
     ... 
     >>> class sr(s,r): pass
     >>> 
     >>> bb=b()
     >>> ss=s()
     >>> rr=r()
     >>> ssrr=sr()
     >>> 4.5
     4.5
     >>> str(ss)
     's'
     >>> 
     """
 
 def test_hist_pof():
     ip = get_ipython()
     ip.run_cell(u"1+2", store_history=True)
     #raise Exception(ip.history_manager.session_number)
     #raise Exception(list(ip.history_manager._get_range_session()))
     with TemporaryDirectory() as td:
         tf = os.path.join(td, 'hist.py')
         ip.run_line_magic('history', '-pof %s' % tf)
         assert os.path.isfile(tf)
 
 
 @dec.skip_without('sqlite3')
 def test_macro():
     ip = get_ipython()
     ip.history_manager.reset()   # Clear any existing history.
     cmds = ["a=1", "def b():\n  return a**2", "print(a,b())"]
     for i, cmd in enumerate(cmds, start=1):
         ip.history_manager.store_inputs(i, cmd)
     ip.magic("macro test 1-3")
     nt.assert_equal(ip.user_ns["test"].value, "\n".join(cmds)+"\n")
     
     # List macros
     nt.assert_in("test", ip.magic("macro"))
 
 
 @dec.skip_without('sqlite3')
 def test_macro_run():
     """Test that we can run a multi-line macro successfully."""
     ip = get_ipython()
     ip.history_manager.reset()
     cmds = ["a=10", "a+=1", py3compat.doctest_refactor_print("print a"),
                                                             "%macro test 2-3"]
     for cmd in cmds:
         ip.run_cell(cmd, store_history=True)
     nt.assert_equal(ip.user_ns["test"].value,
                             py3compat.doctest_refactor_print("a+=1\nprint a\n"))
     with tt.AssertPrints("12"):
         ip.run_cell("test")
     with tt.AssertPrints("13"):
         ip.run_cell("test")
 
 
 def test_magic_magic():
     """Test %magic"""
     ip = get_ipython()
     with capture_output() as captured:
         ip.magic("magic")
     
     stdout = captured.stdout
     nt.assert_in('%magic', stdout)
     nt.assert_in('IPython', stdout)
     nt.assert_in('Available', stdout)
 
 
 @dec.skipif_not_numpy
 def test_numpy_reset_array_undec():
     "Test '%reset array' functionality"
     _ip.ex('import numpy as np')
     _ip.ex('a = np.empty(2)')
     nt.assert_in('a', _ip.user_ns)
     _ip.magic('reset -f array')
     nt.assert_not_in('a', _ip.user_ns)
 
 def test_reset_out():
     "Test '%reset out' magic"
     _ip.run_cell("parrot = 'dead'", store_history=True)
     # test '%reset -f out', make an Out prompt
     _ip.run_cell("parrot", store_history=True)
     nt.assert_true('dead' in [_ip.user_ns[x] for x in ('_','__','___')])
     _ip.magic('reset -f out')
     nt.assert_false('dead' in [_ip.user_ns[x] for x in ('_','__','___')])
     nt.assert_equal(len(_ip.user_ns['Out']), 0)
 
 def test_reset_in():
     "Test '%reset in' magic"
     # test '%reset -f in'
     _ip.run_cell("parrot", store_history=True)
     nt.assert_true('parrot' in [_ip.user_ns[x] for x in ('_i','_ii','_iii')])
     _ip.magic('%reset -f in')
     nt.assert_false('parrot' in [_ip.user_ns[x] for x in ('_i','_ii','_iii')])
     nt.assert_equal(len(set(_ip.user_ns['In'])), 1)
 
 def test_reset_dhist():
     "Test '%reset dhist' magic"
     _ip.run_cell("tmp = [d for d in _dh]") # copy before clearing
     _ip.magic('cd ' + os.path.dirname(nt.__file__))
     _ip.magic('cd -')
     nt.assert_true(len(_ip.user_ns['_dh']) > 0)
     _ip.magic('reset -f dhist')
     nt.assert_equal(len(_ip.user_ns['_dh']), 0)
     _ip.run_cell("_dh = [d for d in tmp]") #restore
 
 def test_reset_in_length():
     "Test that '%reset in' preserves In[] length"
     _ip.run_cell("print 'foo'")
     _ip.run_cell("reset -f in")
     nt.assert_equal(len(_ip.user_ns['In']), _ip.displayhook.prompt_count+1)
 
 def test_tb_syntaxerror():
     """test %tb after a SyntaxError"""
     ip = get_ipython()
     ip.run_cell("for")
     
     # trap and validate stdout
     save_stdout = sys.stdout
     try:
         sys.stdout = StringIO()
         ip.run_cell("%tb")
         out = sys.stdout.getvalue()
     finally:
         sys.stdout = save_stdout
     # trim output, and only check the last line
     last_line = out.rstrip().splitlines()[-1].strip()
     nt.assert_equal(last_line, "SyntaxError: invalid syntax")
 
 
 def test_time():
     ip = get_ipython()
     
     with tt.AssertPrints("Wall time: "):
         ip.run_cell("%time None")
     
     ip.run_cell("def f(kmjy):\n"
                 "    %time print (2*kmjy)")
     
     with tt.AssertPrints("Wall time: "):
         with tt.AssertPrints("hihi", suppress=False):
             ip.run_cell("f('hi')")
 
 
 @dec.skip_win32
 def test_time2():
     ip = get_ipython()
     
     with tt.AssertPrints("CPU times: user "):
         ip.run_cell("%time None")
 
 def test_time3():
     """Erroneous magic function calls, issue gh-3334"""
     ip = get_ipython()
     ip.user_ns.pop('run', None)
     
     with tt.AssertNotPrints("not found", channel='stderr'):
         ip.run_cell("%%time\n"
                     "run = 0\n"
                     "run += 1")
 
 def test_doctest_mode():
     "Toggle doctest_mode twice, it should be a no-op and run without error"
     _ip.magic('doctest_mode')
     _ip.magic('doctest_mode')
 
 
 def test_parse_options():
     """Tests for basic options parsing in magics."""
     # These are only the most minimal of tests, more should be added later.  At
     # the very least we check that basic text/unicode calls work OK.
     m = DummyMagics(_ip)
     nt.assert_equal(m.parse_options('foo', '')[1], 'foo')
     nt.assert_equal(m.parse_options(u'foo', '')[1], u'foo')
 
     
 def test_dirops():
     """Test various directory handling operations."""
     # curpath = lambda :os.path.splitdrive(os.getcwd())[1].replace('\\','/')
     curpath = os.getcwd
     startdir = os.getcwd()
     ipdir = os.path.realpath(_ip.ipython_dir)
     try:
         _ip.magic('cd "%s"' % ipdir)
         nt.assert_equal(curpath(), ipdir)
         _ip.magic('cd -')
         nt.assert_equal(curpath(), startdir)
         _ip.magic('pushd "%s"' % ipdir)
         nt.assert_equal(curpath(), ipdir)
         _ip.magic('popd')
         nt.assert_equal(curpath(), startdir)
     finally:
         os.chdir(startdir)
 
 
 def test_xmode():
     # Calling xmode three times should be a no-op
     xmode = _ip.InteractiveTB.mode
     for i in range(3):
         _ip.magic("xmode")
     nt.assert_equal(_ip.InteractiveTB.mode, xmode)
     
 def test_reset_hard():
     monitor = []
     class A(object):
         def __del__(self):
             monitor.append(1)
         def __repr__(self):
             return "<A instance>"
             
     _ip.user_ns["a"] = A()
     _ip.run_cell("a")
     
     nt.assert_equal(monitor, [])
     _ip.magic("reset -f")
     nt.assert_equal(monitor, [1])
     
 class TestXdel(tt.TempFileMixin):
     def test_xdel(self):
         """Test that references from %run are cleared by xdel."""
         src = ("class A(object):\n"
                "    monitor = []\n"
                "    def __del__(self):\n"
                "        self.monitor.append(1)\n"
                "a = A()\n")
         self.mktmp(src)
         # %run creates some hidden references...
         _ip.magic("run %s" % self.fname)
         # ... as does the displayhook.
         _ip.run_cell("a")
         
         monitor = _ip.user_ns["A"].monitor
         nt.assert_equal(monitor, [])
         
         _ip.magic("xdel a")
         
         # Check that a's __del__ method has been called.
         nt.assert_equal(monitor, [1])
 
 def doctest_who():
     """doctest for %who
     
     In [1]: %reset -f
     
     In [2]: alpha = 123
     
     In [3]: beta = 'beta'
     
     In [4]: %who int
     alpha
     
     In [5]: %who str
     beta
     
     In [6]: %whos
     Variable   Type    Data/Info
     ----------------------------
     alpha      int     123
     beta       str     beta
     
     In [7]: %who_ls
     Out[7]: ['alpha', 'beta']
     """
 
 def test_whos():
     """Check that whos is protected against objects where repr() fails."""
     class A(object):
         def __repr__(self):
             raise Exception()
     _ip.user_ns['a'] = A()
     _ip.magic("whos")
 
 @py3compat.u_format
 def doctest_precision():
     """doctest for %precision
     
     In [1]: f = get_ipython().display_formatter.formatters['text/plain']
     
     In [2]: %precision 5
     Out[2]: {u}'%.5f'
     
     In [3]: f.float_format
     Out[3]: {u}'%.5f'
     
     In [4]: %precision %e
     Out[4]: {u}'%e'
     
     In [5]: f(3.1415927)
     Out[5]: {u}'3.141593e+00'
     """
 
 def test_psearch():
     with tt.AssertPrints("dict.fromkeys"):
         _ip.run_cell("dict.fr*?")
 
 def test_timeit_shlex():
     """test shlex issues with timeit (#1109)"""
     _ip.ex("def f(*a,**kw): pass")
     _ip.magic('timeit -n1 "this is a bug".count(" ")')
     _ip.magic('timeit -r1 -n1 f(" ", 1)')
     _ip.magic('timeit -r1 -n1 f(" ", 1, " ", 2, " ")')
     _ip.magic('timeit -r1 -n1 ("a " + "b")')
     _ip.magic('timeit -r1 -n1 f("a " + "b")')
     _ip.magic('timeit -r1 -n1 f("a " + "b ")')
 
 
 def test_timeit_arguments():
     "Test valid timeit arguments, should not cause SyntaxError (GH #1269)"
     _ip.magic("timeit ('#')")
 
 
 def test_timeit_special_syntax():
     "Test %%timeit with IPython special syntax"
     @register_line_magic
     def lmagic(line):
         ip = get_ipython()
         ip.user_ns['lmagic_out'] = line
 
     # line mode test
     _ip.run_line_magic('timeit', '-n1 -r1 %lmagic my line')
     nt.assert_equal(_ip.user_ns['lmagic_out'], 'my line')
     # cell mode test
     _ip.run_cell_magic('timeit', '-n1 -r1', '%lmagic my line2')
     nt.assert_equal(_ip.user_ns['lmagic_out'], 'my line2')
 
 def test_timeit_return():
     """
     test wether timeit -o return object
     """
 
     res = _ip.run_line_magic('timeit','-n10 -r10 -o 1')
     assert(res is not None)
 
 def test_timeit_quiet():
     """
     test quiet option of timeit magic
     """
     with tt.AssertNotPrints("loops"):
         _ip.run_cell("%timeit -n1 -r1 -q 1")
 
 def test_timeit_return_quiet():
     with tt.AssertNotPrints("loops"):
         res = _ip.run_line_magic('timeit', '-n1 -r1 -q -o 1')
     assert (res is not None)
 
 @dec.skipif(execution.profile is None)
 def test_prun_special_syntax():
     "Test %%prun with IPython special syntax"
     @register_line_magic
     def lmagic(line):
         ip = get_ipython()
         ip.user_ns['lmagic_out'] = line
 
     # line mode test
     _ip.run_line_magic('prun', '-q %lmagic my line')
     nt.assert_equal(_ip.user_ns['lmagic_out'], 'my line')
     # cell mode test
     _ip.run_cell_magic('prun', '-q', '%lmagic my line2')
     nt.assert_equal(_ip.user_ns['lmagic_out'], 'my line2')
 
 @dec.skipif(execution.profile is None)
 def test_prun_quotes():
     "Test that prun does not clobber string escapes (GH #1302)"
     _ip.magic(r"prun -q x = '\t'")
     nt.assert_equal(_ip.user_ns['x'], '\t')
 
 def test_extension():
     # Debugging information for failures of this test
     print('sys.path:')
     for p in sys.path:
         print(' ', p)
     print('CWD', os.getcwd())
 
     nt.assert_raises(ImportError, _ip.magic, "load_ext daft_extension")
     daft_path = os.path.join(os.path.dirname(__file__), "daft_extension")
     sys.path.insert(0, daft_path)
     try:
         _ip.user_ns.pop('arq', None)
         invalidate_caches()   # Clear import caches
         _ip.magic("load_ext daft_extension")
         nt.assert_equal(_ip.user_ns['arq'], 185)
         _ip.magic("unload_ext daft_extension")
         assert 'arq' not in _ip.user_ns
     finally:
         sys.path.remove(daft_path)
 
 
 def test_notebook_export_json():
     _ip = get_ipython()
     _ip.history_manager.reset()   # Clear any existing history.
     cmds = [u"a=1", u"def b():\n  return a**2", u"print('noël, été', b())"]
     for i, cmd in enumerate(cmds, start=1):
         _ip.history_manager.store_inputs(i, cmd)
     with TemporaryDirectory() as td:
         outfile = os.path.join(td, "nb.ipynb")
         _ip.magic("notebook -e %s" % outfile)
 
 
 class TestEnv(TestCase):
 
     def test_env(self):
         env = _ip.magic("env")
         self.assertTrue(isinstance(env, dict))
 
     def test_env_get_set_simple(self):
         env = _ip.magic("env var val1")
         self.assertEqual(env, None)
         self.assertEqual(os.environ['var'], 'val1')
         self.assertEqual(_ip.magic("env var"), 'val1')
         env = _ip.magic("env var=val2")
         self.assertEqual(env, None)
         self.assertEqual(os.environ['var'], 'val2')
 
     def test_env_get_set_complex(self):
         env = _ip.magic("env var 'val1 '' 'val2")
         self.assertEqual(env, None)
         self.assertEqual(os.environ['var'], "'val1 '' 'val2")
         self.assertEqual(_ip.magic("env var"), "'val1 '' 'val2")
         env = _ip.magic('env var=val2 val3="val4')
         self.assertEqual(env, None)
         self.assertEqual(os.environ['var'], 'val2 val3="val4')
 
     def test_env_set_bad_input(self):
         self.assertRaises(UsageError, lambda: _ip.magic("set_env var"))
 
     def test_env_set_whitespace(self):
         self.assertRaises(UsageError, lambda: _ip.magic("env var A=B"))
 
 
 class CellMagicTestCase(TestCase):
 
     def check_ident(self, magic):
         # Manually called, we get the result
         out = _ip.run_cell_magic(magic, 'a', 'b')
         nt.assert_equal(out, ('a','b'))
         # Via run_cell, it goes into the user's namespace via displayhook
         _ip.run_cell('%%' + magic +' c\nd')
         nt.assert_equal(_ip.user_ns['_'], ('c','d'))
 
     def test_cell_magic_func_deco(self):
         "Cell magic using simple decorator"
         @register_cell_magic
         def cellm(line, cell):
             return line, cell
 
         self.check_ident('cellm')
 
     def test_cell_magic_reg(self):
         "Cell magic manually registered"
         def cellm(line, cell):
             return line, cell
 
         _ip.register_magic_function(cellm, 'cell', 'cellm2')
         self.check_ident('cellm2')
 
     def test_cell_magic_class(self):
         "Cell magics declared via a class"
         @magics_class
         class MyMagics(Magics):
 
             @cell_magic
             def cellm3(self, line, cell):
                 return line, cell
 
         _ip.register_magics(MyMagics)
         self.check_ident('cellm3')
 
     def test_cell_magic_class2(self):
         "Cell magics declared via a class, #2"
         @magics_class
         class MyMagics2(Magics):
 
             @cell_magic('cellm4')
             def cellm33(self, line, cell):
                 return line, cell
             
         _ip.register_magics(MyMagics2)
         self.check_ident('cellm4')
         # Check that nothing is registered as 'cellm33'
         c33 = _ip.find_cell_magic('cellm33')
         nt.assert_equal(c33, None)
 
 def test_file():
     """Basic %%file"""
     ip = get_ipython()
     with TemporaryDirectory() as td:
         fname = os.path.join(td, 'file1')
         ip.run_cell_magic("file", fname, u'\n'.join([
             'line1',
             'line2',
         ]))
         with open(fname) as f:
             s = f.read()
         nt.assert_in('line1\n', s)
         nt.assert_in('line2', s)
 
 def test_file_var_expand():
     """%%file $filename"""
     ip = get_ipython()
     with TemporaryDirectory() as td:
         fname = os.path.join(td, 'file1')
         ip.user_ns['filename'] = fname
         ip.run_cell_magic("file", '$filename', u'\n'.join([
             'line1',
             'line2',
         ]))
         with open(fname) as f:
             s = f.read()
         nt.assert_in('line1\n', s)
         nt.assert_in('line2', s)
 
 def test_file_unicode():
     """%%file with unicode cell"""
     ip = get_ipython()
     with TemporaryDirectory() as td:
         fname = os.path.join(td, 'file1')
         ip.run_cell_magic("file", fname, u'\n'.join([
             u'liné1',
             u'liné2',
         ]))
         with io.open(fname, encoding='utf-8') as f:
             s = f.read()
         nt.assert_in(u'liné1\n', s)
         nt.assert_in(u'liné2', s)
 
 def test_file_amend():
     """%%file -a amends files"""
     ip = get_ipython()
     with TemporaryDirectory() as td:
         fname = os.path.join(td, 'file2')
         ip.run_cell_magic("file", fname, u'\n'.join([
             'line1',
             'line2',
         ]))
         ip.run_cell_magic("file", "-a %s" % fname, u'\n'.join([
             'line3',
             'line4',
         ]))
         with open(fname) as f:
             s = f.read()
         nt.assert_in('line1\n', s)
         nt.assert_in('line3\n', s)
         
     
 def test_script_config():
     ip = get_ipython()
     ip.config.ScriptMagics.script_magics = ['whoda']
     sm = script.ScriptMagics(shell=ip)
     nt.assert_in('whoda', sm.magics['cell'])
 
 @dec.skip_win32
 def test_script_out():
     ip = get_ipython()
     ip.run_cell_magic("script", "--out output sh", "echo 'hi'")
     nt.assert_equal(ip.user_ns['output'], 'hi\n')
 
 @dec.skip_win32
 def test_script_err():
     ip = get_ipython()
     ip.run_cell_magic("script", "--err error sh", "echo 'hello' >&2")
     nt.assert_equal(ip.user_ns['error'], 'hello\n')
 
 @dec.skip_win32
 def test_script_out_err():
     ip = get_ipython()
     ip.run_cell_magic("script", "--out output --err error sh", "echo 'hi'\necho 'hello' >&2")
     nt.assert_equal(ip.user_ns['output'], 'hi\n')
     nt.assert_equal(ip.user_ns['error'], 'hello\n')
 
 @dec.skip_win32
 def test_script_bg_out():
     ip = get_ipython()
     ip.run_cell_magic("script", "--bg --out output sh", "echo 'hi'")
     nt.assert_equal(ip.user_ns['output'].read(), b'hi\n')
 
 @dec.skip_win32
 def test_script_bg_err():
     ip = get_ipython()
     ip.run_cell_magic("script", "--bg --err error sh", "echo 'hello' >&2")
     nt.assert_equal(ip.user_ns['error'].read(), b'hello\n')
 
 @dec.skip_win32
 def test_script_bg_out_err():
     ip = get_ipython()
     ip.run_cell_magic("script", "--bg --out output --err error sh", "echo 'hi'\necho 'hello' >&2")
     nt.assert_equal(ip.user_ns['output'].read(), b'hi\n')
     nt.assert_equal(ip.user_ns['error'].read(), b'hello\n')
 
 def test_script_defaults():
     ip = get_ipython()
     for cmd in ['sh', 'bash', 'perl', 'ruby']:
         try:
             find_cmd(cmd)
         except Exception:
             pass
         else:
             nt.assert_in(cmd, ip.magics_manager.magics['cell'])
 
 
 @magics_class
 class FooFoo(Magics):
     """class with both %foo and %%foo magics"""
     @line_magic('foo')
     def line_foo(self, line):
         "I am line foo"
         pass
 
     @cell_magic("foo")
     def cell_foo(self, line, cell):
         "I am cell foo, not line foo"
         pass
 
 def test_line_cell_info():
     """%%foo and %foo magics are distinguishable to inspect"""
     ip = get_ipython()
     ip.magics_manager.register(FooFoo)
     oinfo = ip.object_inspect('foo')
     nt.assert_true(oinfo['found'])
     nt.assert_true(oinfo['ismagic'])
     
     oinfo = ip.object_inspect('%%foo')
     nt.assert_true(oinfo['found'])
     nt.assert_true(oinfo['ismagic'])
     nt.assert_equal(oinfo['docstring'], FooFoo.cell_foo.__doc__)
 
     oinfo = ip.object_inspect('%foo')
     nt.assert_true(oinfo['found'])
     nt.assert_true(oinfo['ismagic'])
     nt.assert_equal(oinfo['docstring'], FooFoo.line_foo.__doc__)
 
 def test_multiple_magics():
     ip = get_ipython()
     foo1 = FooFoo(ip)
     foo2 = FooFoo(ip)
     mm = ip.magics_manager
     mm.register(foo1)
     nt.assert_true(mm.magics['line']['foo'].__self__ is foo1)
     mm.register(foo2)
     nt.assert_true(mm.magics['line']['foo'].__self__ is foo2)
 
 def test_alias_magic():
     """Test %alias_magic."""
     ip = get_ipython()
     mm = ip.magics_manager
 
     # Basic operation: both cell and line magics are created, if possible.
     ip.run_line_magic('alias_magic', 'timeit_alias timeit')
     nt.assert_in('timeit_alias', mm.magics['line'])
     nt.assert_in('timeit_alias', mm.magics['cell'])
 
     # --cell is specified, line magic not created.
     ip.run_line_magic('alias_magic', '--cell timeit_cell_alias timeit')
     nt.assert_not_in('timeit_cell_alias', mm.magics['line'])
     nt.assert_in('timeit_cell_alias', mm.magics['cell'])
 
     # Test that line alias is created successfully.
     ip.run_line_magic('alias_magic', '--line env_alias env')
     nt.assert_equal(ip.run_line_magic('env', ''),
                     ip.run_line_magic('env_alias', ''))
 
 def test_save():
     """Test %save."""
     ip = get_ipython()
     ip.history_manager.reset()   # Clear any existing history.
     cmds = [u"a=1", u"def b():\n  return a**2", u"print(a, b())"]
     for i, cmd in enumerate(cmds, start=1):
         ip.history_manager.store_inputs(i, cmd)
     with TemporaryDirectory() as tmpdir:
         file = os.path.join(tmpdir, "testsave.py")
         ip.run_line_magic("save", "%s 1-10" % file)
         with open(file) as f:
             content = f.read()
             nt.assert_equal(content.count(cmds[0]), 1)
             nt.assert_in('coding: utf-8', content)
         ip.run_line_magic("save", "-a %s 1-10" % file)
         with open(file) as f:
             content = f.read()
             nt.assert_equal(content.count(cmds[0]), 2)
             nt.assert_in('coding: utf-8', content)
 
 
 def test_store():
     """Test %store."""
     ip = get_ipython()
     ip.run_line_magic('load_ext', 'storemagic')
     
     # make sure the storage is empty
     ip.run_line_magic('store', '-z')
     ip.user_ns['var'] = 42
     ip.run_line_magic('store', 'var')
     ip.user_ns['var'] = 39
     ip.run_line_magic('store', '-r')
     nt.assert_equal(ip.user_ns['var'], 42)
 
     ip.run_line_magic('store', '-d var')
     ip.user_ns['var'] = 39
     ip.run_line_magic('store' , '-r')
     nt.assert_equal(ip.user_ns['var'], 39)
 
 
 def _run_edit_test(arg_s, exp_filename=None,
                         exp_lineno=-1,
                         exp_contents=None,
                         exp_is_temp=None):
     ip = get_ipython()
     M = code.CodeMagics(ip)
     last_call = ['','']
     opts,args = M.parse_options(arg_s,'prxn:')
     filename, lineno, is_temp = M._find_edit_target(ip, args, opts, last_call)
         
     if exp_filename is not None:
         nt.assert_equal(exp_filename, filename)
     if exp_contents is not None:
         with io.open(filename, 'r', encoding='utf-8') as f:
             contents = f.read()
         nt.assert_equal(exp_contents, contents)
     if exp_lineno != -1:
         nt.assert_equal(exp_lineno, lineno)
     if exp_is_temp is not None:
         nt.assert_equal(exp_is_temp, is_temp)
 
 
 def test_edit_interactive():
     """%edit on interactively defined objects"""
     ip = get_ipython()
     n = ip.execution_count
     ip.run_cell(u"def foo(): return 1", store_history=True)
     
     try:
         _run_edit_test("foo")
     except code.InteractivelyDefined as e:
         nt.assert_equal(e.index, n)
     else:
         raise AssertionError("Should have raised InteractivelyDefined")
 
 
 def test_edit_cell():
     """%edit [cell id]"""
     ip = get_ipython()
     
     ip.run_cell(u"def foo(): return 1", store_history=True)
     
     # test
     _run_edit_test("1", exp_contents=ip.user_ns['In'][1], exp_is_temp=True)
 
 def test_bookmark():
     ip = get_ipython()
     ip.run_line_magic('bookmark', 'bmname')
     with tt.AssertPrints('bmname'):
         ip.run_line_magic('bookmark', '-l')
     ip.run_line_magic('bookmark', '-d bmname')
 
 def test_ls_magic():
     ip = get_ipython()
     json_formatter = ip.display_formatter.formatters['application/json']
     json_formatter.enabled = True
     lsmagic = ip.magic('lsmagic')
     with warnings.catch_warnings(record=True) as w:
         j = json_formatter(lsmagic)
     nt.assert_equal(sorted(j), ['cell', 'line'])
     nt.assert_equal(w, []) # no warnings
 
 def test_strip_initial_indent():
     def sii(s):
         lines = s.splitlines()
         return '\n'.join(code.strip_initial_indent(lines))
 
     nt.assert_equal(sii("  a = 1\nb = 2"), "a = 1\nb = 2")
     nt.assert_equal(sii("  a\n    b\nc"), "a\n  b\nc")
     nt.assert_equal(sii("a\n  b"), "a\n  b")

IPython/core/tests/test_magic_terminal.py

@@ -1,202 +1,197 b''
 """Tests for various magic functions specific to the terminal frontend.
 
 Needs to be run by nose (to make ipython session available).
 """
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 import sys
+from io import StringIO
 from unittest import TestCase
 
 import nose.tools as nt
 
 from IPython.testing import tools as tt
-from IPython.utils.py3compat import PY3
-
-if PY3:
-    from io import StringIO
-else:
-    from StringIO import StringIO
 
 #-----------------------------------------------------------------------------
 # Globals
 #-----------------------------------------------------------------------------
 ip = get_ipython()
 
 #-----------------------------------------------------------------------------
 # Test functions begin
 #-----------------------------------------------------------------------------
 
 def check_cpaste(code, should_fail=False):
     """Execute code via 'cpaste' and ensure it was executed, unless
     should_fail is set.
     """
     ip.user_ns['code_ran'] = False
 
     src = StringIO()
     if not hasattr(src, 'encoding'):
         # IPython expects stdin to have an encoding attribute
         src.encoding = None
     src.write(code)
     src.write('\n--\n')
     src.seek(0)
 
     stdin_save = sys.stdin
     sys.stdin = src
 
     try:
         context = tt.AssertPrints if should_fail else tt.AssertNotPrints
         with context("Traceback (most recent call last)"):
                 ip.magic('cpaste')
 
         if not should_fail:
             assert ip.user_ns['code_ran'], "%r failed" % code
     finally:
         sys.stdin = stdin_save
 
 def test_cpaste():
     """Test cpaste magic"""
 
     def runf():
         """Marker function: sets a flag when executed.
         """
         ip.user_ns['code_ran'] = True
         return 'runf' # return string so '+ runf()' doesn't result in success
 
     tests = {'pass': ["runf()",
                       "In [1]: runf()",
                       "In [1]: if 1:\n   ...:     runf()",
                       "> > > runf()",
                       ">>> runf()",
                       "   >>> runf()",
                       ],
 
              'fail': ["1 + runf()",
                       "++ runf()",
              ]}
 
     ip.user_ns['runf'] = runf
 
     for code in tests['pass']:
         check_cpaste(code)
 
     for code in tests['fail']:
         check_cpaste(code, should_fail=True)
 
 
 class PasteTestCase(TestCase):
     """Multiple tests for clipboard pasting"""
 
     def paste(self, txt, flags='-q'):
         """Paste input text, by default in quiet mode"""
         ip.hooks.clipboard_get = lambda : txt
         ip.magic('paste '+flags)
 
     def setUp(self):
         # Inject fake clipboard hook but save original so we can restore it later
         self.original_clip = ip.hooks.clipboard_get
 
     def tearDown(self): 
         # Restore original hook
         ip.hooks.clipboard_get = self.original_clip
        
     def test_paste(self):
         ip.user_ns.pop('x', None)
         self.paste('x = 1')
         nt.assert_equal(ip.user_ns['x'], 1)
         ip.user_ns.pop('x')
 
     def test_paste_pyprompt(self):
         ip.user_ns.pop('x', None)
         self.paste('>>> x=2')
         nt.assert_equal(ip.user_ns['x'], 2)
         ip.user_ns.pop('x')
 
     def test_paste_py_multi(self):
         self.paste("""
         >>> x = [1,2,3]
         >>> y = []
         >>> for i in x:
         ...     y.append(i**2)
         ... 
         """)
         nt.assert_equal(ip.user_ns['x'], [1,2,3])
         nt.assert_equal(ip.user_ns['y'], [1,4,9])
 
     def test_paste_py_multi_r(self):
         "Now, test that self.paste -r works"
         self.test_paste_py_multi()
         nt.assert_equal(ip.user_ns.pop('x'), [1,2,3])
         nt.assert_equal(ip.user_ns.pop('y'), [1,4,9])
         nt.assert_false('x' in ip.user_ns)
         ip.magic('paste -r')
         nt.assert_equal(ip.user_ns['x'], [1,2,3])
         nt.assert_equal(ip.user_ns['y'], [1,4,9])
 
     def test_paste_email(self):
         "Test pasting of email-quoted contents"
         self.paste("""\
         >> def foo(x):
         >>     return x + 1
         >> xx = foo(1.1)""")
         nt.assert_equal(ip.user_ns['xx'], 2.1)
 
     def test_paste_email2(self):
         "Email again; some programs add a space also at each quoting level"
         self.paste("""\
         > > def foo(x):
         > >     return x + 1
         > > yy = foo(2.1)     """)
         nt.assert_equal(ip.user_ns['yy'], 3.1)
 
     def test_paste_email_py(self):
         "Email quoting of interactive input"
         self.paste("""\
         >> >>> def f(x):
         >> ...   return x+1
         >> ... 
         >> >>> zz = f(2.5)      """)
         nt.assert_equal(ip.user_ns['zz'], 3.5)
 
     def test_paste_echo(self):
         "Also test self.paste echoing, by temporarily faking the writer"
         w = StringIO()
         writer = ip.write
         ip.write = w.write
         code = """
         a = 100
         b = 200"""
         try:
             self.paste(code,'')
             out = w.getvalue()
         finally:
             ip.write = writer
         nt.assert_equal(ip.user_ns['a'], 100)
         nt.assert_equal(ip.user_ns['b'], 200)
         nt.assert_equal(out, code+"\n## -- End pasted text --\n")
 
     def test_paste_leading_commas(self):
         "Test multiline strings with leading commas"
         tm = ip.magics_manager.registry['TerminalMagics']
         s = '''\
 a = """
 ,1,2,3
 """'''
         ip.user_ns.pop('foo', None)
         tm.store_or_execute(s, 'foo')
         nt.assert_in('foo', ip.user_ns)
 
 
     def test_paste_trailing_question(self):
         "Test pasting sources with trailing question marks"
         tm = ip.magics_manager.registry['TerminalMagics']
         s = '''\
 def funcfoo():
    if True: #am i true?
        return 'fooresult'
 '''
         ip.user_ns.pop('funcfoo', None)
         self.paste(s)
         nt.assert_equal(ip.user_ns['funcfoo'](), 'fooresult')

IPython/core/tests/test_oinspect.py

@@ -1,419 +1,409 b''
 """Tests for the object inspection functionality.
 """
 
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
 
 from inspect import Signature, Parameter
 import os
 import re
 import sys
 
 import nose.tools as nt
 
 from .. import oinspect
 from IPython.core.magic import (Magics, magics_class, line_magic,
                                 cell_magic, line_cell_magic,
                                 register_line_magic, register_cell_magic,
                                 register_line_cell_magic)
 from decorator import decorator
 from IPython import get_ipython
 from IPython.testing.decorators import skipif
 from IPython.testing.tools import AssertPrints, AssertNotPrints
 from IPython.utils.path import compress_user
 from IPython.utils import py3compat
 
 
 #-----------------------------------------------------------------------------
 # Globals and constants
 #-----------------------------------------------------------------------------
 
 inspector = oinspect.Inspector()
 ip = get_ipython()
 
 #-----------------------------------------------------------------------------
 # Local utilities
 #-----------------------------------------------------------------------------
 
 # WARNING: since this test checks the line number where a function is
 # defined, if any code is inserted above, the following line will need to be
 # updated.  Do NOT insert any whitespace between the next line and the function
 # definition below.
 THIS_LINE_NUMBER = 43  # Put here the actual number of this line
 
 from unittest import TestCase
 
 class Test(TestCase):
 
     def test_find_source_lines(self):
         self.assertEqual(oinspect.find_source_lines(Test.test_find_source_lines), 
                     THIS_LINE_NUMBER+6)
 
 
 # A couple of utilities to ensure these tests work the same from a source or a
 # binary install
 def pyfile(fname):
     return os.path.normcase(re.sub('.py[co]$', '.py', fname))
 
 
 def match_pyfiles(f1, f2):
     nt.assert_equal(pyfile(f1), pyfile(f2))
 
 
 def test_find_file():
     match_pyfiles(oinspect.find_file(test_find_file), os.path.abspath(__file__))
 
 
 def test_find_file_decorated1():
 
     @decorator
     def noop1(f):
         def wrapper(*a, **kw):
             return f(*a, **kw)
         return wrapper
 
     @noop1
     def f(x):
         "My docstring"
     
     match_pyfiles(oinspect.find_file(f), os.path.abspath(__file__))
     nt.assert_equal(f.__doc__, "My docstring")
 
 
 def test_find_file_decorated2():
 
     @decorator
     def noop2(f, *a, **kw):
         return f(*a, **kw)
 
     @noop2
     @noop2
     @noop2
     def f(x):
         "My docstring 2"
     
     match_pyfiles(oinspect.find_file(f), os.path.abspath(__file__))
     nt.assert_equal(f.__doc__, "My docstring 2")
     
 
 def test_find_file_magic():
     run = ip.find_line_magic('run')
     nt.assert_not_equal(oinspect.find_file(run), None)
 
 
 # A few generic objects we can then inspect in the tests below
 
 class Call(object):
     """This is the class docstring."""
 
     def __init__(self, x, y=1):
         """This is the constructor docstring."""
 
     def __call__(self, *a, **kw):
         """This is the call docstring."""
 
     def method(self, x, z=2):
         """Some method's docstring"""
 
 class HasSignature(object):
     """This is the class docstring."""
     __signature__ = Signature([Parameter('test', Parameter.POSITIONAL_OR_KEYWORD)])
 
     def __init__(self, *args):
         """This is the init docstring"""
 
 
 class SimpleClass(object):
     def method(self, x, z=2):
         """Some method's docstring"""
 
 
 class OldStyle:
     """An old-style class for testing."""
     pass
 
 
 def f(x, y=2, *a, **kw):
     """A simple function."""
 
 
 def g(y, z=3, *a, **kw):
     pass  # no docstring
 
 
 @register_line_magic
 def lmagic(line):
     "A line magic"
 
 
 @register_cell_magic
 def cmagic(line, cell):
     "A cell magic"
 
 
 @register_line_cell_magic
 def lcmagic(line, cell=None):
     "A line/cell magic"
 
 
 @magics_class
 class SimpleMagics(Magics):
     @line_magic
     def Clmagic(self, cline):
         "A class-based line magic"
         
     @cell_magic
     def Ccmagic(self, cline, ccell):
         "A class-based cell magic"
         
     @line_cell_magic
     def Clcmagic(self, cline, ccell=None):
         "A class-based line/cell magic"
 
 
 class Awkward(object):
     def __getattr__(self, name):
         raise Exception(name)
 
 class NoBoolCall:
     """
     callable with `__bool__` raising should still be inspect-able.
     """
 
     def __call__(self):
         """does nothing"""
         pass
 
     def __bool__(self):
         """just raise NotImplemented"""
         raise NotImplementedError('Must be implemented')
 
 
 class SerialLiar(object):
     """Attribute accesses always get another copy of the same class.
 
     unittest.mock.call does something similar, but it's not ideal for testing
     as the failure mode is to eat all your RAM. This gives up after 10k levels.
     """
     def __init__(self, max_fibbing_twig, lies_told=0):
         if lies_told > 10000:
             raise RuntimeError('Nose too long, honesty is the best policy')
         self.max_fibbing_twig = max_fibbing_twig
         self.lies_told = lies_told
         max_fibbing_twig[0] = max(max_fibbing_twig[0], lies_told)
 
     def __getattr__(self, item):
         return SerialLiar(self.max_fibbing_twig, self.lies_told + 1)
 
 #-----------------------------------------------------------------------------
 # Tests
 #-----------------------------------------------------------------------------
 
 def test_info():
     "Check that Inspector.info fills out various fields as expected."
     i = inspector.info(Call, oname='Call')
     nt.assert_equal(i['type_name'], 'type')
     expted_class = str(type(type))  # <class 'type'> (Python 3) or <type 'type'>
     nt.assert_equal(i['base_class'], expted_class)
     nt.assert_regex(i['string_form'], "<class 'IPython.core.tests.test_oinspect.Call'( at 0x[0-9a-f]{1,9})?>")
     fname = __file__
     if fname.endswith(".pyc"):
         fname = fname[:-1]
     # case-insensitive comparison needed on some filesystems
     # e.g. Windows:
     nt.assert_equal(i['file'].lower(), compress_user(fname).lower())
     nt.assert_equal(i['definition'], None)
     nt.assert_equal(i['docstring'], Call.__doc__)
     nt.assert_equal(i['source'], None)
     nt.assert_true(i['isclass'])
-    _self_py2 = '' if py3compat.PY3 else 'self, '
-    nt.assert_equal(i['init_definition'], "Call(%sx, y=1)" % _self_py2)
+    nt.assert_equal(i['init_definition'], "Call(x, y=1)")
     nt.assert_equal(i['init_docstring'], Call.__init__.__doc__)
 
     i = inspector.info(Call, detail_level=1)
     nt.assert_not_equal(i['source'], None)
     nt.assert_equal(i['docstring'], None)
 
     c = Call(1)
     c.__doc__ = "Modified instance docstring"
     i = inspector.info(c)
     nt.assert_equal(i['type_name'], 'Call')
     nt.assert_equal(i['docstring'], "Modified instance docstring")
     nt.assert_equal(i['class_docstring'], Call.__doc__)
     nt.assert_equal(i['init_docstring'], Call.__init__.__doc__)
     nt.assert_equal(i['call_docstring'], Call.__call__.__doc__)
 
-    # Test old-style classes, which for example may not have an __init__ method.
-    if not py3compat.PY3:
-        i = inspector.info(OldStyle)
-        nt.assert_equal(i['type_name'], 'classobj')
-
-        i = inspector.info(OldStyle())
-        nt.assert_equal(i['type_name'], 'instance')
-        nt.assert_equal(i['docstring'], OldStyle.__doc__)
-
 def test_class_signature():
     info = inspector.info(HasSignature, 'HasSignature')
     nt.assert_equal(info['init_definition'], "HasSignature(test)")
     nt.assert_equal(info['init_docstring'], HasSignature.__init__.__doc__)
 
 def test_info_awkward():
     # Just test that this doesn't throw an error.
     inspector.info(Awkward())
 
 def test_bool_raise():
     inspector.info(NoBoolCall())
 
 def test_info_serialliar():
     fib_tracker = [0]
     inspector.info(SerialLiar(fib_tracker))
 
     # Nested attribute access should be cut off at 100 levels deep to avoid
     # infinite loops: https://github.com/ipython/ipython/issues/9122
     nt.assert_less(fib_tracker[0], 9000)
 
 def test_calldef_none():
     # We should ignore __call__ for all of these.
     for obj in [f, SimpleClass().method, any, str.upper]:
         print(obj)
         i = inspector.info(obj)
         nt.assert_is(i['call_def'], None)
 
 def f_kwarg(pos, *, kwonly):
     pass
 
 def test_definition_kwonlyargs():
     i = inspector.info(f_kwarg, oname='f_kwarg')  # analysis:ignore
     nt.assert_equal(i['definition'], "f_kwarg(pos, *, kwonly)")
 
 def test_getdoc():
     class A(object):
         """standard docstring"""
         pass
 
     class B(object):
         """standard docstring"""
         def getdoc(self):
             return "custom docstring"
     
     class C(object):
         """standard docstring"""
         def getdoc(self):
             return None
     
     a = A()
     b = B()
     c = C()
     
     nt.assert_equal(oinspect.getdoc(a), "standard docstring")
     nt.assert_equal(oinspect.getdoc(b), "custom docstring")
     nt.assert_equal(oinspect.getdoc(c), "standard docstring")
 
 
 def test_empty_property_has_no_source():
     i = inspector.info(property(), detail_level=1)
     nt.assert_is(i['source'], None)
 
 
 def test_property_sources():
     import zlib
 
     class A(object):
         @property
         def foo(self):
             return 'bar'
 
         foo = foo.setter(lambda self, v: setattr(self, 'bar', v))
 
         id = property(id)
         compress = property(zlib.compress)
 
     i = inspector.info(A.foo, detail_level=1)
     nt.assert_in('def foo(self):', i['source'])
     nt.assert_in('lambda self, v:', i['source'])
 
     i = inspector.info(A.id, detail_level=1)
     nt.assert_in('fget = <function id>', i['source'])
 
     i = inspector.info(A.compress, detail_level=1)
     nt.assert_in('fget = <function zlib.compress>', i['source'])
 
 
 def test_property_docstring_is_in_info_for_detail_level_0():
     class A(object):
         @property
         def foobar(self):
             """This is `foobar` property."""
             pass
 
     ip.user_ns['a_obj'] = A()
     nt.assert_equal(
         'This is `foobar` property.',
         ip.object_inspect('a_obj.foobar', detail_level=0)['docstring'])
 
     ip.user_ns['a_cls'] = A
     nt.assert_equal(
         'This is `foobar` property.',
         ip.object_inspect('a_cls.foobar', detail_level=0)['docstring'])
 
 
 def test_pdef():
     # See gh-1914
     def foo(): pass
     inspector.pdef(foo, 'foo')
 
 
 def test_pinfo_nonascii():
     # See gh-1177
     from . import nonascii2
     ip.user_ns['nonascii2'] = nonascii2
     ip._inspect('pinfo', 'nonascii2', detail_level=1)
 
 
 def test_pinfo_docstring_no_source():
     """Docstring should be included with detail_level=1 if there is no source"""
     with AssertPrints('Docstring:'):
         ip._inspect('pinfo', 'str.format', detail_level=0)
     with AssertPrints('Docstring:'):
         ip._inspect('pinfo', 'str.format', detail_level=1)
 
 
 def test_pinfo_no_docstring_if_source():
     """Docstring should not be included with detail_level=1 if source is found"""
     def foo():
         """foo has a docstring"""
 
     ip.user_ns['foo'] = foo
 
     with AssertPrints('Docstring:'):
         ip._inspect('pinfo', 'foo', detail_level=0)
     with AssertPrints('Source:'):
         ip._inspect('pinfo', 'foo', detail_level=1)
     with AssertNotPrints('Docstring:'):
         ip._inspect('pinfo', 'foo', detail_level=1)
 
 
 def test_pinfo_magic():
     with AssertPrints('Docstring:'):
         ip._inspect('pinfo', 'lsmagic', detail_level=0)
 
     with AssertPrints('Source:'):
         ip._inspect('pinfo', 'lsmagic', detail_level=1)
 
 
 def test_init_colors():
     # ensure colors are not present in signature info
     info = inspector.info(HasSignature)
     init_def = info['init_definition']
     nt.assert_not_in('[0m', init_def)
 
 
 def test_builtin_init():
     info = inspector.info(list)
     init_def = info['init_definition']
     # Python < 3.4 can't get init definition from builtins,
     # but still exercise the inspection in case of error-raising bugs.
     if sys.version_info >= (3,4):
         nt.assert_is_not_none(init_def)
 

IPython/core/tests/test_ultratb.py

@@ -1,351 +1,347 b''
 # encoding: utf-8
 """Tests for IPython.core.ultratb
 """
 import io
 import sys
 import os.path
 from textwrap import dedent
 import traceback
 import unittest
 from unittest import mock
 
 from ..ultratb import ColorTB, VerboseTB, find_recursion
 
 
 from IPython.testing import tools as tt
 from IPython.testing.decorators import onlyif_unicode_paths
 from IPython.utils.syspathcontext import prepended_to_syspath
 from IPython.utils.tempdir import TemporaryDirectory
-from IPython.utils.py3compat import PY3
 
 ip = get_ipython()
 
 file_1 = """1
 2
 3
 def f():
   1/0
 """
 
 file_2 = """def f():
   1/0
 """
 
 class ChangedPyFileTest(unittest.TestCase):
     def test_changing_py_file(self):
         """Traceback produced if the line where the error occurred is missing?
         
         https://github.com/ipython/ipython/issues/1456
         """
         with TemporaryDirectory() as td:
             fname = os.path.join(td, "foo.py")
             with open(fname, "w") as f:
                 f.write(file_1)
             
             with prepended_to_syspath(td):
                 ip.run_cell("import foo")
             
             with tt.AssertPrints("ZeroDivisionError"):
                 ip.run_cell("foo.f()")
             
             # Make the file shorter, so the line of the error is missing.
             with open(fname, "w") as f:
                 f.write(file_2)
             
             # For some reason, this was failing on the *second* call after
             # changing the file, so we call f() twice.
             with tt.AssertNotPrints("Internal Python error", channel='stderr'):
                 with tt.AssertPrints("ZeroDivisionError"):
                     ip.run_cell("foo.f()")
                 with tt.AssertPrints("ZeroDivisionError"):
                     ip.run_cell("foo.f()")
 
 iso_8859_5_file = u'''# coding: iso-8859-5
 
 def fail():
     """дбИЖ"""
     1/0     # дбИЖ
 '''
 
 class NonAsciiTest(unittest.TestCase):
     @onlyif_unicode_paths
     def test_nonascii_path(self):
         # Non-ascii directory name as well.
         with TemporaryDirectory(suffix=u'é') as td:
             fname = os.path.join(td, u"fooé.py")
             with open(fname, "w") as f:
                 f.write(file_1)
             
             with prepended_to_syspath(td):
                 ip.run_cell("import foo")
             
             with tt.AssertPrints("ZeroDivisionError"):
                 ip.run_cell("foo.f()")
     
     def test_iso8859_5(self):
         with TemporaryDirectory() as td:
             fname = os.path.join(td, 'dfghjkl.py')
 
             with io.open(fname, 'w', encoding='iso-8859-5') as f:
                 f.write(iso_8859_5_file)
             
             with prepended_to_syspath(td):
                 ip.run_cell("from dfghjkl import fail")
             
             with tt.AssertPrints("ZeroDivisionError"):
                 with tt.AssertPrints(u'дбИЖ', suppress=False):
                     ip.run_cell('fail()')
     
     def test_nonascii_msg(self):
         cell = u"raise Exception('é')"
         expected = u"Exception('é')"
         ip.run_cell("%xmode plain")
         with tt.AssertPrints(expected):
             ip.run_cell(cell)
 
         ip.run_cell("%xmode verbose")
         with tt.AssertPrints(expected):
             ip.run_cell(cell)
 
         ip.run_cell("%xmode context")
         with tt.AssertPrints(expected):
             ip.run_cell(cell)
 
 
 class NestedGenExprTestCase(unittest.TestCase):
     """
     Regression test for the following issues:
     https://github.com/ipython/ipython/issues/8293
     https://github.com/ipython/ipython/issues/8205
     """
     def test_nested_genexpr(self):
         code = dedent(
             """\
             class SpecificException(Exception):
                 pass
 
             def foo(x):
                 raise SpecificException("Success!")
 
             sum(sum(foo(x) for _ in [0]) for x in [0])
             """
         )
         with tt.AssertPrints('SpecificException: Success!', suppress=False):
             ip.run_cell(code)
 
 
 indentationerror_file = """if True:
 zoon()
 """
 
 class IndentationErrorTest(unittest.TestCase):
     def test_indentationerror_shows_line(self):
         # See issue gh-2398
         with tt.AssertPrints("IndentationError"):
             with tt.AssertPrints("zoon()", suppress=False):
                 ip.run_cell(indentationerror_file)
         
         with TemporaryDirectory() as td:
             fname = os.path.join(td, "foo.py")
             with open(fname, "w") as f:
                 f.write(indentationerror_file)
             
             with tt.AssertPrints("IndentationError"):
                 with tt.AssertPrints("zoon()", suppress=False):
                     ip.magic('run %s' % fname)
 
 se_file_1 = """1
 2
 7/
 """
 
 se_file_2 = """7/
 """
 
 class SyntaxErrorTest(unittest.TestCase):
     def test_syntaxerror_without_lineno(self):
         with tt.AssertNotPrints("TypeError"):
             with tt.AssertPrints("line unknown"):
                 ip.run_cell("raise SyntaxError()")
 
     def test_changing_py_file(self):
         with TemporaryDirectory() as td:
             fname = os.path.join(td, "foo.py")
             with open(fname, 'w') as f:
                 f.write(se_file_1)
 
             with tt.AssertPrints(["7/", "SyntaxError"]):
                 ip.magic("run " + fname)
 
             # Modify the file
             with open(fname, 'w') as f:
                 f.write(se_file_2)
 
             # The SyntaxError should point to the correct line
             with tt.AssertPrints(["7/", "SyntaxError"]):
                 ip.magic("run " + fname)
 
     def test_non_syntaxerror(self):
         # SyntaxTB may be called with an error other than a SyntaxError
         # See e.g. gh-4361
         try:
             raise ValueError('QWERTY')
         except ValueError:
             with tt.AssertPrints('QWERTY'):
                 ip.showsyntaxerror()
 
 
 class Python3ChainedExceptionsTest(unittest.TestCase):
     DIRECT_CAUSE_ERROR_CODE = """
 try:
     x = 1 + 2
     print(not_defined_here)
 except Exception as e:
     x += 55
     x - 1
     y = {}
     raise KeyError('uh') from e
     """
 
     EXCEPTION_DURING_HANDLING_CODE = """
 try:
     x = 1 + 2
     print(not_defined_here)
 except Exception as e:
     x += 55
     x - 1
     y = {}
     raise KeyError('uh')
     """
 
     SUPPRESS_CHAINING_CODE = """
 try:
     1/0
 except Exception:
     raise ValueError("Yikes") from None
     """
 
     def test_direct_cause_error(self):
-        if PY3:
-            with tt.AssertPrints(["KeyError", "NameError", "direct cause"]):
-                ip.run_cell(self.DIRECT_CAUSE_ERROR_CODE)
+        with tt.AssertPrints(["KeyError", "NameError", "direct cause"]):
+            ip.run_cell(self.DIRECT_CAUSE_ERROR_CODE)
 
     def test_exception_during_handling_error(self):
-        if PY3:
-            with tt.AssertPrints(["KeyError", "NameError", "During handling"]):
-                ip.run_cell(self.EXCEPTION_DURING_HANDLING_CODE)
+        with tt.AssertPrints(["KeyError", "NameError", "During handling"]):
+            ip.run_cell(self.EXCEPTION_DURING_HANDLING_CODE)
 
     def test_suppress_exception_chaining(self):
-        if PY3:
-            with tt.AssertNotPrints("ZeroDivisionError"), \
-                    tt.AssertPrints("ValueError", suppress=False):
-                ip.run_cell(self.SUPPRESS_CHAINING_CODE)
+        with tt.AssertNotPrints("ZeroDivisionError"), \
+             tt.AssertPrints("ValueError", suppress=False):
+            ip.run_cell(self.SUPPRESS_CHAINING_CODE)
 
 
 class RecursionTest(unittest.TestCase):
     DEFINITIONS = """
 def non_recurs():
     1/0
 
 def r1():
     r1()
 
 def r3a():
     r3b()
 
 def r3b():
     r3c()
 
 def r3c():
     r3a()
 
 def r3o1():
     r3a()
 
 def r3o2():
     r3o1()
 """
     def setUp(self):
         ip.run_cell(self.DEFINITIONS)
 
     def test_no_recursion(self):
         with tt.AssertNotPrints("frames repeated"):
             ip.run_cell("non_recurs()")
 
     def test_recursion_one_frame(self):
         with tt.AssertPrints("1 frames repeated"):
             ip.run_cell("r1()")
 
     def test_recursion_three_frames(self):
         with tt.AssertPrints("3 frames repeated"):
             ip.run_cell("r3o2()")
 
     def test_find_recursion(self):
         captured = []
         def capture_exc(*args, **kwargs):
             captured.append(sys.exc_info())
         with mock.patch.object(ip, 'showtraceback', capture_exc):
             ip.run_cell("r3o2()")
 
         self.assertEqual(len(captured), 1)
         etype, evalue, tb = captured[0]
         self.assertIn("recursion", str(evalue))
 
         records = ip.InteractiveTB.get_records(tb, 3, ip.InteractiveTB.tb_offset)
         for r in records[:10]:
             print(r[1:4])
 
         # The outermost frames should be:
         # 0: the 'cell' that was running when the exception came up
         # 1: r3o2()
         # 2: r3o1()
         # 3: r3a()
         # Then repeating r3b, r3c, r3a
         last_unique, repeat_length = find_recursion(etype, evalue, records)
         self.assertEqual(last_unique, 2)
         self.assertEqual(repeat_length, 3)
 
 
 #----------------------------------------------------------------------------
 
 # module testing (minimal)
 def test_handlers():
     def spam(c, d_e):
         (d, e) = d_e
         x = c + d
         y = c * d
         foo(x, y)
 
     def foo(a, b, bar=1):
         eggs(a, b + bar)
 
     def eggs(f, g, z=globals()):
         h = f + g
         i = f - g
         return h / i
 
     buff = io.StringIO()
 
     buff.write('')
     buff.write('*** Before ***')
     try:
         buff.write(spam(1, (2, 3)))
     except:
         traceback.print_exc(file=buff)
 
     handler = ColorTB(ostream=buff)
     buff.write('*** ColorTB ***')
     try:
         buff.write(spam(1, (2, 3)))
     except:
         handler(*sys.exc_info())
     buff.write('')
 
     handler = VerboseTB(ostream=buff)
     buff.write('*** VerboseTB ***')
     try:
         buff.write(spam(1, (2, 3)))
     except:
         handler(*sys.exc_info())
     buff.write('')
 

IPython/extensions/autoreload.py

@@ -1,544 +1,532 b''
 """IPython extension to reload modules before executing user code.
 
 ``autoreload`` reloads modules automatically before entering the execution of
 code typed at the IPython prompt.
 
 This makes for example the following workflow possible:
 
 .. sourcecode:: ipython
 
    In [1]: %load_ext autoreload
 
    In [2]: %autoreload 2
 
    In [3]: from foo import some_function
 
    In [4]: some_function()
    Out[4]: 42
 
    In [5]: # open foo.py in an editor and change some_function to return 43
 
    In [6]: some_function()
    Out[6]: 43
 
 The module was reloaded without reloading it explicitly, and the object
 imported with ``from foo import ...`` was also updated.
 
 Usage
 =====
 
 The following magic commands are provided:
 
 ``%autoreload``
 
     Reload all modules (except those excluded by ``%aimport``)
     automatically now.
 
 ``%autoreload 0``
 
     Disable automatic reloading.
 
 ``%autoreload 1``
 
     Reload all modules imported with ``%aimport`` every time before
     executing the Python code typed.
 
 ``%autoreload 2``
 
     Reload all modules (except those excluded by ``%aimport``) every
     time before executing the Python code typed.
 
 ``%aimport``
 
     List modules which are to be automatically imported or not to be imported.
 
 ``%aimport foo``
 
     Import module 'foo' and mark it to be autoreloaded for ``%autoreload 1``
 
 ``%aimport foo, bar``
 
     Import modules 'foo', 'bar' and mark them to be autoreloaded for ``%autoreload 1``
 
 ``%aimport -foo``
 
     Mark module 'foo' to not be autoreloaded.
 
 Caveats
 =======
 
 Reloading Python modules in a reliable way is in general difficult,
 and unexpected things may occur. ``%autoreload`` tries to work around
 common pitfalls by replacing function code objects and parts of
 classes previously in the module with new versions. This makes the
 following things to work:
 
 - Functions and classes imported via 'from xxx import foo' are upgraded
   to new versions when 'xxx' is reloaded.
 
 - Methods and properties of classes are upgraded on reload, so that
   calling 'c.foo()' on an object 'c' created before the reload causes
   the new code for 'foo' to be executed.
 
 Some of the known remaining caveats are:
 
 - Replacing code objects does not always succeed: changing a @property
   in a class to an ordinary method or a method to a member variable
   can cause problems (but in old objects only).
 
 - Functions that are removed (eg. via monkey-patching) from a module
   before it is reloaded are not upgraded.
 
 - C extension modules cannot be reloaded, and so cannot be autoreloaded.
 """
 
 skip_doctest = True
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2000 Thomas Heller
 #  Copyright (C) 2008 Pauli Virtanen <[e-mail]>
 #  Copyright (C) 2012  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 #
 # This IPython module is written by Pauli Virtanen, based on the autoreload
 # code by Thomas Heller.
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 import os
 import sys
 import traceback
 import types
 import weakref
 from importlib import import_module
 
 try:
     # Reload is not defined by default in Python3.
     reload
 except NameError:
     from imp import reload
 
 from IPython.utils import openpy
 from IPython.utils.py3compat import PY3
 
 #------------------------------------------------------------------------------
 # Autoreload functionality
 #------------------------------------------------------------------------------
 
 class ModuleReloader(object):
     enabled = False
     """Whether this reloader is enabled"""
 
     check_all = True
     """Autoreload all modules, not just those listed in 'modules'"""
 
     def __init__(self):
         # Modules that failed to reload: {module: mtime-on-failed-reload, ...}
         self.failed = {}
         # Modules specially marked as autoreloadable.
         self.modules = {}
         # Modules specially marked as not autoreloadable.
         self.skip_modules = {}
         # (module-name, name) -> weakref, for replacing old code objects
         self.old_objects = {}
         # Module modification timestamps
         self.modules_mtimes = {}
 
         # Cache module modification times
         self.check(check_all=True, do_reload=False)
 
     def mark_module_skipped(self, module_name):
         """Skip reloading the named module in the future"""
         try:
             del self.modules[module_name]
         except KeyError:
             pass
         self.skip_modules[module_name] = True
 
     def mark_module_reloadable(self, module_name):
         """Reload the named module in the future (if it is imported)"""
         try:
             del self.skip_modules[module_name]
         except KeyError:
             pass
         self.modules[module_name] = True
 
     def aimport_module(self, module_name):
         """Import a module, and mark it reloadable
 
         Returns
         -------
         top_module : module
             The imported module if it is top-level, or the top-level
         top_name : module
             Name of top_module
 
         """
         self.mark_module_reloadable(module_name)
 
         import_module(module_name)
         top_name = module_name.split('.')[0]
         top_module = sys.modules[top_name]
         return top_module, top_name
 
     def filename_and_mtime(self, module):
         if not hasattr(module, '__file__') or module.__file__ is None:
             return None, None
 
         if getattr(module, '__name__', None) == '__main__':
             # we cannot reload(__main__)
             return None, None
 
         filename = module.__file__
         path, ext = os.path.splitext(filename)
 
         if ext.lower() == '.py':
             py_filename = filename
         else:
             try:
                 py_filename = openpy.source_from_cache(filename)
             except ValueError:
                 return None, None
 
         try:
             pymtime = os.stat(py_filename).st_mtime
         except OSError:
             return None, None
 
         return py_filename, pymtime
 
     def check(self, check_all=False, do_reload=True):
         """Check whether some modules need to be reloaded."""
 
         if not self.enabled and not check_all:
             return
 
         if check_all or self.check_all:
             modules = list(sys.modules.keys())
         else:
             modules = list(self.modules.keys())
 
         for modname in modules:
             m = sys.modules.get(modname, None)
 
             if modname in self.skip_modules:
                 continue
 
             py_filename, pymtime = self.filename_and_mtime(m)
             if py_filename is None:
                 continue
 
             try:
                 if pymtime <= self.modules_mtimes[modname]:
                     continue
             except KeyError:
                 self.modules_mtimes[modname] = pymtime
                 continue
             else:
                 if self.failed.get(py_filename, None) == pymtime:
                     continue
 
             self.modules_mtimes[modname] = pymtime
 
             # If we've reached this point, we should try to reload the module
             if do_reload:
                 try:
                     superreload(m, reload, self.old_objects)
                     if py_filename in self.failed:
                         del self.failed[py_filename]
                 except:
                     print("[autoreload of %s failed: %s]" % (
                             modname, traceback.format_exc(1)), file=sys.stderr)
                     self.failed[py_filename] = pymtime
 
 #------------------------------------------------------------------------------
 # superreload
 #------------------------------------------------------------------------------
 
-if PY3:
-    func_attrs = ['__code__', '__defaults__', '__doc__',
-                  '__closure__', '__globals__', '__dict__']
-else:
-    func_attrs = ['func_code', 'func_defaults', 'func_doc',
-                  'func_closure', 'func_globals', 'func_dict']
+
+func_attrs = ['__code__', '__defaults__', '__doc__',
+              '__closure__', '__globals__', '__dict__']
 
 
 def update_function(old, new):
     """Upgrade the code object of a function"""
     for name in func_attrs:
         try:
             setattr(old, name, getattr(new, name))
         except (AttributeError, TypeError):
             pass
 
 
 def update_class(old, new):
     """Replace stuff in the __dict__ of a class, and upgrade
     method code objects"""
     for key in list(old.__dict__.keys()):
         old_obj = getattr(old, key)
 
         try:
             new_obj = getattr(new, key)
         except AttributeError:
             # obsolete attribute: remove it
             try:
                 delattr(old, key)
             except (AttributeError, TypeError):
                 pass
             continue
 
         if update_generic(old_obj, new_obj): continue
 
         try:
             setattr(old, key, getattr(new, key))
         except (AttributeError, TypeError):
             pass # skip non-writable attributes
 
 
 def update_property(old, new):
     """Replace get/set/del functions of a property"""
     update_generic(old.fdel, new.fdel)
     update_generic(old.fget, new.fget)
     update_generic(old.fset, new.fset)
 
 
 def isinstance2(a, b, typ):
     return isinstance(a, typ) and isinstance(b, typ)
 
 
 UPDATE_RULES = [
     (lambda a, b: isinstance2(a, b, type),
      update_class),
     (lambda a, b: isinstance2(a, b, types.FunctionType),
      update_function),
     (lambda a, b: isinstance2(a, b, property),
      update_property),
 ]
-
-
-if PY3:
-    UPDATE_RULES.extend([(lambda a, b: isinstance2(a, b, types.MethodType),
-                          lambda a, b: update_function(a.__func__, b.__func__)),
-                        ])
-else:
-    UPDATE_RULES.extend([(lambda a, b: isinstance2(a, b, types.ClassType),
-                          update_class),
-                         (lambda a, b: isinstance2(a, b, types.MethodType),
-                          lambda a, b: update_function(a.__func__, b.__func__)),
-                        ])
+UPDATE_RULES.extend([(lambda a, b: isinstance2(a, b, types.MethodType),
+                      lambda a, b: update_function(a.__func__, b.__func__)),
+])
 
 
 def update_generic(a, b):
     for type_check, update in UPDATE_RULES:
         if type_check(a, b):
             update(a, b)
             return True
     return False
 
 
 class StrongRef(object):
     def __init__(self, obj):
         self.obj = obj
     def __call__(self):
         return self.obj
 
 
 def superreload(module, reload=reload, old_objects={}):
     """Enhanced version of the builtin reload function.
 
     superreload remembers objects previously in the module, and
 
     - upgrades the class dictionary of every old class in the module
     - upgrades the code object of every old function and method
     - clears the module's namespace before reloading
 
     """
 
     # collect old objects in the module
     for name, obj in list(module.__dict__.items()):
         if not hasattr(obj, '__module__') or obj.__module__ != module.__name__:
             continue
         key = (module.__name__, name)
         try:
             old_objects.setdefault(key, []).append(weakref.ref(obj))
         except TypeError:
             # weakref doesn't work for all types;
             # create strong references for 'important' cases
             if not PY3 and isinstance(obj, types.ClassType):
                 old_objects.setdefault(key, []).append(StrongRef(obj))
 
     # reload module
     try:
         # clear namespace first from old cruft
         old_dict = module.__dict__.copy()
         old_name = module.__name__
         module.__dict__.clear()
         module.__dict__['__name__'] = old_name
         module.__dict__['__loader__'] = old_dict['__loader__']
     except (TypeError, AttributeError, KeyError):
         pass
 
     try:
         module = reload(module)
     except:
         # restore module dictionary on failed reload
         module.__dict__.update(old_dict)
         raise
 
     # iterate over all objects and update functions & classes
     for name, new_obj in list(module.__dict__.items()):
         key = (module.__name__, name)
         if key not in old_objects: continue
 
         new_refs = []
         for old_ref in old_objects[key]:
             old_obj = old_ref()
             if old_obj is None: continue
             new_refs.append(old_ref)
             update_generic(old_obj, new_obj)
 
         if new_refs:
             old_objects[key] = new_refs
         else:
             del old_objects[key]
 
     return module
 
 #------------------------------------------------------------------------------
 # IPython connectivity
 #------------------------------------------------------------------------------
 
 from IPython.core.magic import Magics, magics_class, line_magic
 
 @magics_class
 class AutoreloadMagics(Magics):
     def __init__(self, *a, **kw):
         super(AutoreloadMagics, self).__init__(*a, **kw)
         self._reloader = ModuleReloader()
         self._reloader.check_all = False
         self.loaded_modules = set(sys.modules)
 
     @line_magic
     def autoreload(self, parameter_s=''):
         r"""%autoreload => Reload modules automatically
 
         %autoreload
         Reload all modules (except those excluded by %aimport) automatically
         now.
 
         %autoreload 0
         Disable automatic reloading.
 
         %autoreload 1
         Reload all modules imported with %aimport every time before executing
         the Python code typed.
 
         %autoreload 2
         Reload all modules (except those excluded by %aimport) every time
         before executing the Python code typed.
 
         Reloading Python modules in a reliable way is in general
         difficult, and unexpected things may occur. %autoreload tries to
         work around common pitfalls by replacing function code objects and
         parts of classes previously in the module with new versions. This
         makes the following things to work:
 
         - Functions and classes imported via 'from xxx import foo' are upgraded
           to new versions when 'xxx' is reloaded.
 
         - Methods and properties of classes are upgraded on reload, so that
           calling 'c.foo()' on an object 'c' created before the reload causes
           the new code for 'foo' to be executed.
 
         Some of the known remaining caveats are:
 
         - Replacing code objects does not always succeed: changing a @property
           in a class to an ordinary method or a method to a member variable
           can cause problems (but in old objects only).
 
         - Functions that are removed (eg. via monkey-patching) from a module
           before it is reloaded are not upgraded.
 
         - C extension modules cannot be reloaded, and so cannot be
           autoreloaded.
 
         """
         if parameter_s == '':
             self._reloader.check(True)
         elif parameter_s == '0':
             self._reloader.enabled = False
         elif parameter_s == '1':
             self._reloader.check_all = False
             self._reloader.enabled = True
         elif parameter_s == '2':
             self._reloader.check_all = True
             self._reloader.enabled = True
 
     @line_magic
     def aimport(self, parameter_s='', stream=None):
         """%aimport => Import modules for automatic reloading.
 
         %aimport
         List modules to automatically import and not to import.
 
         %aimport foo
         Import module 'foo' and mark it to be autoreloaded for %autoreload 1
 
         %aimport foo, bar
         Import modules 'foo', 'bar' and mark them to be autoreloaded for %autoreload 1
 
         %aimport -foo
         Mark module 'foo' to not be autoreloaded for %autoreload 1
         """
         modname = parameter_s
         if not modname:
             to_reload = sorted(self._reloader.modules.keys())
             to_skip = sorted(self._reloader.skip_modules.keys())
             if stream is None:
                 stream = sys.stdout
             if self._reloader.check_all:
                 stream.write("Modules to reload:\nall-except-skipped\n")
             else:
                 stream.write("Modules to reload:\n%s\n" % ' '.join(to_reload))
             stream.write("\nModules to skip:\n%s\n" % ' '.join(to_skip))
         elif modname.startswith('-'):
             modname = modname[1:]
             self._reloader.mark_module_skipped(modname)
         else:
             for _module in ([_.strip() for _ in modname.split(',')]):
                 top_module, top_name = self._reloader.aimport_module(_module)
 
                 # Inject module to user namespace
                 self.shell.push({top_name: top_module})
 
     def pre_run_cell(self):
         if self._reloader.enabled:
             try:
                 self._reloader.check()
             except:
                 pass
 
     def post_execute_hook(self):
         """Cache the modification times of any modules imported in this execution
         """
         newly_loaded_modules = set(sys.modules) - self.loaded_modules
         for modname in newly_loaded_modules:
             _, pymtime = self._reloader.filename_and_mtime(sys.modules[modname])
             if pymtime is not None:
                 self._reloader.modules_mtimes[modname] = pymtime
 
         self.loaded_modules.update(newly_loaded_modules)
 
 
 def load_ipython_extension(ip):
     """Load the extension in IPython."""
     auto_reload = AutoreloadMagics(ip)
     ip.register_magics(auto_reload)
     ip.events.register('pre_run_cell', auto_reload.pre_run_cell)
     ip.events.register('post_execute', auto_reload.post_execute_hook)

IPython/extensions/tests/test_autoreload.py

@@ -1,321 +1,316 b''
 """Tests for autoreload extension.
 """
 #-----------------------------------------------------------------------------
 #  Copyright (c) 2012 IPython Development Team.
 #
 #  Distributed under the terms of the Modified BSD License.
 #
 #  The full license is in the file COPYING.txt, distributed with this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 import os
 import sys
 import tempfile
 import shutil
 import random
 import time
+from io import StringIO
 
 import nose.tools as nt
 import IPython.testing.tools as tt
 
 from IPython.extensions.autoreload import AutoreloadMagics
 from IPython.core.events import EventManager, pre_run_cell
-from IPython.utils.py3compat import PY3
-
-if PY3:
-    from io import StringIO
-else:
-    from StringIO import StringIO
 
 #-----------------------------------------------------------------------------
 # Test fixture
 #-----------------------------------------------------------------------------
 
 noop = lambda *a, **kw: None
 
 class FakeShell(object):
 
     def __init__(self):
         self.ns = {}
         self.events = EventManager(self, {'pre_run_cell', pre_run_cell})
         self.auto_magics = AutoreloadMagics(shell=self)
         self.events.register('pre_run_cell', self.auto_magics.pre_run_cell)
 
     register_magics = set_hook = noop
 
     def run_code(self, code):
         self.events.trigger('pre_run_cell')
         exec(code, self.ns)
         self.auto_magics.post_execute_hook()
 
     def push(self, items):
         self.ns.update(items)
 
     def magic_autoreload(self, parameter):
         self.auto_magics.autoreload(parameter)
 
     def magic_aimport(self, parameter, stream=None):
         self.auto_magics.aimport(parameter, stream=stream)
         self.auto_magics.post_execute_hook()
 
 
 class Fixture(object):
     """Fixture for creating test module files"""
 
     test_dir = None
     old_sys_path = None
     filename_chars = "abcdefghijklmopqrstuvwxyz0123456789"
 
     def setUp(self):
         self.test_dir = tempfile.mkdtemp()
         self.old_sys_path = list(sys.path)
         sys.path.insert(0, self.test_dir)
         self.shell = FakeShell()
 
     def tearDown(self):
         shutil.rmtree(self.test_dir)
         sys.path = self.old_sys_path
 
         self.test_dir = None
         self.old_sys_path = None
         self.shell = None
 
     def get_module(self):
         module_name = "tmpmod_" + "".join(random.sample(self.filename_chars,20))
         if module_name in sys.modules:
             del sys.modules[module_name]
         file_name = os.path.join(self.test_dir, module_name + ".py")
         return module_name, file_name
 
     def write_file(self, filename, content):
         """
         Write a file, and force a timestamp difference of at least one second
 
         Notes
         -----
         Python's .pyc files record the timestamp of their compilation
         with a time resolution of one second.
 
         Therefore, we need to force a timestamp difference between .py
         and .pyc, without having the .py file be timestamped in the
         future, and without changing the timestamp of the .pyc file
         (because that is stored in the file).  The only reliable way
         to achieve this seems to be to sleep.
         """
 
         # Sleep one second + eps
         time.sleep(1.05)
 
         # Write
         f = open(filename, 'w')
         try:
             f.write(content)
         finally:
             f.close()
 
     def new_module(self, code):
         mod_name, mod_fn = self.get_module()
         f = open(mod_fn, 'w')
         try:
             f.write(code)
         finally:
             f.close()
         return mod_name, mod_fn
 
 #-----------------------------------------------------------------------------
 # Test automatic reloading
 #-----------------------------------------------------------------------------
 
 class TestAutoreload(Fixture):
     def _check_smoketest(self, use_aimport=True):
         """
         Functional test for the automatic reloader using either
         '%autoreload 1' or '%autoreload 2'
         """
 
         mod_name, mod_fn = self.new_module("""
 x = 9
 
 z = 123  # this item will be deleted
 
 def foo(y):
     return y + 3
 
 class Baz(object):
     def __init__(self, x):
         self.x = x
     def bar(self, y):
         return self.x + y
     @property
     def quux(self):
         return 42
     def zzz(self):
         '''This method will be deleted below'''
         return 99
 
 class Bar:    # old-style class: weakref doesn't work for it on Python < 2.7
     def foo(self):
         return 1
 """)
 
         #
         # Import module, and mark for reloading
         #
         if use_aimport:
             self.shell.magic_autoreload("1")
             self.shell.magic_aimport(mod_name)
             stream = StringIO()
             self.shell.magic_aimport("", stream=stream)
             nt.assert_in(("Modules to reload:\n%s" % mod_name), stream.getvalue())
 
             with nt.assert_raises(ImportError):
                 self.shell.magic_aimport("tmpmod_as318989e89ds")
         else:
             self.shell.magic_autoreload("2")
             self.shell.run_code("import %s" % mod_name)
             stream = StringIO()
             self.shell.magic_aimport("", stream=stream)
             nt.assert_true("Modules to reload:\nall-except-skipped" in
                            stream.getvalue())
         nt.assert_in(mod_name, self.shell.ns)
 
         mod = sys.modules[mod_name]
 
         #
         # Test module contents
         #
         old_foo = mod.foo
         old_obj = mod.Baz(9)
         old_obj2 = mod.Bar()
 
         def check_module_contents():
             nt.assert_equal(mod.x, 9)
             nt.assert_equal(mod.z, 123)
 
             nt.assert_equal(old_foo(0), 3)
             nt.assert_equal(mod.foo(0), 3)
 
             obj = mod.Baz(9)
             nt.assert_equal(old_obj.bar(1), 10)
             nt.assert_equal(obj.bar(1), 10)
             nt.assert_equal(obj.quux, 42)
             nt.assert_equal(obj.zzz(), 99)
 
             obj2 = mod.Bar()
             nt.assert_equal(old_obj2.foo(), 1)
             nt.assert_equal(obj2.foo(), 1)
 
         check_module_contents()
 
         #
         # Simulate a failed reload: no reload should occur and exactly
         # one error message should be printed
         #
         self.write_file(mod_fn, """
 a syntax error
 """)
 
         with tt.AssertPrints(('[autoreload of %s failed:' % mod_name), channel='stderr'):
             self.shell.run_code("pass") # trigger reload
         with tt.AssertNotPrints(('[autoreload of %s failed:' % mod_name), channel='stderr'):
             self.shell.run_code("pass") # trigger another reload
         check_module_contents()
 
         #
         # Rewrite module (this time reload should succeed)
         #
         self.write_file(mod_fn, """
 x = 10
 
 def foo(y):
     return y + 4
 
 class Baz(object):
     def __init__(self, x):
         self.x = x
     def bar(self, y):
         return self.x + y + 1
     @property
     def quux(self):
         return 43
 
 class Bar:    # old-style class
     def foo(self):
         return 2
 """)
 
         def check_module_contents():
             nt.assert_equal(mod.x, 10)
             nt.assert_false(hasattr(mod, 'z'))
 
             nt.assert_equal(old_foo(0), 4) # superreload magic!
             nt.assert_equal(mod.foo(0), 4)
 
             obj = mod.Baz(9)
             nt.assert_equal(old_obj.bar(1), 11) # superreload magic!
             nt.assert_equal(obj.bar(1), 11)
 
             nt.assert_equal(old_obj.quux, 43)
             nt.assert_equal(obj.quux, 43)
 
             nt.assert_false(hasattr(old_obj, 'zzz'))
             nt.assert_false(hasattr(obj, 'zzz'))
 
             obj2 = mod.Bar()
             nt.assert_equal(old_obj2.foo(), 2)
             nt.assert_equal(obj2.foo(), 2)
 
         self.shell.run_code("pass") # trigger reload
         check_module_contents()
 
         #
         # Another failure case: deleted file (shouldn't reload)
         #
         os.unlink(mod_fn)
 
         self.shell.run_code("pass") # trigger reload
         check_module_contents()
 
         #
         # Disable autoreload and rewrite module: no reload should occur
         #
         if use_aimport:
             self.shell.magic_aimport("-" + mod_name)
             stream = StringIO()
             self.shell.magic_aimport("", stream=stream)
             nt.assert_true(("Modules to skip:\n%s" % mod_name) in
                            stream.getvalue())
 
             # This should succeed, although no such module exists
             self.shell.magic_aimport("-tmpmod_as318989e89ds")
         else:
             self.shell.magic_autoreload("0")
 
         self.write_file(mod_fn, """
 x = -99
 """)
 
         self.shell.run_code("pass") # trigger reload
         self.shell.run_code("pass")
         check_module_contents()
 
         #
         # Re-enable autoreload: reload should now occur
         #
         if use_aimport:
             self.shell.magic_aimport(mod_name)
         else:
             self.shell.magic_autoreload("")
 
         self.shell.run_code("pass") # trigger reload
         nt.assert_equal(mod.x, -99)
 
     def test_smoketest_aimport(self):
         self._check_smoketest(use_aimport=True)
 
     def test_smoketest_autoreload(self):
         self._check_smoketest(use_aimport=False)

IPython/lib/latextools.py

@@ -1,201 +1,201 b''
 # -*- coding: utf-8 -*-
 """Tools for handling LaTeX."""
 
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
 from io import BytesIO, open
 import os
 import tempfile
 import shutil
 import subprocess
 from base64 import encodebytes
 
 from IPython.utils.process import find_cmd, FindCmdError
 from traitlets.config import get_config
 from traitlets.config.configurable import SingletonConfigurable
 from traitlets import List, Bool, Unicode
-from IPython.utils.py3compat import cast_unicode, cast_unicode_py2 as u, PY3
+from IPython.utils.py3compat import cast_unicode, cast_unicode_py2 as u
 
 
 class LaTeXTool(SingletonConfigurable):
     """An object to store configuration of the LaTeX tool."""
     def _config_default(self):
         return get_config()
     
     backends = List(
         Unicode(), ["matplotlib", "dvipng"],
         help="Preferred backend to draw LaTeX math equations. "
         "Backends in the list are checked one by one and the first "
         "usable one is used.  Note that `matplotlib` backend "
         "is usable only for inline style equations.  To draw  "
         "display style equations, `dvipng` backend must be specified. ",
         # It is a List instead of Enum, to make configuration more
         # flexible.  For example, to use matplotlib mainly but dvipng
         # for display style, the default ["matplotlib", "dvipng"] can
         # be used.  To NOT use dvipng so that other repr such as
         # unicode pretty printing is used, you can use ["matplotlib"].
         ).tag(config=True)
 
     use_breqn = Bool(
         True,
         help="Use breqn.sty to automatically break long equations. "
         "This configuration takes effect only for dvipng backend.",
         ).tag(config=True)
 
     packages = List(
         ['amsmath', 'amsthm', 'amssymb', 'bm'],
         help="A list of packages to use for dvipng backend. "
         "'breqn' will be automatically appended when use_breqn=True.",
         ).tag(config=True)
 
     preamble = Unicode(
         help="Additional preamble to use when generating LaTeX source "
         "for dvipng backend.",
         ).tag(config=True)
 
 
 def latex_to_png(s, encode=False, backend=None, wrap=False):
     """Render a LaTeX string to PNG.
 
     Parameters
     ----------
     s : str
         The raw string containing valid inline LaTeX.
     encode : bool, optional
         Should the PNG data base64 encoded to make it JSON'able.
     backend : {matplotlib, dvipng}
         Backend for producing PNG data.
     wrap : bool
         If true, Automatically wrap `s` as a LaTeX equation.
 
     None is returned when the backend cannot be used.
 
     """
     s = cast_unicode(s)
     allowed_backends = LaTeXTool.instance().backends
     if backend is None:
         backend = allowed_backends[0]
     if backend not in allowed_backends:
         return None
     if backend == 'matplotlib':
         f = latex_to_png_mpl
     elif backend == 'dvipng':
         f = latex_to_png_dvipng
     else:
         raise ValueError('No such backend {0}'.format(backend))
     bin_data = f(s, wrap)
     if encode and bin_data:
         bin_data = encodebytes(bin_data)
     return bin_data
 
 
 def latex_to_png_mpl(s, wrap):
     try:
         from matplotlib import mathtext
         from pyparsing import ParseFatalException
     except ImportError:
         return None
 
     # mpl mathtext doesn't support display math, force inline
     s = s.replace('$$', '$')
     if wrap:
         s = u'${0}$'.format(s)
 
     try:
         mt = mathtext.MathTextParser('bitmap')
         f = BytesIO()
         mt.to_png(f, s, fontsize=12)
         return f.getvalue()
     except (ValueError, RuntimeError, ParseFatalException):
         return None
 
 
 def latex_to_png_dvipng(s, wrap):
     try:
         find_cmd('latex')
         find_cmd('dvipng')
     except FindCmdError:
         return None
     try:
         workdir = tempfile.mkdtemp()
         tmpfile = os.path.join(workdir, "tmp.tex")
         dvifile = os.path.join(workdir, "tmp.dvi")
         outfile = os.path.join(workdir, "tmp.png")
 
         with open(tmpfile, "w", encoding='utf8') as f:
             f.writelines(genelatex(s, wrap))
 
         with open(os.devnull, 'wb') as devnull:
             subprocess.check_call(
                 ["latex", "-halt-on-error", "-interaction", "batchmode", tmpfile],
                 cwd=workdir, stdout=devnull, stderr=devnull)
 
             subprocess.check_call(
                 ["dvipng", "-T", "tight", "-x", "1500", "-z", "9",
                  "-bg", "transparent", "-o", outfile, dvifile], cwd=workdir,
                 stdout=devnull, stderr=devnull)
 
         with open(outfile, "rb") as f:
             return f.read()
     except subprocess.CalledProcessError:
         return None
     finally:
         shutil.rmtree(workdir)
 
 
 def kpsewhich(filename):
     """Invoke kpsewhich command with an argument `filename`."""
     try:
         find_cmd("kpsewhich")
         proc = subprocess.Popen(
             ["kpsewhich", filename],
             stdout=subprocess.PIPE, stderr=subprocess.PIPE)
         (stdout, stderr) = proc.communicate()
         return stdout.strip().decode('utf8', 'replace')
     except FindCmdError:
         pass
 
 
 def genelatex(body, wrap):
     """Generate LaTeX document for dvipng backend."""
     lt = LaTeXTool.instance()
     breqn = wrap and lt.use_breqn and kpsewhich("breqn.sty")
     yield u(r'\documentclass{article}')
     packages = lt.packages
     if breqn:
         packages = packages + ['breqn']
     for pack in packages:
         yield u(r'\usepackage{{{0}}}'.format(pack))
     yield u(r'\pagestyle{empty}')
     if lt.preamble:
         yield lt.preamble
     yield u(r'\begin{document}')
     if breqn:
         yield u(r'\begin{dmath*}')
         yield body
         yield u(r'\end{dmath*}')
     elif wrap:
         yield u'$${0}$$'.format(body)
     else:
         yield body
     yield u'\end{document}'
 
 
 _data_uri_template_png = u"""<img src="data:image/png;base64,%s" alt=%s />"""
 
 def latex_to_html(s, alt='image'):
     """Render LaTeX to HTML with embedded PNG data using data URIs.
 
     Parameters
     ----------
     s : str
         The raw string containing valid inline LateX.
     alt : str
         The alt text to use for the HTML.
     """
     base64_data = latex_to_png(s, encode=True).decode('ascii')
     if base64_data:
         return _data_uri_template_png  % (base64_data, alt)
 
 

IPython/lib/pretty.py

@@ -1,866 +1,858 b''
 # -*- coding: utf-8 -*-
 """
 Python advanced pretty printer.  This pretty printer is intended to
 replace the old `pprint` python module which does not allow developers
 to provide their own pretty print callbacks.
 
 This module is based on ruby's `prettyprint.rb` library by `Tanaka Akira`.
 
 
 Example Usage
 -------------
 
 To directly print the representation of an object use `pprint`::
 
     from pretty import pprint
     pprint(complex_object)
 
 To get a string of the output use `pretty`::
 
     from pretty import pretty
     string = pretty(complex_object)
 
 
 Extending
 ---------
 
 The pretty library allows developers to add pretty printing rules for their
 own objects.  This process is straightforward.  All you have to do is to
 add a `_repr_pretty_` method to your object and call the methods on the
 pretty printer passed::
 
     class MyObject(object):
 
         def _repr_pretty_(self, p, cycle):
             ...
 
 Here is an example implementation of a `_repr_pretty_` method for a list
 subclass::
 
     class MyList(list):
 
         def _repr_pretty_(self, p, cycle):
             if cycle:
                 p.text('MyList(...)')
             else:
                 with p.group(8, 'MyList([', '])'):
                     for idx, item in enumerate(self):
                         if idx:
                             p.text(',')
                             p.breakable()
                         p.pretty(item)
 
 The `cycle` parameter is `True` if pretty detected a cycle.  You *have* to
 react to that or the result is an infinite loop.  `p.text()` just adds
 non breaking text to the output, `p.breakable()` either adds a whitespace
 or breaks here.  If you pass it an argument it's used instead of the
 default space.  `p.pretty` prettyprints another object using the pretty print
 method.
 
 The first parameter to the `group` function specifies the extra indentation
 of the next line.  In this example the next item will either be on the same
 line (if the items are short enough) or aligned with the right edge of the
 opening bracket of `MyList`.
 
 If you just want to indent something you can use the group function
 without open / close parameters.  You can also use this code::
 
     with p.indent(2):
         ...
 
 Inheritance diagram:
 
 .. inheritance-diagram:: IPython.lib.pretty
    :parts: 3
 
 :copyright: 2007 by Armin Ronacher.
             Portions (c) 2009 by Robert Kern.
 :license: BSD License.
 """
 from contextlib import contextmanager
 import sys
 import types
 import re
 import datetime
 from collections import deque
 
-from IPython.utils.py3compat import PY3, PYPY, cast_unicode
+from IPython.utils.py3compat import PYPY, cast_unicode
 from IPython.utils.encoding import get_stream_enc
 
 from io import StringIO
 
 
 __all__ = ['pretty', 'pprint', 'PrettyPrinter', 'RepresentationPrinter',
     'for_type', 'for_type_by_name']
 
 
 MAX_SEQ_LENGTH = 1000
 _re_pattern_type = type(re.compile(''))
 
 def _safe_getattr(obj, attr, default=None):
     """Safe version of getattr.
     
     Same as getattr, but will return ``default`` on any Exception,
     rather than raising.
     """
     try:
         return getattr(obj, attr, default)
     except Exception:
         return default
 
-if PY3:
-    CUnicodeIO = StringIO
-else:
-    class CUnicodeIO(StringIO):
-        """StringIO that casts str to unicode on Python 2"""
-        def write(self, text):
-            return super(CUnicodeIO, self).write(
-                cast_unicode(text, encoding=get_stream_enc(sys.stdout)))
-
+CUnicodeIO = StringIO
 
 def pretty(obj, verbose=False, max_width=79, newline='\n', max_seq_length=MAX_SEQ_LENGTH):
     """
     Pretty print the object's representation.
     """
     stream = CUnicodeIO()
     printer = RepresentationPrinter(stream, verbose, max_width, newline, max_seq_length=max_seq_length)
     printer.pretty(obj)
     printer.flush()
     return stream.getvalue()
 
 
 def pprint(obj, verbose=False, max_width=79, newline='\n', max_seq_length=MAX_SEQ_LENGTH):
     """
     Like `pretty` but print to stdout.
     """
     printer = RepresentationPrinter(sys.stdout, verbose, max_width, newline, max_seq_length=max_seq_length)
     printer.pretty(obj)
     printer.flush()
     sys.stdout.write(newline)
     sys.stdout.flush()
 
 class _PrettyPrinterBase(object):
 
     @contextmanager
     def indent(self, indent):
         """with statement support for indenting/dedenting."""
         self.indentation += indent
         try:
             yield
         finally:
             self.indentation -= indent
 
     @contextmanager
     def group(self, indent=0, open='', close=''):
         """like begin_group / end_group but for the with statement."""
         self.begin_group(indent, open)
         try:
             yield
         finally:
             self.end_group(indent, close)
 
 class PrettyPrinter(_PrettyPrinterBase):
     """
     Baseclass for the `RepresentationPrinter` prettyprinter that is used to
     generate pretty reprs of objects.  Contrary to the `RepresentationPrinter`
     this printer knows nothing about the default pprinters or the `_repr_pretty_`
     callback method.
     """
 
     def __init__(self, output, max_width=79, newline='\n', max_seq_length=MAX_SEQ_LENGTH):
         self.output = output
         self.max_width = max_width
         self.newline = newline
         self.max_seq_length = max_seq_length
         self.output_width = 0
         self.buffer_width = 0
         self.buffer = deque()
 
         root_group = Group(0)
         self.group_stack = [root_group]
         self.group_queue = GroupQueue(root_group)
         self.indentation = 0
 
     def _break_outer_groups(self):
         while self.max_width < self.output_width + self.buffer_width:
             group = self.group_queue.deq()
             if not group:
                 return
             while group.breakables:
                 x = self.buffer.popleft()
                 self.output_width = x.output(self.output, self.output_width)
                 self.buffer_width -= x.width
             while self.buffer and isinstance(self.buffer[0], Text):
                 x = self.buffer.popleft()
                 self.output_width = x.output(self.output, self.output_width)
                 self.buffer_width -= x.width
 
     def text(self, obj):
         """Add literal text to the output."""
         width = len(obj)
         if self.buffer:
             text = self.buffer[-1]
             if not isinstance(text, Text):
                 text = Text()
                 self.buffer.append(text)
             text.add(obj, width)
             self.buffer_width += width
             self._break_outer_groups()
         else:
             self.output.write(obj)
             self.output_width += width
 
     def breakable(self, sep=' '):
         """
         Add a breakable separator to the output.  This does not mean that it
         will automatically break here.  If no breaking on this position takes
         place the `sep` is inserted which default to one space.
         """
         width = len(sep)
         group = self.group_stack[-1]
         if group.want_break:
             self.flush()
             self.output.write(self.newline)
             self.output.write(' ' * self.indentation)
             self.output_width = self.indentation
             self.buffer_width = 0
         else:
             self.buffer.append(Breakable(sep, width, self))
             self.buffer_width += width
             self._break_outer_groups()
             
     def break_(self):
         """
         Explicitly insert a newline into the output, maintaining correct indentation.
         """
         self.flush()
         self.output.write(self.newline)
         self.output.write(' ' * self.indentation)
         self.output_width = self.indentation
         self.buffer_width = 0
         
 
     def begin_group(self, indent=0, open=''):
         """
         Begin a group.  If you want support for python < 2.5 which doesn't has
         the with statement this is the preferred way:
 
             p.begin_group(1, '{')
             ...
             p.end_group(1, '}')
 
         The python 2.5 expression would be this:
 
             with p.group(1, '{', '}'):
                 ...
 
         The first parameter specifies the indentation for the next line (usually
         the width of the opening text), the second the opening text.  All
         parameters are optional.
         """
         if open:
             self.text(open)
         group = Group(self.group_stack[-1].depth + 1)
         self.group_stack.append(group)
         self.group_queue.enq(group)
         self.indentation += indent
     
     def _enumerate(self, seq):
         """like enumerate, but with an upper limit on the number of items"""
         for idx, x in enumerate(seq):
             if self.max_seq_length and idx >= self.max_seq_length:
                 self.text(',')
                 self.breakable()
                 self.text('...')
                 return
             yield idx, x
     
     def end_group(self, dedent=0, close=''):
         """End a group. See `begin_group` for more details."""
         self.indentation -= dedent
         group = self.group_stack.pop()
         if not group.breakables:
             self.group_queue.remove(group)
         if close:
             self.text(close)
 
     def flush(self):
         """Flush data that is left in the buffer."""
         for data in self.buffer:
             self.output_width += data.output(self.output, self.output_width)
         self.buffer.clear()
         self.buffer_width = 0
 
 
 def _get_mro(obj_class):
     """ Get a reasonable method resolution order of a class and its superclasses
     for both old-style and new-style classes.
     """
     if not hasattr(obj_class, '__mro__'):
         # Old-style class. Mix in object to make a fake new-style class.
         try:
             obj_class = type(obj_class.__name__, (obj_class, object), {})
         except TypeError:
             # Old-style extension type that does not descend from object.
             # FIXME: try to construct a more thorough MRO.
             mro = [obj_class]
         else:
             mro = obj_class.__mro__[1:-1]
     else:
         mro = obj_class.__mro__
     return mro
 
 
 class RepresentationPrinter(PrettyPrinter):
     """
     Special pretty printer that has a `pretty` method that calls the pretty
     printer for a python object.
 
     This class stores processing data on `self` so you must *never* use
     this class in a threaded environment.  Always lock it or reinstanciate
     it.
 
     Instances also have a verbose flag callbacks can access to control their
     output.  For example the default instance repr prints all attributes and
     methods that are not prefixed by an underscore if the printer is in
     verbose mode.
     """
 
     def __init__(self, output, verbose=False, max_width=79, newline='\n',
         singleton_pprinters=None, type_pprinters=None, deferred_pprinters=None,
         max_seq_length=MAX_SEQ_LENGTH):
 
         PrettyPrinter.__init__(self, output, max_width, newline, max_seq_length=max_seq_length)
         self.verbose = verbose
         self.stack = []
         if singleton_pprinters is None:
             singleton_pprinters = _singleton_pprinters.copy()
         self.singleton_pprinters = singleton_pprinters
         if type_pprinters is None:
             type_pprinters = _type_pprinters.copy()
         self.type_pprinters = type_pprinters
         if deferred_pprinters is None:
             deferred_pprinters = _deferred_type_pprinters.copy()
         self.deferred_pprinters = deferred_pprinters
 
     def pretty(self, obj):
         """Pretty print the given object."""
         obj_id = id(obj)
         cycle = obj_id in self.stack
         self.stack.append(obj_id)
         self.begin_group()
         try:
             obj_class = _safe_getattr(obj, '__class__', None) or type(obj)
             # First try to find registered singleton printers for the type.
             try:
                 printer = self.singleton_pprinters[obj_id]
             except (TypeError, KeyError):
                 pass
             else:
                 return printer(obj, self, cycle)
             # Next walk the mro and check for either:
             #   1) a registered printer
             #   2) a _repr_pretty_ method
             for cls in _get_mro(obj_class):
                 if cls in self.type_pprinters:
                     # printer registered in self.type_pprinters
                     return self.type_pprinters[cls](obj, self, cycle)
                 else:
                     # deferred printer
                     printer = self._in_deferred_types(cls)
                     if printer is not None:
                         return printer(obj, self, cycle)
                     else:
                         # Finally look for special method names.
                         # Some objects automatically create any requested
                         # attribute. Try to ignore most of them by checking for
                         # callability.
                         if '_repr_pretty_' in cls.__dict__:
                             meth = cls._repr_pretty_
                             if callable(meth):
                                 return meth(obj, self, cycle)
             return _default_pprint(obj, self, cycle)
         finally:
             self.end_group()
             self.stack.pop()
 
     def _in_deferred_types(self, cls):
         """
         Check if the given class is specified in the deferred type registry.
 
         Returns the printer from the registry if it exists, and None if the
         class is not in the registry. Successful matches will be moved to the
         regular type registry for future use.
         """
         mod = _safe_getattr(cls, '__module__', None)
         name = _safe_getattr(cls, '__name__', None)
         key = (mod, name)
         printer = None
         if key in self.deferred_pprinters:
             # Move the printer over to the regular registry.
             printer = self.deferred_pprinters.pop(key)
             self.type_pprinters[cls] = printer
         return printer
 
 
 class Printable(object):
 
     def output(self, stream, output_width):
         return output_width
 
 
 class Text(Printable):
 
     def __init__(self):
         self.objs = []
         self.width = 0
 
     def output(self, stream, output_width):
         for obj in self.objs:
             stream.write(obj)
         return output_width + self.width
 
     def add(self, obj, width):
         self.objs.append(obj)
         self.width += width
 
 
 class Breakable(Printable):
 
     def __init__(self, seq, width, pretty):
         self.obj = seq
         self.width = width
         self.pretty = pretty
         self.indentation = pretty.indentation
         self.group = pretty.group_stack[-1]
         self.group.breakables.append(self)
 
     def output(self, stream, output_width):
         self.group.breakables.popleft()
         if self.group.want_break:
             stream.write(self.pretty.newline)
             stream.write(' ' * self.indentation)
             return self.indentation
         if not self.group.breakables:
             self.pretty.group_queue.remove(self.group)
         stream.write(self.obj)
         return output_width + self.width
 
 
 class Group(Printable):
 
     def __init__(self, depth):
         self.depth = depth
         self.breakables = deque()
         self.want_break = False
 
 
 class GroupQueue(object):
 
     def __init__(self, *groups):
         self.queue = []
         for group in groups:
             self.enq(group)
 
     def enq(self, group):
         depth = group.depth
         while depth > len(self.queue) - 1:
             self.queue.append([])
         self.queue[depth].append(group)
 
     def deq(self):
         for stack in self.queue:
             for idx, group in enumerate(reversed(stack)):
                 if group.breakables:
                     del stack[idx]
                     group.want_break = True
                     return group
             for group in stack:
                 group.want_break = True
             del stack[:]
 
     def remove(self, group):
         try:
             self.queue[group.depth].remove(group)
         except ValueError:
             pass
 
 try:
     _baseclass_reprs = (object.__repr__, types.InstanceType.__repr__)
 except AttributeError:  # Python 3
     _baseclass_reprs = (object.__repr__,)
 
 
 def _default_pprint(obj, p, cycle):
     """
     The default print function.  Used if an object does not provide one and
     it's none of the builtin objects.
     """
     klass = _safe_getattr(obj, '__class__', None) or type(obj)
     if _safe_getattr(klass, '__repr__', None) not in _baseclass_reprs:
         # A user-provided repr. Find newlines and replace them with p.break_()
         _repr_pprint(obj, p, cycle)
         return
     p.begin_group(1, '<')
     p.pretty(klass)
     p.text(' at 0x%x' % id(obj))
     if cycle:
         p.text(' ...')
     elif p.verbose:
         first = True
         for key in dir(obj):
             if not key.startswith('_'):
                 try:
                     value = getattr(obj, key)
                 except AttributeError:
                     continue
                 if isinstance(value, types.MethodType):
                     continue
                 if not first:
                     p.text(',')
                 p.breakable()
                 p.text(key)
                 p.text('=')
                 step = len(key) + 1
                 p.indentation += step
                 p.pretty(value)
                 p.indentation -= step
                 first = False
     p.end_group(1, '>')
 
 
 def _seq_pprinter_factory(start, end, basetype):
     """
     Factory that returns a pprint function useful for sequences.  Used by
     the default pprint for tuples, dicts, and lists.
     """
     def inner(obj, p, cycle):
         typ = type(obj)
         if basetype is not None and typ is not basetype and typ.__repr__ != basetype.__repr__:
             # If the subclass provides its own repr, use it instead.
             return p.text(typ.__repr__(obj))
 
         if cycle:
             return p.text(start + '...' + end)
         step = len(start)
         p.begin_group(step, start)
         for idx, x in p._enumerate(obj):
             if idx:
                 p.text(',')
                 p.breakable()
             p.pretty(x)
         if len(obj) == 1 and type(obj) is tuple:
             # Special case for 1-item tuples.
             p.text(',')
         p.end_group(step, end)
     return inner
 
 
 def _set_pprinter_factory(start, end, basetype):
     """
     Factory that returns a pprint function useful for sets and frozensets.
     """
     def inner(obj, p, cycle):
         typ = type(obj)
         if basetype is not None and typ is not basetype and typ.__repr__ != basetype.__repr__:
             # If the subclass provides its own repr, use it instead.
             return p.text(typ.__repr__(obj))
 
         if cycle:
             return p.text(start + '...' + end)
         if len(obj) == 0:
             # Special case.
             p.text(basetype.__name__ + '()')
         else:
             step = len(start)
             p.begin_group(step, start)
             # Like dictionary keys, we will try to sort the items if there aren't too many
             items = obj
             if not (p.max_seq_length and len(obj) >= p.max_seq_length):
                 try:
                     items = sorted(obj)
                 except Exception:
                     # Sometimes the items don't sort.
                     pass
             for idx, x in p._enumerate(items):
                 if idx:
                     p.text(',')
                     p.breakable()
                 p.pretty(x)
             p.end_group(step, end)
     return inner
 
 
 def _dict_pprinter_factory(start, end, basetype=None):
     """
     Factory that returns a pprint function used by the default pprint of
     dicts and dict proxies.
     """
     def inner(obj, p, cycle):
         typ = type(obj)
         if basetype is not None and typ is not basetype and typ.__repr__ != basetype.__repr__:
             # If the subclass provides its own repr, use it instead.
             return p.text(typ.__repr__(obj))
 
         if cycle:
             return p.text('{...}')
         step = len(start)
         p.begin_group(step, start)
         keys = obj.keys()
         # if dict isn't large enough to be truncated, sort keys before displaying
         if not (p.max_seq_length and len(obj) >= p.max_seq_length):
             try:
                 keys = sorted(keys)
             except Exception:
                 # Sometimes the keys don't sort.
                 pass
         for idx, key in p._enumerate(keys):
             if idx:
                 p.text(',')
                 p.breakable()
             p.pretty(key)
             p.text(': ')
             p.pretty(obj[key])
         p.end_group(step, end)
     return inner
 
 
 def _super_pprint(obj, p, cycle):
     """The pprint for the super type."""
     p.begin_group(8, '<super: ')
     p.pretty(obj.__thisclass__)
     p.text(',')
     p.breakable()
     if PYPY: # In PyPy, super() objects don't have __self__ attributes
         dself = obj.__repr__.__self__
         p.pretty(None if dself is obj else dself)
     else:
         p.pretty(obj.__self__)
     p.end_group(8, '>')
 
 
 def _re_pattern_pprint(obj, p, cycle):
     """The pprint function for regular expression patterns."""
     p.text('re.compile(')
     pattern = repr(obj.pattern)
     if pattern[:1] in 'uU':
         pattern = pattern[1:]
         prefix = 'ur'
     else:
         prefix = 'r'
     pattern = prefix + pattern.replace('\\\\', '\\')
     p.text(pattern)
     if obj.flags:
         p.text(',')
         p.breakable()
         done_one = False
         for flag in ('TEMPLATE', 'IGNORECASE', 'LOCALE', 'MULTILINE', 'DOTALL',
             'UNICODE', 'VERBOSE', 'DEBUG'):
             if obj.flags & getattr(re, flag):
                 if done_one:
                     p.text('|')
                 p.text('re.' + flag)
                 done_one = True
     p.text(')')
 
 
 def _type_pprint(obj, p, cycle):
     """The pprint for classes and types."""
     # Heap allocated types might not have the module attribute,
     # and others may set it to None.
 
     # Checks for a __repr__ override in the metaclass. Can't compare the
     # type(obj).__repr__ directly because in PyPy the representation function
     # inherited from type isn't the same type.__repr__
     if [m for m in _get_mro(type(obj)) if "__repr__" in vars(m)][:1] != [type]:
         _repr_pprint(obj, p, cycle)
         return
 
     mod = _safe_getattr(obj, '__module__', None)
     try:
         name = obj.__qualname__
         if not isinstance(name, str):
             # This can happen if the type implements __qualname__ as a property
             # or other descriptor in Python 2.
             raise Exception("Try __name__")
     except Exception:
         name = obj.__name__
         if not isinstance(name, str):
             name = '<unknown type>'
 
     if mod in (None, '__builtin__', 'builtins', 'exceptions'):
         p.text(name)
     else:
         p.text(mod + '.' + name)
 
 
 def _repr_pprint(obj, p, cycle):
     """A pprint that just redirects to the normal repr function."""
     # Find newlines and replace them with p.break_()
     output = repr(obj)
     for idx,output_line in enumerate(output.splitlines()):
         if idx:
             p.break_()
         p.text(output_line)
 
 
 def _function_pprint(obj, p, cycle):
     """Base pprint for all functions and builtin functions."""
     name = _safe_getattr(obj, '__qualname__', obj.__name__)
     mod = obj.__module__
     if mod and mod not in ('__builtin__', 'builtins', 'exceptions'):
         name = mod + '.' + name
     p.text('<function %s>' % name)
 
 
 def _exception_pprint(obj, p, cycle):
     """Base pprint for all exceptions."""
     name = getattr(obj.__class__, '__qualname__', obj.__class__.__name__)
     if obj.__class__.__module__ not in ('exceptions', 'builtins'):
         name = '%s.%s' % (obj.__class__.__module__, name)
     step = len(name) + 1
     p.begin_group(step, name + '(')
     for idx, arg in enumerate(getattr(obj, 'args', ())):
         if idx:
             p.text(',')
             p.breakable()
         p.pretty(arg)
     p.end_group(step, ')')
 
 
 #: the exception base
 try:
     _exception_base = BaseException
 except NameError:
     _exception_base = Exception
 
 
 #: printers for builtin types
 _type_pprinters = {
     int:                        _repr_pprint,
     float:                      _repr_pprint,
     str:                        _repr_pprint,
     tuple:                      _seq_pprinter_factory('(', ')', tuple),
     list:                       _seq_pprinter_factory('[', ']', list),
     dict:                       _dict_pprinter_factory('{', '}', dict),
     
     set:                        _set_pprinter_factory('{', '}', set),
     frozenset:                  _set_pprinter_factory('frozenset({', '})', frozenset),
     super:                      _super_pprint,
     _re_pattern_type:           _re_pattern_pprint,
     type:                       _type_pprint,
     types.FunctionType:         _function_pprint,
     types.BuiltinFunctionType:  _function_pprint,
     types.MethodType:           _repr_pprint,
     
     datetime.datetime:          _repr_pprint,
     datetime.timedelta:         _repr_pprint,
     _exception_base:            _exception_pprint
 }
 
 try:
     # In PyPy, types.DictProxyType is dict, setting the dictproxy printer
     # using dict.setdefault avoids overwritting the dict printer
     _type_pprinters.setdefault(types.DictProxyType,
                                _dict_pprinter_factory('dict_proxy({', '})'))
     _type_pprinters[types.ClassType] = _type_pprint
     _type_pprinters[types.SliceType] = _repr_pprint
 except AttributeError: # Python 3
     _type_pprinters[types.MappingProxyType] = \
         _dict_pprinter_factory('mappingproxy({', '})')
     _type_pprinters[slice] = _repr_pprint
     
 try:
     _type_pprinters[long] = _repr_pprint
     _type_pprinters[unicode] = _repr_pprint
 except NameError:
     _type_pprinters[range] = _repr_pprint
     _type_pprinters[bytes] = _repr_pprint
 
 #: printers for types specified by name
 _deferred_type_pprinters = {
 }
 
 def for_type(typ, func):
     """
     Add a pretty printer for a given type.
     """
     oldfunc = _type_pprinters.get(typ, None)
     if func is not None:
         # To support easy restoration of old pprinters, we need to ignore Nones.
         _type_pprinters[typ] = func
     return oldfunc
 
 def for_type_by_name(type_module, type_name, func):
     """
     Add a pretty printer for a type specified by the module and name of a type
     rather than the type object itself.
     """
     key = (type_module, type_name)
     oldfunc = _deferred_type_pprinters.get(key, None)
     if func is not None:
         # To support easy restoration of old pprinters, we need to ignore Nones.
         _deferred_type_pprinters[key] = func
     return oldfunc
 
 
 #: printers for the default singletons
 _singleton_pprinters = dict.fromkeys(map(id, [None, True, False, Ellipsis,
                                       NotImplemented]), _repr_pprint)
 
 
 def _defaultdict_pprint(obj, p, cycle):
     name = obj.__class__.__name__
     with p.group(len(name) + 1, name + '(', ')'):
         if cycle:
             p.text('...')
         else:
             p.pretty(obj.default_factory)
             p.text(',')
             p.breakable()
             p.pretty(dict(obj))
 
 def _ordereddict_pprint(obj, p, cycle):
     name = obj.__class__.__name__
     with p.group(len(name) + 1, name + '(', ')'):
         if cycle:
             p.text('...')
         elif len(obj):
             p.pretty(list(obj.items()))
 
 def _deque_pprint(obj, p, cycle):
     name = obj.__class__.__name__
     with p.group(len(name) + 1, name + '(', ')'):
         if cycle:
             p.text('...')
         else:
             p.pretty(list(obj))
 
 
 def _counter_pprint(obj, p, cycle):
     name = obj.__class__.__name__
     with p.group(len(name) + 1, name + '(', ')'):
         if cycle:
             p.text('...')
         elif len(obj):
             p.pretty(dict(obj))
 
 for_type_by_name('collections', 'defaultdict', _defaultdict_pprint)
 for_type_by_name('collections', 'OrderedDict', _ordereddict_pprint)
 for_type_by_name('collections', 'deque', _deque_pprint)
 for_type_by_name('collections', 'Counter', _counter_pprint)
 
 if __name__ == '__main__':
     from random import randrange
     class Foo(object):
         def __init__(self):
             self.foo = 1
             self.bar = re.compile(r'\s+')
             self.blub = dict.fromkeys(range(30), randrange(1, 40))
             self.hehe = 23424.234234
             self.list = ["blub", "blah", self]
 
         def get_foo(self):
             print("foo")
 
     pprint(Foo(), verbose=True)

IPython/sphinxext/ipython_directive.py

@@ -1,1183 +1,1177 b''
 # -*- coding: utf-8 -*-
 """
 Sphinx directive to support embedded IPython code.
 
 This directive allows pasting of entire interactive IPython sessions, prompts
 and all, and their code will actually get re-executed at doc build time, with
 all prompts renumbered sequentially. It also allows you to input code as a pure
 python input by giving the argument python to the directive. The output looks
 like an interactive ipython section.
 
 To enable this directive, simply list it in your Sphinx ``conf.py`` file
 (making sure the directory where you placed it is visible to sphinx, as is
 needed for all Sphinx directives). For example, to enable syntax highlighting
 and the IPython directive::
 
     extensions = ['IPython.sphinxext.ipython_console_highlighting',
                   'IPython.sphinxext.ipython_directive']
 
 The IPython directive outputs code-blocks with the language 'ipython'. So
 if you do not have the syntax highlighting extension enabled as well, then
 all rendered code-blocks will be uncolored. By default this directive assumes
 that your prompts are unchanged IPython ones, but this can be customized.
 The configurable options that can be placed in conf.py are:
 
 ipython_savefig_dir:
     The directory in which to save the figures. This is relative to the
     Sphinx source directory. The default is `html_static_path`.
 ipython_rgxin:
     The compiled regular expression to denote the start of IPython input
     lines. The default is re.compile('In \[(\d+)\]:\s?(.*)\s*'). You
     shouldn't need to change this.
 ipython_rgxout:
     The compiled regular expression to denote the start of IPython output
     lines. The default is re.compile('Out\[(\d+)\]:\s?(.*)\s*'). You
     shouldn't need to change this.
 ipython_promptin:
     The string to represent the IPython input prompt in the generated ReST.
     The default is 'In [%d]:'. This expects that the line numbers are used
     in the prompt.
 ipython_promptout:
     The string to represent the IPython prompt in the generated ReST. The
     default is 'Out [%d]:'. This expects that the line numbers are used
     in the prompt.
 ipython_mplbackend:
     The string which specifies if the embedded Sphinx shell should import
     Matplotlib and set the backend. The value specifies a backend that is
     passed to `matplotlib.use()` before any lines in `ipython_execlines` are
     executed. If not specified in conf.py, then the default value of 'agg' is
     used. To use the IPython directive without matplotlib as a dependency, set
     the value to `None`. It may end up that matplotlib is still imported
     if the user specifies so in `ipython_execlines` or makes use of the
     @savefig pseudo decorator.
 ipython_execlines:
     A list of strings to be exec'd in the embedded Sphinx shell. Typical
     usage is to make certain packages always available. Set this to an empty
     list if you wish to have no imports always available. If specified in
     conf.py as `None`, then it has the effect of making no imports available.
     If omitted from conf.py altogether, then the default value of
     ['import numpy as np', 'import matplotlib.pyplot as plt'] is used.
 ipython_holdcount
     When the @suppress pseudo-decorator is used, the execution count can be
     incremented or not. The default behavior is to hold the execution count,
     corresponding to a value of `True`. Set this to `False` to increment
     the execution count after each suppressed command.
 
 As an example, to use the IPython directive when `matplotlib` is not available,
 one sets the backend to `None`::
 
     ipython_mplbackend = None
 
 An example usage of the directive is:
 
 .. code-block:: rst
 
     .. ipython::
 
         In [1]: x = 1
 
         In [2]: y = x**2
 
         In [3]: print(y)
 
 See http://matplotlib.org/sampledoc/ipython_directive.html for additional
 documentation.
 
 Pseudo-Decorators
 =================
 
 Note: Only one decorator is supported per input. If more than one decorator
 is specified, then only the last one is used.
 
 In addition to the Pseudo-Decorators/options described at the above link,
 several enhancements have been made. The directive will emit a message to the
 console at build-time if code-execution resulted in an exception or warning.
 You can suppress these on a per-block basis by specifying the :okexcept:
 or :okwarning: options:
 
 .. code-block:: rst
 
     .. ipython::
         :okexcept:
         :okwarning:
 
         In [1]: 1/0
         In [2]: # raise warning.
 
 ToDo
 ----
 
 - Turn the ad-hoc test() function into a real test suite.
 - Break up ipython-specific functionality from matplotlib stuff into better
   separated code.
 
 Authors
 -------
 
 - John D Hunter: orignal author.
 - Fernando Perez: refactoring, documentation, cleanups, port to 0.11.
 - VáclavŠmilauer <eudoxos-AT-arcig.cz>: Prompt generalizations.
 - Skipper Seabold, refactoring, cleanups, pure python addition
 """
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 # Stdlib
 import atexit
 import os
 import re
 import sys
 import tempfile
 import ast
 import warnings
 import shutil
-
+from io import StringIO
 
 # Third-party
 from docutils.parsers.rst import directives
 from sphinx.util.compat import Directive
 
 # Our own
 from traitlets.config import Config
 from IPython import InteractiveShell
 from IPython.core.profiledir import ProfileDir
 from IPython.utils import io
-from IPython.utils.py3compat import PY3
-
-if PY3:
-    from io import StringIO
-else:
-    from StringIO import StringIO
 
 #-----------------------------------------------------------------------------
 # Globals
 #-----------------------------------------------------------------------------
 # for tokenizing blocks
 COMMENT, INPUT, OUTPUT =  range(3)
 
 #-----------------------------------------------------------------------------
 # Functions and class declarations
 #-----------------------------------------------------------------------------
 
 def block_parser(part, rgxin, rgxout, fmtin, fmtout):
     """
     part is a string of ipython text, comprised of at most one
     input, one output, comments, and blank lines.  The block parser
     parses the text into a list of::
 
       blocks = [ (TOKEN0, data0), (TOKEN1, data1), ...]
 
     where TOKEN is one of [COMMENT | INPUT | OUTPUT ] and
     data is, depending on the type of token::
 
       COMMENT : the comment string
 
       INPUT: the (DECORATOR, INPUT_LINE, REST) where
          DECORATOR: the input decorator (or None)
          INPUT_LINE: the input as string (possibly multi-line)
          REST : any stdout generated by the input line (not OUTPUT)
 
       OUTPUT: the output string, possibly multi-line
 
     """
     block = []
     lines = part.split('\n')
     N = len(lines)
     i = 0
     decorator = None
     while 1:
 
         if i==N:
             # nothing left to parse -- the last line
             break
 
         line = lines[i]
         i += 1
         line_stripped = line.strip()
         if line_stripped.startswith('#'):
             block.append((COMMENT, line))
             continue
 
         if line_stripped.startswith('@'):
             # Here is where we assume there is, at most, one decorator.
             # Might need to rethink this.
             decorator = line_stripped
             continue
 
         # does this look like an input line?
         matchin = rgxin.match(line)
         if matchin:
             lineno, inputline = int(matchin.group(1)), matchin.group(2)
 
             # the ....: continuation string
             continuation = '   %s:'%''.join(['.']*(len(str(lineno))+2))
             Nc = len(continuation)
             # input lines can continue on for more than one line, if
             # we have a '\' line continuation char or a function call
             # echo line 'print'.  The input line can only be
             # terminated by the end of the block or an output line, so
             # we parse out the rest of the input line if it is
             # multiline as well as any echo text
 
             rest = []
             while i<N:
 
                 # look ahead; if the next line is blank, or a comment, or
                 # an output line, we're done
 
                 nextline = lines[i]
                 matchout = rgxout.match(nextline)
                 #print "nextline=%s, continuation=%s, starts=%s"%(nextline, continuation, nextline.startswith(continuation))
                 if matchout or nextline.startswith('#'):
                     break
                 elif nextline.startswith(continuation):
                     # The default ipython_rgx* treat the space following the colon as optional.
                     # However, If the space is there we must consume it or code
                     # employing the cython_magic extension will fail to execute.
                     #
                     # This works with the default ipython_rgx* patterns,
                     # If you modify them, YMMV.
                     nextline = nextline[Nc:]
                     if nextline and nextline[0] == ' ':
                         nextline = nextline[1:]
 
                     inputline += '\n' +  nextline
                 else:
                     rest.append(nextline)
                 i+= 1
 
             block.append((INPUT, (decorator, inputline, '\n'.join(rest))))
             continue
 
         # if it looks like an output line grab all the text to the end
         # of the block
         matchout = rgxout.match(line)
         if matchout:
             lineno, output = int(matchout.group(1)), matchout.group(2)
             if i<N-1:
                 output = '\n'.join([output] + lines[i:])
 
             block.append((OUTPUT, output))
             break
 
     return block
 
 
 class EmbeddedSphinxShell(object):
     """An embedded IPython instance to run inside Sphinx"""
 
     def __init__(self, exec_lines=None):
 
         self.cout = StringIO()
 
         if exec_lines is None:
             exec_lines = []
 
         # Create config object for IPython
         config = Config()
         config.HistoryManager.hist_file = ':memory:'
         config.InteractiveShell.autocall = False
         config.InteractiveShell.autoindent = False
         config.InteractiveShell.colors = 'NoColor'
 
         # create a profile so instance history isn't saved
         tmp_profile_dir = tempfile.mkdtemp(prefix='profile_')
         profname = 'auto_profile_sphinx_build'
         pdir = os.path.join(tmp_profile_dir,profname)
         profile = ProfileDir.create_profile_dir(pdir)
 
         # Create and initialize global ipython, but don't start its mainloop.
         # This will persist across different EmbededSphinxShell instances.
         IP = InteractiveShell.instance(config=config, profile_dir=profile)
         atexit.register(self.cleanup)
 
         sys.stdout = self.cout
         sys.stderr = self.cout
 
         # For debugging, so we can see normal output, use this:
         #from IPython.utils.io import Tee
         #sys.stdout = Tee(self.cout, channel='stdout') # dbg
         #sys.stderr = Tee(self.cout, channel='stderr') # dbg
 
         # Store a few parts of IPython we'll need.
         self.IP = IP
         self.user_ns = self.IP.user_ns
         self.user_global_ns = self.IP.user_global_ns
 
         self.input = ''
         self.output = ''
         self.tmp_profile_dir = tmp_profile_dir
 
         self.is_verbatim = False
         self.is_doctest = False
         self.is_suppress = False
 
         # Optionally, provide more detailed information to shell.
         # this is assigned by the SetUp method of IPythonDirective
         # to point at itself.
         #
         # So, you can access handy things at self.directive.state
         self.directive = None
 
         # on the first call to the savefig decorator, we'll import
         # pyplot as plt so we can make a call to the plt.gcf().savefig
         self._pyplot_imported = False
 
         # Prepopulate the namespace.
         for line in exec_lines:
             self.process_input_line(line, store_history=False)
 
     def cleanup(self):
         shutil.rmtree(self.tmp_profile_dir, ignore_errors=True)
 
     def clear_cout(self):
         self.cout.seek(0)
         self.cout.truncate(0)
 
     def process_input_line(self, line, store_history=True):
         """process the input, capturing stdout"""
 
         stdout = sys.stdout
         splitter = self.IP.input_splitter
         try:
             sys.stdout = self.cout
             splitter.push(line)
             more = splitter.push_accepts_more()
             if not more:
                 source_raw = splitter.raw_reset()
                 self.IP.run_cell(source_raw, store_history=store_history)
         finally:
             sys.stdout = stdout
 
     def process_image(self, decorator):
         """
         # build out an image directive like
         # .. image:: somefile.png
         #    :width 4in
         #
         # from an input like
         # savefig somefile.png width=4in
         """
         savefig_dir = self.savefig_dir
         source_dir = self.source_dir
         saveargs = decorator.split(' ')
         filename = saveargs[1]
         # insert relative path to image file in source
         outfile = os.path.relpath(os.path.join(savefig_dir,filename),
                     source_dir)
 
         imagerows = ['.. image:: %s'%outfile]
 
         for kwarg in saveargs[2:]:
             arg, val = kwarg.split('=')
             arg = arg.strip()
             val = val.strip()
             imagerows.append('   :%s: %s'%(arg, val))
 
         image_file = os.path.basename(outfile) # only return file name
         image_directive = '\n'.join(imagerows)
         return image_file, image_directive
 
     # Callbacks for each type of token
     def process_input(self, data, input_prompt, lineno):
         """
         Process data block for INPUT token.
 
         """
         decorator, input, rest = data
         image_file = None
         image_directive = None
 
         is_verbatim = decorator=='@verbatim' or self.is_verbatim
         is_doctest = (decorator is not None and \
                      decorator.startswith('@doctest')) or self.is_doctest
         is_suppress = decorator=='@suppress' or self.is_suppress
         is_okexcept = decorator=='@okexcept' or self.is_okexcept
         is_okwarning = decorator=='@okwarning' or self.is_okwarning
         is_savefig = decorator is not None and \
                      decorator.startswith('@savefig')
 
         input_lines = input.split('\n')
         if len(input_lines) > 1:
             if input_lines[-1] != "":
                 input_lines.append('') # make sure there's a blank line
                                        # so splitter buffer gets reset
 
         continuation = '   %s:'%''.join(['.']*(len(str(lineno))+2))
 
         if is_savefig:
             image_file, image_directive = self.process_image(decorator)
 
         ret = []
         is_semicolon = False
 
         # Hold the execution count, if requested to do so.
         if is_suppress and self.hold_count:
             store_history = False
         else:
             store_history = True
 
         # Note: catch_warnings is not thread safe
         with warnings.catch_warnings(record=True) as ws:
             for i, line in enumerate(input_lines):
                 if line.endswith(';'):
                     is_semicolon = True
 
                 if i == 0:
                     # process the first input line
                     if is_verbatim:
                         self.process_input_line('')
                         self.IP.execution_count += 1 # increment it anyway
                     else:
                         # only submit the line in non-verbatim mode
                         self.process_input_line(line, store_history=store_history)
                     formatted_line = '%s %s'%(input_prompt, line)
                 else:
                     # process a continuation line
                     if not is_verbatim:
                         self.process_input_line(line, store_history=store_history)
 
                     formatted_line = '%s %s'%(continuation, line)
 
                 if not is_suppress:
                     ret.append(formatted_line)
 
         if not is_suppress and len(rest.strip()) and is_verbatim:
             # The "rest" is the standard output of the input. This needs to be
             # added when in verbatim mode. If there is no "rest", then we don't
             # add it, as the new line will be added by the processed output.
             ret.append(rest)
 
         # Fetch the processed output. (This is not the submitted output.)
         self.cout.seek(0)
         processed_output = self.cout.read()
         if not is_suppress and not is_semicolon:
             #
             # In IPythonDirective.run, the elements of `ret` are eventually
             # combined such that '' entries correspond to newlines. So if
             # `processed_output` is equal to '', then the adding it to `ret`
             # ensures that there is a blank line between consecutive inputs
             # that have no outputs, as in:
             #
             #    In [1]: x = 4
             #
             #    In [2]: x = 5
             #
             # When there is processed output, it has a '\n' at the tail end. So
             # adding the output to `ret` will provide the necessary spacing
             # between consecutive input/output blocks, as in:
             #
             #   In [1]: x
             #   Out[1]: 5
             #
             #   In [2]: x
             #   Out[2]: 5
             #
             # When there is stdout from the input, it also has a '\n' at the
             # tail end, and so this ensures proper spacing as well. E.g.:
             #
             #   In [1]: print x
             #   5
             #
             #   In [2]: x = 5
             #
             # When in verbatim mode, `processed_output` is empty (because
             # nothing was passed to IP. Sometimes the submitted code block has
             # an Out[] portion and sometimes it does not. When it does not, we
             # need to ensure proper spacing, so we have to add '' to `ret`.
             # However, if there is an Out[] in the submitted code, then we do
             # not want to add a newline as `process_output` has stuff to add.
             # The difficulty is that `process_input` doesn't know if
             # `process_output` will be called---so it doesn't know if there is
             # Out[] in the code block. The requires that we include a hack in
             # `process_block`. See the comments there.
             #
             ret.append(processed_output)
         elif is_semicolon:
             # Make sure there is a newline after the semicolon.
             ret.append('')
 
         # context information
         filename = "Unknown"
         lineno = 0
         if self.directive.state:
             filename = self.directive.state.document.current_source
             lineno = self.directive.state.document.current_line
 
         # output any exceptions raised during execution to stdout
         # unless :okexcept: has been specified.
         if not is_okexcept and "Traceback" in processed_output:
             s =  "\nException in %s at block ending on line %s\n" % (filename, lineno)
             s += "Specify :okexcept: as an option in the ipython:: block to suppress this message\n"
             sys.stdout.write('\n\n>>>' + ('-' * 73))
             sys.stdout.write(s)
             sys.stdout.write(processed_output)
             sys.stdout.write('<<<' + ('-' * 73) + '\n\n')
 
         # output any warning raised during execution to stdout
         # unless :okwarning: has been specified.
         if not is_okwarning:
             for w in ws:
                 s =  "\nWarning in %s at block ending on line %s\n" % (filename, lineno)
                 s += "Specify :okwarning: as an option in the ipython:: block to suppress this message\n"
                 sys.stdout.write('\n\n>>>' + ('-' * 73))
                 sys.stdout.write(s)
                 sys.stdout.write(('-' * 76) + '\n')
                 s=warnings.formatwarning(w.message, w.category,
                                          w.filename, w.lineno, w.line)
                 sys.stdout.write(s)
                 sys.stdout.write('<<<' + ('-' * 73) + '\n')
 
         self.cout.truncate(0)
 
         return (ret, input_lines, processed_output,
                 is_doctest, decorator, image_file, image_directive)
 
 
     def process_output(self, data, output_prompt, input_lines, output,
                        is_doctest, decorator, image_file):
         """
         Process data block for OUTPUT token.
 
         """
         # Recall: `data` is the submitted output, and `output` is the processed
         # output from `input_lines`.
 
         TAB = ' ' * 4
 
         if is_doctest and output is not None:
 
             found = output # This is the processed output
             found = found.strip()
             submitted = data.strip()
 
             if self.directive is None:
                 source = 'Unavailable'
                 content = 'Unavailable'
             else:
                 source = self.directive.state.document.current_source
                 content = self.directive.content
                 # Add tabs and join into a single string.
                 content = '\n'.join([TAB + line for line in content])
 
             # Make sure the output contains the output prompt.
             ind = found.find(output_prompt)
             if ind < 0:
                 e = ('output does not contain output prompt\n\n'
                      'Document source: {0}\n\n'
                      'Raw content: \n{1}\n\n'
                      'Input line(s):\n{TAB}{2}\n\n'
                      'Output line(s):\n{TAB}{3}\n\n')
                 e = e.format(source, content, '\n'.join(input_lines),
                              repr(found), TAB=TAB)
                 raise RuntimeError(e)
             found = found[len(output_prompt):].strip()
 
             # Handle the actual doctest comparison.
             if decorator.strip() == '@doctest':
                 # Standard doctest
                 if found != submitted:
                     e = ('doctest failure\n\n'
                          'Document source: {0}\n\n'
                          'Raw content: \n{1}\n\n'
                          'On input line(s):\n{TAB}{2}\n\n'
                          'we found output:\n{TAB}{3}\n\n'
                          'instead of the expected:\n{TAB}{4}\n\n')
                     e = e.format(source, content, '\n'.join(input_lines),
                                  repr(found), repr(submitted), TAB=TAB)
                     raise RuntimeError(e)
             else:
                 self.custom_doctest(decorator, input_lines, found, submitted)
 
         # When in verbatim mode, this holds additional submitted output
         # to be written in the final Sphinx output.
         # https://github.com/ipython/ipython/issues/5776
         out_data = []
 
         is_verbatim = decorator=='@verbatim' or self.is_verbatim
         if is_verbatim and data.strip():
             # Note that `ret` in `process_block` has '' as its last element if
             # the code block was in verbatim mode. So if there is no submitted
             # output, then we will have proper spacing only if we do not add
             # an additional '' to `out_data`. This is why we condition on
             # `and data.strip()`.
 
             # The submitted output has no output prompt. If we want the
             # prompt and the code to appear, we need to join them now
             # instead of adding them separately---as this would create an
             # undesired newline. How we do this ultimately depends on the
             # format of the output regex. I'll do what works for the default
             # prompt for now, and we might have to adjust if it doesn't work
             # in other cases. Finally, the submitted output does not have
             # a trailing newline, so we must add it manually.
             out_data.append("{0} {1}\n".format(output_prompt, data))
 
         return out_data
 
     def process_comment(self, data):
         """Process data fPblock for COMMENT token."""
         if not self.is_suppress:
             return [data]
 
     def save_image(self, image_file):
         """
         Saves the image file to disk.
         """
         self.ensure_pyplot()
         command = 'plt.gcf().savefig("%s")'%image_file
         #print 'SAVEFIG', command  # dbg
         self.process_input_line('bookmark ipy_thisdir', store_history=False)
         self.process_input_line('cd -b ipy_savedir', store_history=False)
         self.process_input_line(command, store_history=False)
         self.process_input_line('cd -b ipy_thisdir', store_history=False)
         self.process_input_line('bookmark -d ipy_thisdir', store_history=False)
         self.clear_cout()
 
     def process_block(self, block):
         """
         process block from the block_parser and return a list of processed lines
         """
         ret = []
         output = None
         input_lines = None
         lineno = self.IP.execution_count
 
         input_prompt = self.promptin % lineno
         output_prompt = self.promptout % lineno
         image_file = None
         image_directive = None
 
         found_input = False
         for token, data in block:
             if token == COMMENT:
                 out_data = self.process_comment(data)
             elif token == INPUT:
                 found_input = True
                 (out_data, input_lines, output, is_doctest,
                  decorator, image_file, image_directive) = \
                           self.process_input(data, input_prompt, lineno)
             elif token == OUTPUT:
                 if not found_input:
 
                     TAB = ' ' * 4
                     linenumber = 0
                     source = 'Unavailable'
                     content = 'Unavailable'
                     if self.directive:
                         linenumber = self.directive.state.document.current_line
                         source = self.directive.state.document.current_source
                         content = self.directive.content
                         # Add tabs and join into a single string.
                         content = '\n'.join([TAB + line for line in content])
 
                     e = ('\n\nInvalid block: Block contains an output prompt '
                          'without an input prompt.\n\n'
                          'Document source: {0}\n\n'
                          'Content begins at line {1}: \n\n{2}\n\n'
                          'Problematic block within content: \n\n{TAB}{3}\n\n')
                     e = e.format(source, linenumber, content, block, TAB=TAB)
 
                     # Write, rather than include in exception, since Sphinx
                     # will truncate tracebacks.
                     sys.stdout.write(e)
                     raise RuntimeError('An invalid block was detected.')
 
                 out_data = \
                     self.process_output(data, output_prompt, input_lines,
                                         output, is_doctest, decorator,
                                         image_file)
                 if out_data:
                     # Then there was user submitted output in verbatim mode.
                     # We need to remove the last element of `ret` that was
                     # added in `process_input`, as it is '' and would introduce
                     # an undesirable newline.
                     assert(ret[-1] == '')
                     del ret[-1]
 
             if out_data:
                 ret.extend(out_data)
 
         # save the image files
         if image_file is not None:
             self.save_image(image_file)
 
         return ret, image_directive
 
     def ensure_pyplot(self):
         """
         Ensures that pyplot has been imported into the embedded IPython shell.
 
         Also, makes sure to set the backend appropriately if not set already.
 
         """
         # We are here if the @figure pseudo decorator was used. Thus, it's
         # possible that we could be here even if python_mplbackend were set to
         # `None`. That's also strange and perhaps worthy of raising an
         # exception, but for now, we just set the backend to 'agg'.
 
         if not self._pyplot_imported:
             if 'matplotlib.backends' not in sys.modules:
                 # Then ipython_matplotlib was set to None but there was a
                 # call to the @figure decorator (and ipython_execlines did
                 # not set a backend).
                 #raise Exception("No backend was set, but @figure was used!")
                 import matplotlib
                 matplotlib.use('agg')
 
             # Always import pyplot into embedded shell.
             self.process_input_line('import matplotlib.pyplot as plt',
                                     store_history=False)
             self._pyplot_imported = True
 
     def process_pure_python(self, content):
         """
         content is a list of strings. it is unedited directive content
 
         This runs it line by line in the InteractiveShell, prepends
         prompts as needed capturing stderr and stdout, then returns
         the content as a list as if it were ipython code
         """
         output = []
         savefig = False # keep up with this to clear figure
         multiline = False # to handle line continuation
         multiline_start = None
         fmtin = self.promptin
 
         ct = 0
 
         for lineno, line in enumerate(content):
 
             line_stripped = line.strip()
             if not len(line):
                 output.append(line)
                 continue
 
             # handle decorators
             if line_stripped.startswith('@'):
                 output.extend([line])
                 if 'savefig' in line:
                     savefig = True # and need to clear figure
                 continue
 
             # handle comments
             if line_stripped.startswith('#'):
                 output.extend([line])
                 continue
 
             # deal with lines checking for multiline
             continuation  = u'   %s:'% ''.join(['.']*(len(str(ct))+2))
             if not multiline:
                 modified = u"%s %s" % (fmtin % ct, line_stripped)
                 output.append(modified)
                 ct += 1
                 try:
                     ast.parse(line_stripped)
                     output.append(u'')
                 except Exception: # on a multiline
                     multiline = True
                     multiline_start = lineno
             else: # still on a multiline
                 modified = u'%s %s' % (continuation, line)
                 output.append(modified)
 
                 # if the next line is indented, it should be part of multiline
                 if len(content) > lineno + 1:
                     nextline = content[lineno + 1]
                     if len(nextline) - len(nextline.lstrip()) > 3:
                         continue
                 try:
                     mod = ast.parse(
                             '\n'.join(content[multiline_start:lineno+1]))
                     if isinstance(mod.body[0], ast.FunctionDef):
                         # check to see if we have the whole function
                         for element in mod.body[0].body:
                             if isinstance(element, ast.Return):
                                 multiline = False
                     else:
                         output.append(u'')
                         multiline = False
                 except Exception:
                     pass
 
             if savefig: # clear figure if plotted
                 self.ensure_pyplot()
                 self.process_input_line('plt.clf()', store_history=False)
                 self.clear_cout()
                 savefig = False
 
         return output
 
     def custom_doctest(self, decorator, input_lines, found, submitted):
         """
         Perform a specialized doctest.
 
         """
         from .custom_doctests import doctests
 
         args = decorator.split()
         doctest_type = args[1]
         if doctest_type in doctests:
             doctests[doctest_type](self, args, input_lines, found, submitted)
         else:
             e = "Invalid option to @doctest: {0}".format(doctest_type)
             raise Exception(e)
 
 
 class IPythonDirective(Directive):
 
     has_content = True
     required_arguments = 0
     optional_arguments = 4 # python, suppress, verbatim, doctest
     final_argumuent_whitespace = True
     option_spec = { 'python': directives.unchanged,
                     'suppress' : directives.flag,
                     'verbatim' : directives.flag,
                     'doctest' : directives.flag,
                     'okexcept': directives.flag,
                     'okwarning': directives.flag
                   }
 
     shell = None
 
     seen_docs = set()
 
     def get_config_options(self):
         # contains sphinx configuration variables
         config = self.state.document.settings.env.config
 
         # get config variables to set figure output directory
         outdir = self.state.document.settings.env.app.outdir
         savefig_dir = config.ipython_savefig_dir
         source_dir = os.path.dirname(self.state.document.current_source)
         if savefig_dir is None:
             savefig_dir = config.html_static_path or '_static'
         if isinstance(savefig_dir, list):
             savefig_dir = os.path.join(*savefig_dir)
         savefig_dir = os.path.join(outdir, savefig_dir)
 
         # get regex and prompt stuff
         rgxin      = config.ipython_rgxin
         rgxout     = config.ipython_rgxout
         promptin   = config.ipython_promptin
         promptout  = config.ipython_promptout
         mplbackend = config.ipython_mplbackend
         exec_lines = config.ipython_execlines
         hold_count = config.ipython_holdcount
 
         return (savefig_dir, source_dir, rgxin, rgxout,
                 promptin, promptout, mplbackend, exec_lines, hold_count)
 
     def setup(self):
         # Get configuration values.
         (savefig_dir, source_dir, rgxin, rgxout, promptin, promptout,
          mplbackend, exec_lines, hold_count) = self.get_config_options()
 
         if self.shell is None:
             # We will be here many times.  However, when the
             # EmbeddedSphinxShell is created, its interactive shell member
             # is the same for each instance.
 
             if mplbackend and 'matplotlib.backends' not in sys.modules:
                 import matplotlib
                 matplotlib.use(mplbackend)
 
             # Must be called after (potentially) importing matplotlib and
             # setting its backend since exec_lines might import pylab.
             self.shell = EmbeddedSphinxShell(exec_lines)
 
             # Store IPython directive to enable better error messages
             self.shell.directive = self
 
         # reset the execution count if we haven't processed this doc
         #NOTE: this may be borked if there are multiple seen_doc tmp files
         #check time stamp?
         if not self.state.document.current_source in self.seen_docs:
             self.shell.IP.history_manager.reset()
             self.shell.IP.execution_count = 1
             self.seen_docs.add(self.state.document.current_source)
 
         # and attach to shell so we don't have to pass them around
         self.shell.rgxin = rgxin
         self.shell.rgxout = rgxout
         self.shell.promptin = promptin
         self.shell.promptout = promptout
         self.shell.savefig_dir = savefig_dir
         self.shell.source_dir = source_dir
         self.shell.hold_count = hold_count
 
         # setup bookmark for saving figures directory
         self.shell.process_input_line('bookmark ipy_savedir %s'%savefig_dir,
                                       store_history=False)
         self.shell.clear_cout()
 
         return rgxin, rgxout, promptin, promptout
 
     def teardown(self):
         # delete last bookmark
         self.shell.process_input_line('bookmark -d ipy_savedir',
                                       store_history=False)
         self.shell.clear_cout()
 
     def run(self):
         debug = False
 
         #TODO, any reason block_parser can't be a method of embeddable shell
         # then we wouldn't have to carry these around
         rgxin, rgxout, promptin, promptout = self.setup()
 
         options = self.options
         self.shell.is_suppress = 'suppress' in options
         self.shell.is_doctest = 'doctest' in options
         self.shell.is_verbatim = 'verbatim' in options
         self.shell.is_okexcept = 'okexcept' in options
         self.shell.is_okwarning = 'okwarning' in options
 
         # handle pure python code
         if 'python' in self.arguments:
             content = self.content
             self.content = self.shell.process_pure_python(content)
 
         # parts consists of all text within the ipython-block.
         # Each part is an input/output block.
         parts = '\n'.join(self.content).split('\n\n')
 
         lines = ['.. code-block:: ipython', '']
         figures = []
 
         for part in parts:
             block = block_parser(part, rgxin, rgxout, promptin, promptout)
             if len(block):
                 rows, figure = self.shell.process_block(block)
                 for row in rows:
                     lines.extend(['   {0}'.format(line)
                                   for line in row.split('\n')])
 
                 if figure is not None:
                     figures.append(figure)
 
         for figure in figures:
             lines.append('')
             lines.extend(figure.split('\n'))
             lines.append('')
 
         if len(lines) > 2:
             if debug:
                 print('\n'.join(lines))
             else:
                 # This has to do with input, not output. But if we comment
                 # these lines out, then no IPython code will appear in the
                 # final output.
                 self.state_machine.insert_input(
                     lines, self.state_machine.input_lines.source(0))
 
         # cleanup
         self.teardown()
 
         return []
 
 # Enable as a proper Sphinx directive
 def setup(app):
     setup.app = app
 
     app.add_directive('ipython', IPythonDirective)
     app.add_config_value('ipython_savefig_dir', None, 'env')
     app.add_config_value('ipython_rgxin',
                          re.compile('In \[(\d+)\]:\s?(.*)\s*'), 'env')
     app.add_config_value('ipython_rgxout',
                          re.compile('Out\[(\d+)\]:\s?(.*)\s*'), 'env')
     app.add_config_value('ipython_promptin', 'In [%d]:', 'env')
     app.add_config_value('ipython_promptout', 'Out[%d]:', 'env')
 
     # We could just let matplotlib pick whatever is specified as the default
     # backend in the matplotlibrc file, but this would cause issues if the
     # backend didn't work in headless environments. For this reason, 'agg'
     # is a good default backend choice.
     app.add_config_value('ipython_mplbackend', 'agg', 'env')
 
     # If the user sets this config value to `None`, then EmbeddedSphinxShell's
     # __init__ method will treat it as [].
     execlines = ['import numpy as np', 'import matplotlib.pyplot as plt']
     app.add_config_value('ipython_execlines', execlines, 'env')
 
     app.add_config_value('ipython_holdcount', True, 'env')
 
     metadata = {'parallel_read_safe': True, 'parallel_write_safe': True}
     return metadata
 
 # Simple smoke test, needs to be converted to a proper automatic test.
 def test():
 
     examples = [
         r"""
 In [9]: pwd
 Out[9]: '/home/jdhunter/py4science/book'
 
 In [10]: cd bookdata/
 /home/jdhunter/py4science/book/bookdata
 
 In [2]: from pylab import *
 
 In [2]: ion()
 
 In [3]: im = imread('stinkbug.png')
 
 @savefig mystinkbug.png width=4in
 In [4]: imshow(im)
 Out[4]: <matplotlib.image.AxesImage object at 0x39ea850>
 
 """,
         r"""
 
 In [1]: x = 'hello world'
 
 # string methods can be
 # used to alter the string
 @doctest
 In [2]: x.upper()
 Out[2]: 'HELLO WORLD'
 
 @verbatim
 In [3]: x.st<TAB>
 x.startswith  x.strip
 """,
     r"""
 
 In [130]: url = 'http://ichart.finance.yahoo.com/table.csv?s=CROX\
    .....: &d=9&e=22&f=2009&g=d&a=1&br=8&c=2006&ignore=.csv'
 
 In [131]: print url.split('&')
 ['http://ichart.finance.yahoo.com/table.csv?s=CROX', 'd=9', 'e=22', 'f=2009', 'g=d', 'a=1', 'b=8', 'c=2006', 'ignore=.csv']
 
 In [60]: import urllib
 
 """,
     r"""\
 
 In [133]: import numpy.random
 
 @suppress
 In [134]: numpy.random.seed(2358)
 
 @doctest
 In [135]: numpy.random.rand(10,2)
 Out[135]:
 array([[ 0.64524308,  0.59943846],
        [ 0.47102322,  0.8715456 ],
        [ 0.29370834,  0.74776844],
        [ 0.99539577,  0.1313423 ],
        [ 0.16250302,  0.21103583],
        [ 0.81626524,  0.1312433 ],
        [ 0.67338089,  0.72302393],
        [ 0.7566368 ,  0.07033696],
        [ 0.22591016,  0.77731835],
        [ 0.0072729 ,  0.34273127]])
 
 """,
 
     r"""
 In [106]: print x
 jdh
 
 In [109]: for i in range(10):
    .....:     print i
    .....:
    .....:
 0
 1
 2
 3
 4
 5
 6
 7
 8
 9
 """,
 
         r"""
 
 In [144]: from pylab import *
 
 In [145]: ion()
 
 # use a semicolon to suppress the output
 @savefig test_hist.png width=4in
 In [151]: hist(np.random.randn(10000), 100);
 
 
 @savefig test_plot.png width=4in
 In [151]: plot(np.random.randn(10000), 'o');
    """,
 
         r"""
 # use a semicolon to suppress the output
 In [151]: plt.clf()
 
 @savefig plot_simple.png width=4in
 In [151]: plot([1,2,3])
 
 @savefig hist_simple.png width=4in
 In [151]: hist(np.random.randn(10000), 100);
 
 """,
      r"""
 # update the current fig
 In [151]: ylabel('number')
 
 In [152]: title('normal distribution')
 
 
 @savefig hist_with_text.png
 In [153]: grid(True)
 
 @doctest float
 In [154]: 0.1 + 0.2
 Out[154]: 0.3
 
 @doctest float
 In [155]: np.arange(16).reshape(4,4)
 Out[155]:
 array([[ 0,  1,  2,  3],
        [ 4,  5,  6,  7],
        [ 8,  9, 10, 11],
        [12, 13, 14, 15]])
 
 In [1]: x = np.arange(16, dtype=float).reshape(4,4)
 
 In [2]: x[0,0] = np.inf
 
 In [3]: x[0,1] = np.nan
 
 @doctest float
 In [4]: x
 Out[4]:
 array([[ inf,  nan,   2.,   3.],
        [  4.,   5.,   6.,   7.],
        [  8.,   9.,  10.,  11.],
        [ 12.,  13.,  14.,  15.]])
 
 
         """,
         ]
     # skip local-file depending first example:
     examples = examples[1:]
 
     #ipython_directive.DEBUG = True  # dbg
     #options = dict(suppress=True)  # dbg
     options = dict()
     for example in examples:
         content = example.split('\n')
         IPythonDirective('debug', arguments=None, options=options,
                           content=content, lineno=0,
                           content_offset=None, block_text=None,
                           state=None, state_machine=None,
                           )
 
 # Run test suite as a script
 if __name__=='__main__':
     if not os.path.isdir('_static'):
         os.mkdir('_static')
     test()
     print('All OK? Check figures in _static/')

IPython/terminal/interactiveshell.py

@@ -1,500 +1,478 b''
 """IPython terminal interface using prompt_toolkit"""
 
 import os
 import sys
 import warnings
 from warnings import warn
 
 from IPython.core.interactiveshell import InteractiveShell, InteractiveShellABC
 from IPython.utils import io
-from IPython.utils.py3compat import PY3, cast_unicode_py2, input
+from IPython.utils.py3compat import cast_unicode_py2, input
 from IPython.utils.terminal import toggle_set_term_title, set_term_title
 from IPython.utils.process import abbrev_cwd
 from traitlets import Bool, Unicode, Dict, Integer, observe, Instance, Type, default, Enum, Union
 
 from prompt_toolkit.enums import DEFAULT_BUFFER, EditingMode
 from prompt_toolkit.filters import (HasFocus, Condition, IsDone)
 from prompt_toolkit.history import InMemoryHistory
 from prompt_toolkit.shortcuts import create_prompt_application, create_eventloop, create_prompt_layout, create_output
 from prompt_toolkit.interface import CommandLineInterface
 from prompt_toolkit.key_binding.manager import KeyBindingManager
 from prompt_toolkit.layout.processors import ConditionalProcessor, HighlightMatchingBracketProcessor
 from prompt_toolkit.styles import PygmentsStyle, DynamicStyle
 
 from pygments.styles import get_style_by_name, get_all_styles
 from pygments.style import Style
 from pygments.token import Token
 
 from .debugger import TerminalPdb, Pdb
 from .magics import TerminalMagics
 from .pt_inputhooks import get_inputhook_name_and_func
 from .prompts import Prompts, ClassicPrompts, RichPromptDisplayHook
 from .ptutils import IPythonPTCompleter, IPythonPTLexer
 from .shortcuts import register_ipython_shortcuts
 
 DISPLAY_BANNER_DEPRECATED = object()
 
 
 from pygments.style import Style
 
 class _NoStyle(Style): pass
 
 
 
 _style_overrides_light_bg = {
             Token.Prompt: '#0000ff',
             Token.PromptNum: '#0000ee bold',
             Token.OutPrompt: '#cc0000',
             Token.OutPromptNum: '#bb0000 bold',
 }
 
 _style_overrides_linux = {
             Token.Prompt: '#00cc00',
             Token.PromptNum: '#00bb00 bold',
             Token.OutPrompt: '#cc0000',
             Token.OutPromptNum: '#bb0000 bold',
 }
 
 
 
 def get_default_editor():
     try:
-        ed = os.environ['EDITOR']
-        if not PY3:
-            ed = ed.decode()
-        return ed
+        return os.environ['EDITOR']
     except KeyError:
         pass
     except UnicodeError:
         warn("$EDITOR environment variable is not pure ASCII. Using platform "
              "default editor.")
 
     if os.name == 'posix':
         return 'vi'  # the only one guaranteed to be there!
     else:
         return 'notepad' # same in Windows!
 
 # conservatively check for tty
 # overridden streams can result in things like:
 # - sys.stdin = None
 # - no isatty method
 for _name in ('stdin', 'stdout', 'stderr'):
     _stream = getattr(sys, _name)
     if not _stream or not hasattr(_stream, 'isatty') or not _stream.isatty():
         _is_tty = False
         break
 else:
     _is_tty = True
 
 
 _use_simple_prompt = ('IPY_TEST_SIMPLE_PROMPT' in os.environ) or (not _is_tty)
 
 class TerminalInteractiveShell(InteractiveShell):
     space_for_menu = Integer(6, help='Number of line at the bottom of the screen '
                                                   'to reserve for the completion menu'
                             ).tag(config=True)
 
     def _space_for_menu_changed(self, old, new):
         self._update_layout()
 
     pt_cli = None
     debugger_history = None
     _pt_app = None
 
     simple_prompt = Bool(_use_simple_prompt,
         help="""Use `raw_input` for the REPL, without completion, multiline input, and prompt colors.
 
             Useful when controlling IPython as a subprocess, and piping STDIN/OUT/ERR. Known usage are:
             IPython own testing machinery, and emacs inferior-shell integration through elpy.
 
             This mode default to `True` if the `IPY_TEST_SIMPLE_PROMPT`
             environment variable is set, or the current terminal is not a tty.
 
             """
             ).tag(config=True)
 
     @property
     def debugger_cls(self):
         return Pdb if self.simple_prompt else TerminalPdb
 
     confirm_exit = Bool(True,
         help="""
         Set to confirm when you try to exit IPython with an EOF (Control-D
         in Unix, Control-Z/Enter in Windows). By typing 'exit' or 'quit',
         you can force a direct exit without any confirmation.""",
     ).tag(config=True)
 
     editing_mode = Unicode('emacs',
         help="Shortcut style to use at the prompt. 'vi' or 'emacs'.",
     ).tag(config=True)
 
     mouse_support = Bool(False,
         help="Enable mouse support in the prompt"
     ).tag(config=True)
 
     highlighting_style = Union([Unicode('legacy'), Type(klass=Style)],
         help="""The name or class of a Pygments style to use for syntax
         highlighting: \n %s""" % ', '.join(get_all_styles())
     ).tag(config=True)
 
 
     @observe('highlighting_style')
     @observe('colors')
     def _highlighting_style_changed(self, change):
         self.refresh_style()
 
     def refresh_style(self):
         self._style = self._make_style_from_name_or_cls(self.highlighting_style)
 
 
     highlighting_style_overrides = Dict(
         help="Override highlighting format for specific tokens"
     ).tag(config=True)
 
     true_color = Bool(False,
         help=("Use 24bit colors instead of 256 colors in prompt highlighting. "
               "If your terminal supports true color, the following command "
               "should print 'TRUECOLOR' in orange: "
               "printf \"\\x1b[38;2;255;100;0mTRUECOLOR\\x1b[0m\\n\"")
     ).tag(config=True)
 
     editor = Unicode(get_default_editor(),
         help="Set the editor used by IPython (default to $EDITOR/vi/notepad)."
     ).tag(config=True)
 
     prompts_class = Type(Prompts, help='Class used to generate Prompt token for prompt_toolkit').tag(config=True)
 
     prompts = Instance(Prompts)
 
     @default('prompts')
     def _prompts_default(self):
         return self.prompts_class(self)
 
     @observe('prompts')
     def _(self, change):
         self._update_layout()
 
     @default('displayhook_class')
     def _displayhook_class_default(self):
         return RichPromptDisplayHook
 
     term_title = Bool(True,
         help="Automatically set the terminal title"
     ).tag(config=True)
 
     display_completions = Enum(('column', 'multicolumn','readlinelike'),
         help= ( "Options for displaying tab completions, 'column', 'multicolumn', and "
                 "'readlinelike'. These options are for `prompt_toolkit`, see "
                 "`prompt_toolkit` documentation for more information."
                 ),
         default_value='multicolumn').tag(config=True)
 
     highlight_matching_brackets = Bool(True,
         help="Highlight matching brackets .",
     ).tag(config=True)
 
     @observe('term_title')
     def init_term_title(self, change=None):
         # Enable or disable the terminal title.
         if self.term_title:
             toggle_set_term_title(True)
             set_term_title('IPython: ' + abbrev_cwd())
         else:
             toggle_set_term_title(False)
 
     def init_display_formatter(self):
         super(TerminalInteractiveShell, self).init_display_formatter()
         # terminal only supports plain text
         self.display_formatter.active_types = ['text/plain']
 
     def init_prompt_toolkit_cli(self):
         if self.simple_prompt:
             # Fall back to plain non-interactive output for tests.
             # This is very limited, and only accepts a single line.
             def prompt():
                 return cast_unicode_py2(input('In [%d]: ' % self.execution_count))
             self.prompt_for_code = prompt
             return
 
         # Set up keyboard shortcuts
         kbmanager = KeyBindingManager.for_prompt()
         register_ipython_shortcuts(kbmanager.registry, self)
 
         # Pre-populate history from IPython's history database
         history = InMemoryHistory()
         last_cell = u""
         for __, ___, cell in self.history_manager.get_tail(self.history_load_length,
                                                         include_latest=True):
             # Ignore blank lines and consecutive duplicates
             cell = cell.rstrip()
             if cell and (cell != last_cell):
                 history.append(cell)
                 last_cell = cell
 
         self._style = self._make_style_from_name_or_cls(self.highlighting_style)
         style = DynamicStyle(lambda: self._style)
 
         editing_mode = getattr(EditingMode, self.editing_mode.upper())
 
         self._pt_app = create_prompt_application(
                             editing_mode=editing_mode,
                             key_bindings_registry=kbmanager.registry,
                             history=history,
                             completer=IPythonPTCompleter(shell=self),
                             enable_history_search=True,
                             style=style,
                             mouse_support=self.mouse_support,
                             **self._layout_options()
         )
         self._eventloop = create_eventloop(self.inputhook)
         self.pt_cli = CommandLineInterface(
             self._pt_app, eventloop=self._eventloop,
             output=create_output(true_color=self.true_color))
 
     def _make_style_from_name_or_cls(self, name_or_cls):
         """
         Small wrapper that make an IPython compatible style from a style name
 
         We need that to add style for prompt ... etc.
         """
         style_overrides = {}
         if name_or_cls == 'legacy':
             legacy = self.colors.lower()
             if legacy == 'linux':
                 style_cls = get_style_by_name('monokai')
                 style_overrides = _style_overrides_linux
             elif legacy == 'lightbg':
                 style_overrides = _style_overrides_light_bg
                 style_cls = get_style_by_name('pastie')
             elif legacy == 'neutral':
                 # The default theme needs to be visible on both a dark background
                 # and a light background, because we can't tell what the terminal
                 # looks like. These tweaks to the default theme help with that.
                 style_cls = get_style_by_name('default')
                 style_overrides.update({
                     Token.Number: '#007700',
                     Token.Operator: 'noinherit',
                     Token.String: '#BB6622',
                     Token.Name.Function: '#2080D0',
                     Token.Name.Class: 'bold #2080D0',
                     Token.Name.Namespace: 'bold #2080D0',
                     Token.Prompt: '#009900',
                     Token.PromptNum: '#00ff00 bold',
                     Token.OutPrompt: '#990000',
                     Token.OutPromptNum: '#ff0000 bold',
                 })
             elif legacy =='nocolor':
                 style_cls=_NoStyle
                 style_overrides = {}
             else :
                 raise ValueError('Got unknown colors: ', legacy)
         else :
             if isinstance(name_or_cls, str):
                 style_cls = get_style_by_name(name_or_cls)
             else:
                 style_cls = name_or_cls
             style_overrides = {
                 Token.Prompt: '#009900',
                 Token.PromptNum: '#00ff00 bold',
                 Token.OutPrompt: '#990000',
                 Token.OutPromptNum: '#ff0000 bold',
             }
         style_overrides.update(self.highlighting_style_overrides)
         style = PygmentsStyle.from_defaults(pygments_style_cls=style_cls,
                                             style_dict=style_overrides)
 
         return style
 
     def _layout_options(self):
         """
         Return the current layout option for the current Terminal InteractiveShell
         """
         return {
                 'lexer':IPythonPTLexer(),
                 'reserve_space_for_menu':self.space_for_menu,
                 'get_prompt_tokens':self.prompts.in_prompt_tokens,
                 'get_continuation_tokens':self.prompts.continuation_prompt_tokens,
                 'multiline':True,
                 'display_completions_in_columns': (self.display_completions == 'multicolumn'),
 
                 # Highlight matching brackets, but only when this setting is
                 # enabled, and only when the DEFAULT_BUFFER has the focus.
                 'extra_input_processors': [ConditionalProcessor(
                         processor=HighlightMatchingBracketProcessor(chars='[](){}'),
                         filter=HasFocus(DEFAULT_BUFFER) & ~IsDone() &
                             Condition(lambda cli: self.highlight_matching_brackets))],
                 }
 
     def _update_layout(self):
         """
         Ask for a re computation of the application layout, if for example ,
         some configuration options have changed.
         """
         if self._pt_app:
             self._pt_app.layout = create_prompt_layout(**self._layout_options())
 
     def prompt_for_code(self):
         document = self.pt_cli.run(
             pre_run=self.pre_prompt, reset_current_buffer=True)
         return document.text
 
     def enable_win_unicode_console(self):
         if sys.version_info >= (3, 6):
             # Since PEP 528, Python uses the unicode APIs for the Windows
             # console by default, so WUC shouldn't be needed.
             return
 
         import win_unicode_console
-
-        if PY3:
-            win_unicode_console.enable()
-        else:
-            # https://github.com/ipython/ipython/issues/9768
-            from win_unicode_console.streams import (TextStreamWrapper,
-                                 stdout_text_transcoded, stderr_text_transcoded)
-
-            class LenientStrStreamWrapper(TextStreamWrapper):
-                def write(self, s):
-                    if isinstance(s, bytes):
-                        s = s.decode(self.encoding, 'replace')
-
-                    self.base.write(s)
-
-            stdout_text_str = LenientStrStreamWrapper(stdout_text_transcoded)
-            stderr_text_str = LenientStrStreamWrapper(stderr_text_transcoded)
-
-            win_unicode_console.enable(stdout=stdout_text_str,
-                                       stderr=stderr_text_str)
+        win_unicode_console.enable()
 
     def init_io(self):
         if sys.platform not in {'win32', 'cli'}:
             return
 
         self.enable_win_unicode_console()
 
         import colorama
         colorama.init()
 
         # For some reason we make these wrappers around stdout/stderr.
         # For now, we need to reset them so all output gets coloured.
         # https://github.com/ipython/ipython/issues/8669
         # io.std* are deprecated, but don't show our own deprecation warnings
         # during initialization of the deprecated API.
         with warnings.catch_warnings():
             warnings.simplefilter('ignore', DeprecationWarning)
             io.stdout = io.IOStream(sys.stdout)
             io.stderr = io.IOStream(sys.stderr)
 
     def init_magics(self):
         super(TerminalInteractiveShell, self).init_magics()
         self.register_magics(TerminalMagics)
 
     def init_alias(self):
         # The parent class defines aliases that can be safely used with any
         # frontend.
         super(TerminalInteractiveShell, self).init_alias()
 
         # Now define aliases that only make sense on the terminal, because they
         # need direct access to the console in a way that we can't emulate in
         # GUI or web frontend
         if os.name == 'posix':
             for cmd in ['clear', 'more', 'less', 'man']:
                 self.alias_manager.soft_define_alias(cmd, cmd)
 
 
     def __init__(self, *args, **kwargs):
         super(TerminalInteractiveShell, self).__init__(*args, **kwargs)
         self.init_prompt_toolkit_cli()
         self.init_term_title()
         self.keep_running = True
 
         self.debugger_history = InMemoryHistory()
 
     def ask_exit(self):
         self.keep_running = False
 
     rl_next_input = None
 
     def pre_prompt(self):
         if self.rl_next_input:
             self.pt_cli.application.buffer.text = cast_unicode_py2(self.rl_next_input)
             self.rl_next_input = None
 
     def interact(self, display_banner=DISPLAY_BANNER_DEPRECATED):
 
         if display_banner is not DISPLAY_BANNER_DEPRECATED:
             warn('interact `display_banner` argument is deprecated since IPython 5.0. Call `show_banner()` if needed.', DeprecationWarning, stacklevel=2)
 
         self.keep_running = True
         while self.keep_running:
             print(self.separate_in, end='')
 
             try:
                 code = self.prompt_for_code()
             except EOFError:
                 if (not self.confirm_exit) \
                         or self.ask_yes_no('Do you really want to exit ([y]/n)?','y','n'):
                     self.ask_exit()
 
             else:
                 if code:
                     self.run_cell(code, store_history=True)
 
     def mainloop(self, display_banner=DISPLAY_BANNER_DEPRECATED):
         # An extra layer of protection in case someone mashing Ctrl-C breaks
         # out of our internal code.
         if display_banner is not DISPLAY_BANNER_DEPRECATED:
             warn('mainloop `display_banner` argument is deprecated since IPython 5.0. Call `show_banner()` if needed.', DeprecationWarning, stacklevel=2)
         while True:
             try:
                 self.interact()
                 break
             except KeyboardInterrupt:
                 print("\nKeyboardInterrupt escaped interact()\n")
 
     _inputhook = None
     def inputhook(self, context):
         if self._inputhook is not None:
             self._inputhook(context)
 
     active_eventloop = None
     def enable_gui(self, gui=None):
         if gui:
             self.active_eventloop, self._inputhook =\
                 get_inputhook_name_and_func(gui)
         else:
             self.active_eventloop = self._inputhook = None
 
     # Run !system commands directly, not through pipes, so terminal programs
     # work correctly.
     system = InteractiveShell.system_raw
 
     def auto_rewrite_input(self, cmd):
         """Overridden from the parent class to use fancy rewriting prompt"""
         if not self.show_rewritten_input:
             return
 
         tokens = self.prompts.rewrite_prompt_tokens()
         if self.pt_cli:
             self.pt_cli.print_tokens(tokens)
             print(cmd)
         else:
             prompt = ''.join(s for t, s in tokens)
             print(prompt, cmd, sep='')
 
     _prompts_before = None
     def switch_doctest_mode(self, mode):
         """Switch prompts to classic for %doctest_mode"""
         if mode:
             self._prompts_before = self.prompts
             self.prompts = ClassicPrompts(self)
         elif self._prompts_before:
             self.prompts = self._prompts_before
             self._prompts_before = None
         self._update_layout()
 
 
 InteractiveShellABC.register(TerminalInteractiveShell)
 
 if __name__ == '__main__':
     TerminalInteractiveShell.instance().interact()

IPython/terminal/pt_inputhooks/tk.py

@@ -1,95 +1,90 b''
 # Code borrowed from ptpython
 # https://github.com/jonathanslenders/ptpython/blob/86b71a89626114b18898a0af463978bdb32eeb70/ptpython/eventloop.py
 
 # Copyright (c) 2015, Jonathan Slenders
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without modification,
 # are permitted provided that the following conditions are met:
 #
 # * Redistributions of source code must retain the above copyright notice, this
 #   list of conditions and the following disclaimer.
 #
 # * Redistributions in binary form must reproduce the above copyright notice, this
 #   list of conditions and the following disclaimer in the documentation and/or
 #   other materials provided with the distribution.
 #
 # * Neither the name of the {organization} nor the names of its
 #   contributors may be used to endorse or promote products derived from
 #   this software without specific prior written permission.
 #
 # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
 # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 # DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
 # ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
 # ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 """
 Wrapper around the eventloop that gives some time to the Tkinter GUI to process
 events when it's loaded and while we are waiting for input at the REPL. This
 way we don't block the UI of for instance ``turtle`` and other Tk libraries.
 
 (Normally Tkinter registeres it's callbacks in ``PyOS_InputHook`` to integrate
 in readline. ``prompt-toolkit`` doesn't understand that input hook, but this
 will fix it for Tk.)
 """
 import time
 
-from IPython.utils.py3compat import PY3
-
 import _tkinter
-if PY3:
-    import tkinter
-else:
-    import Tkinter as tkinter  # Python 2
+import tkinter
 
 def inputhook(inputhook_context):
     """
     Inputhook for Tk.
     Run the Tk eventloop until prompt-toolkit needs to process the next input.
     """
     # Get the current TK application.
     root = tkinter._default_root
 
     def wait_using_filehandler():
         """
         Run the TK eventloop until the file handler that we got from the
         inputhook becomes readable.
         """
         # Add a handler that sets the stop flag when `prompt-toolkit` has input
         # to process.
         stop = [False]
         def done(*a):
             stop[0] = True
 
         root.createfilehandler(inputhook_context.fileno(), _tkinter.READABLE, done)
 
         # Run the TK event loop as long as we don't receive input.
         while root.dooneevent(_tkinter.ALL_EVENTS):
             if stop[0]:
                 break
 
         root.deletefilehandler(inputhook_context.fileno())
 
     def wait_using_polling():
         """
         Windows TK doesn't support 'createfilehandler'.
         So, run the TK eventloop and poll until input is ready.
         """
         while not inputhook_context.input_is_ready():
             while root.dooneevent(_tkinter.ALL_EVENTS | _tkinter.DONT_WAIT):
                  pass
             # Sleep to make the CPU idle, but not too long, so that the UI
             # stays responsive.
             time.sleep(.01)
 
     if root is not None:
         if hasattr(root, 'createfilehandler'):
             wait_using_filehandler()
         else:
             wait_using_polling()

IPython/testing/decorators.py

@@ -1,378 +1,378 b''
 # -*- coding: utf-8 -*-
 """Decorators for labeling test objects.
 
 Decorators that merely return a modified version of the original function
 object are straightforward.  Decorators that return a new function object need
 to use nose.tools.make_decorator(original_function)(decorator) in returning the
 decorator, in order to preserve metadata such as function name, setup and
 teardown functions and so on - see nose.tools for more information.
 
 This module provides a set of useful decorators meant to be ready to use in
 your own tests.  See the bottom of the file for the ready-made ones, and if you
 find yourself writing a new one that may be of generic use, add it here.
 
 Included decorators:
 
 
 Lightweight testing that remains unittest-compatible.
 
 - An @as_unittest decorator can be used to tag any normal parameter-less
   function as a unittest TestCase.  Then, both nose and normal unittest will
   recognize it as such.  This will make it easier to migrate away from Nose if
   we ever need/want to while maintaining very lightweight tests.
 
 NOTE: This file contains IPython-specific decorators. Using the machinery in
 IPython.external.decorators, we import either numpy.testing.decorators if numpy is
 available, OR use equivalent code in IPython.external._decorators, which
 we've copied verbatim from numpy.
 
 """
 
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
 import sys
 import os
 import tempfile
 import unittest
 import warnings
 from importlib import import_module
 
 from decorator import decorator
 
 # Expose the unittest-driven decorators
 from .ipunittest import ipdoctest, ipdocstring
 
 # Grab the numpy-specific decorators which we keep in a file that we
 # occasionally update from upstream: decorators.py is a copy of
 # numpy.testing.decorators, we expose all of it here.
 from IPython.external.decorators import *
 
 # For onlyif_cmd_exists decorator
-from IPython.utils.py3compat import which, PY2, PY3, PYPY
+from IPython.utils.py3compat import which
 
 #-----------------------------------------------------------------------------
 # Classes and functions
 #-----------------------------------------------------------------------------
 
 # Simple example of the basic idea
 def as_unittest(func):
     """Decorator to make a simple function into a normal test via unittest."""
     class Tester(unittest.TestCase):
         def test(self):
             func()
 
     Tester.__name__ = func.__name__
 
     return Tester
 
 # Utility functions
 
 def apply_wrapper(wrapper, func):
     """Apply a wrapper to a function for decoration.
 
     This mixes Michele Simionato's decorator tool with nose's make_decorator,
     to apply a wrapper in a decorator so that all nose attributes, as well as
     function signature and other properties, survive the decoration cleanly.
     This will ensure that wrapped functions can still be well introspected via
     IPython, for example.
     """
     warnings.warn("The function `apply_wrapper` is deprecated since IPython 4.0",
             DeprecationWarning, stacklevel=2)
     import nose.tools
 
     return decorator(wrapper,nose.tools.make_decorator(func)(wrapper))
 
 
 def make_label_dec(label, ds=None):
     """Factory function to create a decorator that applies one or more labels.
 
     Parameters
     ----------
       label : string or sequence
       One or more labels that will be applied by the decorator to the functions
     it decorates.  Labels are attributes of the decorated function with their
     value set to True.
 
       ds : string
       An optional docstring for the resulting decorator.  If not given, a
       default docstring is auto-generated.
 
     Returns
     -------
       A decorator.
 
     Examples
     --------
 
     A simple labeling decorator:
 
     >>> slow = make_label_dec('slow')
     >>> slow.__doc__
     "Labels a test as 'slow'."
     
     And one that uses multiple labels and a custom docstring:
     
     >>> rare = make_label_dec(['slow','hard'],
     ... "Mix labels 'slow' and 'hard' for rare tests.")
     >>> rare.__doc__
     "Mix labels 'slow' and 'hard' for rare tests."
 
     Now, let's test using this one:
     >>> @rare
     ... def f(): pass
     ...
     >>>
     >>> f.slow
     True
     >>> f.hard
     True
     """
 
     warnings.warn("The function `make_label_dec` is deprecated since IPython 4.0",
             DeprecationWarning, stacklevel=2)
     if isinstance(label, str):
         labels = [label]
     else:
         labels = label
 
     # Validate that the given label(s) are OK for use in setattr() by doing a
     # dry run on a dummy function.
     tmp = lambda : None
     for label in labels:
         setattr(tmp,label,True)
 
     # This is the actual decorator we'll return
     def decor(f):
         for label in labels:
             setattr(f,label,True)
         return f
 
     # Apply the user's docstring, or autogenerate a basic one
     if ds is None:
         ds = "Labels a test as %r." % label
     decor.__doc__ = ds
 
     return decor
 
 
 # Inspired by numpy's skipif, but uses the full apply_wrapper utility to
 # preserve function metadata better and allows the skip condition to be a
 # callable.
 def skipif(skip_condition, msg=None):
     ''' Make function raise SkipTest exception if skip_condition is true
 
     Parameters
     ----------
 
     skip_condition : bool or callable
       Flag to determine whether to skip test. If the condition is a
       callable, it is used at runtime to dynamically make the decision. This
       is useful for tests that may require costly imports, to delay the cost
       until the test suite is actually executed.
     msg : string
       Message to give on raising a SkipTest exception.
 
     Returns
     -------
     decorator : function
       Decorator, which, when applied to a function, causes SkipTest
       to be raised when the skip_condition was True, and the function
       to be called normally otherwise.
 
     Notes
     -----
     You will see from the code that we had to further decorate the
     decorator with the nose.tools.make_decorator function in order to
     transmit function name, and various other metadata.
     '''
 
     def skip_decorator(f):
         # Local import to avoid a hard nose dependency and only incur the
         # import time overhead at actual test-time.
         import nose
 
         # Allow for both boolean or callable skip conditions.
         if callable(skip_condition):
             skip_val = skip_condition
         else:
             skip_val = lambda : skip_condition
 
         def get_msg(func,msg=None):
             """Skip message with information about function being skipped."""
             if msg is None: out = 'Test skipped due to test condition.'
             else: out = msg
             return "Skipping test: %s. %s" % (func.__name__,out)
 
         # We need to define *two* skippers because Python doesn't allow both
         # return with value and yield inside the same function.
         def skipper_func(*args, **kwargs):
             """Skipper for normal test functions."""
             if skip_val():
                 raise nose.SkipTest(get_msg(f,msg))
             else:
                 return f(*args, **kwargs)
 
         def skipper_gen(*args, **kwargs):
             """Skipper for test generators."""
             if skip_val():
                 raise nose.SkipTest(get_msg(f,msg))
             else:
                 for x in f(*args, **kwargs):
                     yield x
 
         # Choose the right skipper to use when building the actual generator.
         if nose.util.isgenerator(f):
             skipper = skipper_gen
         else:
             skipper = skipper_func
 
         return nose.tools.make_decorator(f)(skipper)
 
     return skip_decorator
 
 # A version with the condition set to true, common case just to attach a message
 # to a skip decorator
 def skip(msg=None):
     """Decorator factory - mark a test function for skipping from test suite.
 
     Parameters
     ----------
       msg : string
         Optional message to be added.
 
     Returns
     -------
        decorator : function
          Decorator, which, when applied to a function, causes SkipTest
          to be raised, with the optional message added.
       """
 
     return skipif(True,msg)
 
 
 def onlyif(condition, msg):
     """The reverse from skipif, see skipif for details."""
 
     if callable(condition):
         skip_condition = lambda : not condition()
     else:
         skip_condition = lambda : not condition
 
     return skipif(skip_condition, msg)
 
 #-----------------------------------------------------------------------------
 # Utility functions for decorators
 def module_not_available(module):
     """Can module be imported?  Returns true if module does NOT import.
 
     This is used to make a decorator to skip tests that require module to be
     available, but delay the 'import numpy' to test execution time.
     """
     try:
         mod = import_module(module)
         mod_not_avail = False
     except ImportError:
         mod_not_avail = True
 
     return mod_not_avail
 
 
 def decorated_dummy(dec, name):
     """Return a dummy function decorated with dec, with the given name.
     
     Examples
     --------
     import IPython.testing.decorators as dec
     setup = dec.decorated_dummy(dec.skip_if_no_x11, __name__)
     """
     warnings.warn("The function `decorated_dummy` is deprecated since IPython 4.0",
         DeprecationWarning, stacklevel=2)
     dummy = lambda: None
     dummy.__name__ = name
     return dec(dummy)
 
 #-----------------------------------------------------------------------------
 # Decorators for public use
 
 # Decorators to skip certain tests on specific platforms.
 skip_win32 = skipif(sys.platform == 'win32',
                     "This test does not run under Windows")
 skip_linux = skipif(sys.platform.startswith('linux'),
                     "This test does not run under Linux")
 skip_osx = skipif(sys.platform == 'darwin',"This test does not run under OS X")
 
 
 # Decorators to skip tests if not on specific platforms.
 skip_if_not_win32 = skipif(sys.platform != 'win32',
                            "This test only runs under Windows")
 skip_if_not_linux = skipif(not sys.platform.startswith('linux'),
                            "This test only runs under Linux")
 skip_if_not_osx = skipif(sys.platform != 'darwin',
                          "This test only runs under OSX")
 
 
 _x11_skip_cond = (sys.platform not in ('darwin', 'win32') and
                   os.environ.get('DISPLAY', '') == '')
 _x11_skip_msg = "Skipped under *nix when X11/XOrg not available"
 
 skip_if_no_x11 = skipif(_x11_skip_cond, _x11_skip_msg)
 
 # not a decorator itself, returns a dummy function to be used as setup
 def skip_file_no_x11(name):
     warnings.warn("The function `skip_file_no_x11` is deprecated since IPython 4.0",
             DeprecationWarning, stacklevel=2)
     return decorated_dummy(skip_if_no_x11, name) if _x11_skip_cond else None
 
 # Other skip decorators
 
 # generic skip without module
 skip_without = lambda mod: skipif(module_not_available(mod), "This test requires %s" % mod)
 
 skipif_not_numpy = skip_without('numpy')
 
 skipif_not_matplotlib = skip_without('matplotlib')
 
 skipif_not_sympy = skip_without('sympy')
 
 skip_known_failure = knownfailureif(True,'This test is known to fail')
 
 # A null 'decorator', useful to make more readable code that needs to pick
 # between different decorators based on OS or other conditions
 null_deco = lambda f: f
 
 # Some tests only run where we can use unicode paths. Note that we can't just
 # check os.path.supports_unicode_filenames, which is always False on Linux.
 try:
     f = tempfile.NamedTemporaryFile(prefix=u"tmp€")
 except UnicodeEncodeError:
     unicode_paths = False
 else:
     unicode_paths = True
     f.close()
 
 onlyif_unicode_paths = onlyif(unicode_paths, ("This test is only applicable "
                                     "where we can use unicode in filenames."))
 
 
 def onlyif_cmds_exist(*commands):
     """
     Decorator to skip test when at least one of `commands` is not found.
     """
     for cmd in commands:
         if not which(cmd):
             return skip("This test runs only if command '{0}' "
                         "is installed".format(cmd))
     return null_deco
 
 def onlyif_any_cmd_exists(*commands):
     """
     Decorator to skip test unless at least one of `commands` is found.
     """
     warnings.warn("The function `onlyif_any_cmd_exists` is deprecated since IPython 4.0",
             DeprecationWarning, stacklevel=2)
     for cmd in commands:
         if which(cmd):
             return null_deco
     return skip("This test runs only if one of the commands {0} "
                 "is installed".format(commands))

IPython/testing/plugin/ipdoctest.py

@@ -1,770 +1,766 b''
 """Nose Plugin that supports IPython doctests.
 
 Limitations:
 
 - When generating examples for use as doctests, make sure that you have
   pretty-printing OFF.  This can be done either by setting the
   ``PlainTextFormatter.pprint`` option in your configuration file to  False, or
   by interactively disabling it with  %Pprint.  This is required so that IPython
   output matches that of normal Python, which is used by doctest for internal
   execution.
 
 - Do not rely on specific prompt numbers for results (such as using
   '_34==True', for example).  For IPython tests run via an external process the
   prompt numbers may be different, and IPython tests run as normal python code
   won't even have these special _NN variables set at all.
 """
 
 #-----------------------------------------------------------------------------
 # Module imports
 
 # From the standard library
 import doctest
 import inspect
 import logging
 import os
 import re
 import sys
 from importlib import import_module
+from io import StringIO
 
 from testpath import modified_env
 
 from inspect import getmodule
 
 # We are overriding the default doctest runner, so we need to import a few
 # things from doctest directly
 from doctest import (REPORTING_FLAGS, REPORT_ONLY_FIRST_FAILURE,
                      _unittest_reportflags, DocTestRunner,
                      _extract_future_flags, pdb, _OutputRedirectingPdb,
                      _exception_traceback,
                      linecache)
 
 # Third-party modules
 
 from nose.plugins import doctests, Plugin
 from nose.util import anyp, tolist
 
 # Our own imports
-from IPython.utils.py3compat import builtin_mod, PY3
-
-if PY3:
-    from io import StringIO
-else:
-    from StringIO import StringIO
+from IPython.utils.py3compat import builtin_mod
 
 #-----------------------------------------------------------------------------
 # Module globals and other constants
 #-----------------------------------------------------------------------------
 
 log = logging.getLogger(__name__)
 
 
 #-----------------------------------------------------------------------------
 # Classes and functions
 #-----------------------------------------------------------------------------
 
 def is_extension_module(filename):
     """Return whether the given filename is an extension module.
 
     This simply checks that the extension is either .so or .pyd.
     """
     return os.path.splitext(filename)[1].lower() in ('.so','.pyd')
 
 
 class DocTestSkip(object):
     """Object wrapper for doctests to be skipped."""
 
     ds_skip = """Doctest to skip.
     >>> 1 #doctest: +SKIP
     """
 
     def __init__(self,obj):
         self.obj = obj
 
     def __getattribute__(self,key):
         if key == '__doc__':
             return DocTestSkip.ds_skip
         else:
             return getattr(object.__getattribute__(self,'obj'),key)
 
 # Modified version of the one in the stdlib, that fixes a python bug (doctests
 # not found in extension modules, http://bugs.python.org/issue3158)
 class DocTestFinder(doctest.DocTestFinder):
 
     def _from_module(self, module, object):
         """
         Return true if the given object is defined in the given
         module.
         """
         if module is None:
             return True
         elif inspect.isfunction(object):
             return module.__dict__ is object.__globals__
         elif inspect.isbuiltin(object):
             return module.__name__ == object.__module__
         elif inspect.isclass(object):
             return module.__name__ == object.__module__
         elif inspect.ismethod(object):
             # This one may be a bug in cython that fails to correctly set the
             # __module__ attribute of methods, but since the same error is easy
             # to make by extension code writers, having this safety in place
             # isn't such a bad idea
             return module.__name__ == object.__self__.__class__.__module__
         elif inspect.getmodule(object) is not None:
             return module is inspect.getmodule(object)
         elif hasattr(object, '__module__'):
             return module.__name__ == object.__module__
         elif isinstance(object, property):
             return True # [XX] no way not be sure.
         elif inspect.ismethoddescriptor(object):
             # Unbound PyQt signals reach this point in Python 3.4b3, and we want
             # to avoid throwing an error. See also http://bugs.python.org/issue3158
             return False
         else:
             raise ValueError("object must be a class or function, got %r" % object)
 
     def _find(self, tests, obj, name, module, source_lines, globs, seen):
         """
         Find tests for the given object and any contained objects, and
         add them to `tests`.
         """
         print('_find for:', obj, name, module)  # dbg
         if hasattr(obj,"skip_doctest"):
             #print 'SKIPPING DOCTEST FOR:',obj  # dbg
             obj = DocTestSkip(obj)
         
         doctest.DocTestFinder._find(self,tests, obj, name, module,
                                     source_lines, globs, seen)
 
         # Below we re-run pieces of the above method with manual modifications,
         # because the original code is buggy and fails to correctly identify
         # doctests in extension modules.
 
         # Local shorthands
         from inspect import isroutine, isclass
 
         # Look for tests in a module's contained objects.
         if inspect.ismodule(obj) and self._recurse:
             for valname, val in obj.__dict__.items():
                 valname1 = '%s.%s' % (name, valname)
                 if ( (isroutine(val) or isclass(val))
                      and self._from_module(module, val) ):
 
                     self._find(tests, val, valname1, module, source_lines,
                                globs, seen)
 
         # Look for tests in a class's contained objects.
         if inspect.isclass(obj) and self._recurse:
             #print 'RECURSE into class:',obj  # dbg
             for valname, val in obj.__dict__.items():
                 # Special handling for staticmethod/classmethod.
                 if isinstance(val, staticmethod):
                     val = getattr(obj, valname)
                 if isinstance(val, classmethod):
                     val = getattr(obj, valname).__func__
 
                 # Recurse to methods, properties, and nested classes.
                 if ((inspect.isfunction(val) or inspect.isclass(val) or
                      inspect.ismethod(val) or
                       isinstance(val, property)) and
                       self._from_module(module, val)):
                     valname = '%s.%s' % (name, valname)
                     self._find(tests, val, valname, module, source_lines,
                                globs, seen)
 
 
 class IPDoctestOutputChecker(doctest.OutputChecker):
     """Second-chance checker with support for random tests.
 
     If the default comparison doesn't pass, this checker looks in the expected
     output string for flags that tell us to ignore the output.
     """
 
     random_re = re.compile(r'#\s*random\s+')
 
     def check_output(self, want, got, optionflags):
         """Check output, accepting special markers embedded in the output.
 
         If the output didn't pass the default validation but the special string
         '#random' is included, we accept it."""
 
         # Let the original tester verify first, in case people have valid tests
         # that happen to have a comment saying '#random' embedded in.
         ret = doctest.OutputChecker.check_output(self, want, got,
                                                  optionflags)
         if not ret and self.random_re.search(want):
             #print >> sys.stderr, 'RANDOM OK:',want  # dbg
             return True
 
         return ret
 
 
 class DocTestCase(doctests.DocTestCase):
     """Proxy for DocTestCase: provides an address() method that
     returns the correct address for the doctest case. Otherwise
     acts as a proxy to the test case. To provide hints for address(),
     an obj may also be passed -- this will be used as the test object
     for purposes of determining the test address, if it is provided.
     """
 
     # Note: this method was taken from numpy's nosetester module.
 
     # Subclass nose.plugins.doctests.DocTestCase to work around a bug in
     # its constructor that blocks non-default arguments from being passed
     # down into doctest.DocTestCase
 
     def __init__(self, test, optionflags=0, setUp=None, tearDown=None,
                  checker=None, obj=None, result_var='_'):
         self._result_var = result_var
         doctests.DocTestCase.__init__(self, test,
                                       optionflags=optionflags,
                                       setUp=setUp, tearDown=tearDown,
                                       checker=checker)
         # Now we must actually copy the original constructor from the stdlib
         # doctest class, because we can't call it directly and a bug in nose
         # means it never gets passed the right arguments.
 
         self._dt_optionflags = optionflags
         self._dt_checker = checker
         self._dt_test = test
         self._dt_test_globs_ori = test.globs
         self._dt_setUp = setUp
         self._dt_tearDown = tearDown
 
         # XXX - store this runner once in the object!
         runner = IPDocTestRunner(optionflags=optionflags,
                                  checker=checker, verbose=False)
         self._dt_runner = runner
 
 
         # Each doctest should remember the directory it was loaded from, so
         # things like %run work without too many contortions
         self._ori_dir = os.path.dirname(test.filename)
 
     # Modified runTest from the default stdlib
     def runTest(self):
         test = self._dt_test
         runner = self._dt_runner
 
         old = sys.stdout
         new = StringIO()
         optionflags = self._dt_optionflags
 
         if not (optionflags & REPORTING_FLAGS):
             # The option flags don't include any reporting flags,
             # so add the default reporting flags
             optionflags |= _unittest_reportflags
 
         try:
             # Save our current directory and switch out to the one where the
             # test was originally created, in case another doctest did a
             # directory change.  We'll restore this in the finally clause.
             curdir = os.getcwd()
             #print 'runTest in dir:', self._ori_dir  # dbg
             os.chdir(self._ori_dir)
 
             runner.DIVIDER = "-"*70
             failures, tries = runner.run(test,out=new.write,
                                          clear_globs=False)
         finally:
             sys.stdout = old
             os.chdir(curdir)
 
         if failures:
             raise self.failureException(self.format_failure(new.getvalue()))
 
     def setUp(self):
         """Modified test setup that syncs with ipython namespace"""
         #print "setUp test", self._dt_test.examples # dbg
         if isinstance(self._dt_test.examples[0], IPExample):
             # for IPython examples *only*, we swap the globals with the ipython
             # namespace, after updating it with the globals (which doctest
             # fills with the necessary info from the module being tested).
             self.user_ns_orig = {}
             self.user_ns_orig.update(_ip.user_ns)
             _ip.user_ns.update(self._dt_test.globs)
             # We must remove the _ key in the namespace, so that Python's
             # doctest code sets it naturally
             _ip.user_ns.pop('_', None)
             _ip.user_ns['__builtins__'] = builtin_mod
             self._dt_test.globs = _ip.user_ns
 
         super(DocTestCase, self).setUp()
 
     def tearDown(self):
 
         # Undo the test.globs reassignment we made, so that the parent class
         # teardown doesn't destroy the ipython namespace
         if isinstance(self._dt_test.examples[0], IPExample):
             self._dt_test.globs = self._dt_test_globs_ori
             _ip.user_ns.clear()
             _ip.user_ns.update(self.user_ns_orig)
 
         # XXX - fperez: I am not sure if this is truly a bug in nose 0.11, but
         # it does look like one to me: its tearDown method tries to run
         #
         # delattr(builtin_mod, self._result_var)
         #
         # without checking that the attribute really is there; it implicitly
         # assumes it should have been set via displayhook.  But if the
         # displayhook was never called, this doesn't necessarily happen.  I
         # haven't been able to find a little self-contained example outside of
         # ipython that would show the problem so I can report it to the nose
         # team, but it does happen a lot in our code.
         #
         # So here, we just protect as narrowly as possible by trapping an
         # attribute error whose message would be the name of self._result_var,
         # and letting any other error propagate.
         try:
             super(DocTestCase, self).tearDown()
         except AttributeError as exc:
             if exc.args[0] != self._result_var:
                 raise
 
 
 # A simple subclassing of the original with a different class name, so we can
 # distinguish and treat differently IPython examples from pure python ones.
 class IPExample(doctest.Example): pass
 
 
 class IPExternalExample(doctest.Example):
     """Doctest examples to be run in an external process."""
 
     def __init__(self, source, want, exc_msg=None, lineno=0, indent=0,
                  options=None):
         # Parent constructor
         doctest.Example.__init__(self,source,want,exc_msg,lineno,indent,options)
 
         # An EXTRA newline is needed to prevent pexpect hangs
         self.source += '\n'
 
 
 class IPDocTestParser(doctest.DocTestParser):
     """
     A class used to parse strings containing doctest examples.
 
     Note: This is a version modified to properly recognize IPython input and
     convert any IPython examples into valid Python ones.
     """
     # This regular expression is used to find doctest examples in a
     # string.  It defines three groups: `source` is the source code
     # (including leading indentation and prompts); `indent` is the
     # indentation of the first (PS1) line of the source code; and
     # `want` is the expected output (including leading indentation).
 
     # Classic Python prompts or default IPython ones
     _PS1_PY = r'>>>'
     _PS2_PY = r'\.\.\.'
 
     _PS1_IP = r'In\ \[\d+\]:'
     _PS2_IP = r'\ \ \ \.\.\.+:'
 
     _RE_TPL = r'''
         # Source consists of a PS1 line followed by zero or more PS2 lines.
         (?P<source>
             (?:^(?P<indent> [ ]*) (?P<ps1> %s) .*)    # PS1 line
             (?:\n           [ ]*  (?P<ps2> %s) .*)*)  # PS2 lines
         \n? # a newline
         # Want consists of any non-blank lines that do not start with PS1.
         (?P<want> (?:(?![ ]*$)    # Not a blank line
                      (?![ ]*%s)   # Not a line starting with PS1
                      (?![ ]*%s)   # Not a line starting with PS2
                      .*$\n?       # But any other line
                   )*)
                   '''
 
     _EXAMPLE_RE_PY = re.compile( _RE_TPL % (_PS1_PY,_PS2_PY,_PS1_PY,_PS2_PY),
                                  re.MULTILINE | re.VERBOSE)
 
     _EXAMPLE_RE_IP = re.compile( _RE_TPL % (_PS1_IP,_PS2_IP,_PS1_IP,_PS2_IP),
                                  re.MULTILINE | re.VERBOSE)
 
     # Mark a test as being fully random.  In this case, we simply append the
     # random marker ('#random') to each individual example's output.  This way
     # we don't need to modify any other code.
     _RANDOM_TEST = re.compile(r'#\s*all-random\s+')
 
     # Mark tests to be executed in an external process - currently unsupported.
     _EXTERNAL_IP = re.compile(r'#\s*ipdoctest:\s*EXTERNAL')
 
     def ip2py(self,source):
         """Convert input IPython source into valid Python."""
         block = _ip.input_transformer_manager.transform_cell(source)
         if len(block.splitlines()) == 1:
             return _ip.prefilter(block)
         else:
             return block
 
     def parse(self, string, name='<string>'):
         """
         Divide the given string into examples and intervening text,
         and return them as a list of alternating Examples and strings.
         Line numbers for the Examples are 0-based.  The optional
         argument `name` is a name identifying this string, and is only
         used for error messages.
         """
 
         #print 'Parse string:\n',string # dbg
 
         string = string.expandtabs()
         # If all lines begin with the same indentation, then strip it.
         min_indent = self._min_indent(string)
         if min_indent > 0:
             string = '\n'.join([l[min_indent:] for l in string.split('\n')])
 
         output = []
         charno, lineno = 0, 0
 
         # We make 'all random' tests by adding the '# random' mark to every
         # block of output in the test.
         if self._RANDOM_TEST.search(string):
             random_marker = '\n# random'
         else:
             random_marker = ''
 
         # Whether to convert the input from ipython to python syntax
         ip2py = False
         # Find all doctest examples in the string.  First, try them as Python
         # examples, then as IPython ones
         terms = list(self._EXAMPLE_RE_PY.finditer(string))
         if terms:
             # Normal Python example
             #print '-'*70  # dbg
             #print 'PyExample, Source:\n',string  # dbg
             #print '-'*70  # dbg
             Example = doctest.Example
         else:
             # It's an ipython example.  Note that IPExamples are run
             # in-process, so their syntax must be turned into valid python.
             # IPExternalExamples are run out-of-process (via pexpect) so they
             # don't need any filtering (a real ipython will be executing them).
             terms = list(self._EXAMPLE_RE_IP.finditer(string))
             if self._EXTERNAL_IP.search(string):
                 #print '-'*70  # dbg
                 #print 'IPExternalExample, Source:\n',string  # dbg
                 #print '-'*70  # dbg
                 Example = IPExternalExample
             else:
                 #print '-'*70  # dbg
                 #print 'IPExample, Source:\n',string  # dbg
                 #print '-'*70  # dbg
                 Example = IPExample
                 ip2py = True
 
         for m in terms:
             # Add the pre-example text to `output`.
             output.append(string[charno:m.start()])
             # Update lineno (lines before this example)
             lineno += string.count('\n', charno, m.start())
             # Extract info from the regexp match.
             (source, options, want, exc_msg) = \
                      self._parse_example(m, name, lineno,ip2py)
 
             # Append the random-output marker (it defaults to empty in most
             # cases, it's only non-empty for 'all-random' tests):
             want += random_marker
 
             if Example is IPExternalExample:
                 options[doctest.NORMALIZE_WHITESPACE] = True
                 want += '\n'
 
             # Create an Example, and add it to the list.
             if not self._IS_BLANK_OR_COMMENT(source):
                 output.append(Example(source, want, exc_msg,
                                       lineno=lineno,
                                       indent=min_indent+len(m.group('indent')),
                                       options=options))
             # Update lineno (lines inside this example)
             lineno += string.count('\n', m.start(), m.end())
             # Update charno.
             charno = m.end()
         # Add any remaining post-example text to `output`.
         output.append(string[charno:])
         return output
 
     def _parse_example(self, m, name, lineno,ip2py=False):
         """
         Given a regular expression match from `_EXAMPLE_RE` (`m`),
         return a pair `(source, want)`, where `source` is the matched
         example's source code (with prompts and indentation stripped);
         and `want` is the example's expected output (with indentation
         stripped).
 
         `name` is the string's name, and `lineno` is the line number
         where the example starts; both are used for error messages.
 
         Optional:
         `ip2py`: if true, filter the input via IPython to convert the syntax
         into valid python.
         """
 
         # Get the example's indentation level.
         indent = len(m.group('indent'))
 
         # Divide source into lines; check that they're properly
         # indented; and then strip their indentation & prompts.
         source_lines = m.group('source').split('\n')
 
         # We're using variable-length input prompts
         ps1 = m.group('ps1')
         ps2 = m.group('ps2')
         ps1_len = len(ps1)
 
         self._check_prompt_blank(source_lines, indent, name, lineno,ps1_len)
         if ps2:
             self._check_prefix(source_lines[1:], ' '*indent + ps2, name, lineno)
 
         source = '\n'.join([sl[indent+ps1_len+1:] for sl in source_lines])
 
         if ip2py:
             # Convert source input from IPython into valid Python syntax
             source = self.ip2py(source)
 
         # Divide want into lines; check that it's properly indented; and
         # then strip the indentation.  Spaces before the last newline should
         # be preserved, so plain rstrip() isn't good enough.
         want = m.group('want')
         want_lines = want.split('\n')
         if len(want_lines) > 1 and re.match(r' *$', want_lines[-1]):
             del want_lines[-1]  # forget final newline & spaces after it
         self._check_prefix(want_lines, ' '*indent, name,
                            lineno + len(source_lines))
 
         # Remove ipython output prompt that might be present in the first line
         want_lines[0] = re.sub(r'Out\[\d+\]: \s*?\n?','',want_lines[0])
 
         want = '\n'.join([wl[indent:] for wl in want_lines])
 
         # If `want` contains a traceback message, then extract it.
         m = self._EXCEPTION_RE.match(want)
         if m:
             exc_msg = m.group('msg')
         else:
             exc_msg = None
 
         # Extract options from the source.
         options = self._find_options(source, name, lineno)
 
         return source, options, want, exc_msg
 
     def _check_prompt_blank(self, lines, indent, name, lineno, ps1_len):
         """
         Given the lines of a source string (including prompts and
         leading indentation), check to make sure that every prompt is
         followed by a space character.  If any line is not followed by
         a space character, then raise ValueError.
 
         Note: IPython-modified version which takes the input prompt length as a
         parameter, so that prompts of variable length can be dealt with.
         """
         space_idx = indent+ps1_len
         min_len = space_idx+1
         for i, line in enumerate(lines):
             if len(line) >=  min_len and line[space_idx] != ' ':
                 raise ValueError('line %r of the docstring for %s '
                                  'lacks blank after %s: %r' %
                                  (lineno+i+1, name,
                                   line[indent:space_idx], line))
 
 
 SKIP = doctest.register_optionflag('SKIP')
 
 
 class IPDocTestRunner(doctest.DocTestRunner,object):
     """Test runner that synchronizes the IPython namespace with test globals.
     """
 
     def run(self, test, compileflags=None, out=None, clear_globs=True):
 
         # Hack: ipython needs access to the execution context of the example,
         # so that it can propagate user variables loaded by %run into
         # test.globs.  We put them here into our modified %run as a function
         # attribute.  Our new %run will then only make the namespace update
         # when called (rather than unconconditionally updating test.globs here
         # for all examples, most of which won't be calling %run anyway).
         #_ip._ipdoctest_test_globs = test.globs
         #_ip._ipdoctest_test_filename = test.filename
 
         test.globs.update(_ip.user_ns)
 
         # Override terminal size to standardise traceback format
         with modified_env({'COLUMNS': '80', 'LINES': '24'}):
             return super(IPDocTestRunner,self).run(test,
                                                    compileflags,out,clear_globs)
 
 
 class DocFileCase(doctest.DocFileCase):
     """Overrides to provide filename
     """
     def address(self):
         return (self._dt_test.filename, None, None)
 
 
 class ExtensionDoctest(doctests.Doctest):
     """Nose Plugin that supports doctests in extension modules.
     """
     name = 'extdoctest'   # call nosetests with --with-extdoctest
     enabled = True
 
     def options(self, parser, env=os.environ):
         Plugin.options(self, parser, env)
         parser.add_option('--doctest-tests', action='store_true',
                           dest='doctest_tests',
                           default=env.get('NOSE_DOCTEST_TESTS',True),
                           help="Also look for doctests in test modules. "
                           "Note that classes, methods and functions should "
                           "have either doctests or non-doctest tests, "
                           "not both. [NOSE_DOCTEST_TESTS]")
         parser.add_option('--doctest-extension', action="append",
                           dest="doctestExtension",
                           help="Also look for doctests in files with "
                           "this extension [NOSE_DOCTEST_EXTENSION]")
         # Set the default as a list, if given in env; otherwise
         # an additional value set on the command line will cause
         # an error.
         env_setting = env.get('NOSE_DOCTEST_EXTENSION')
         if env_setting is not None:
             parser.set_defaults(doctestExtension=tolist(env_setting))
 
 
     def configure(self, options, config):
         Plugin.configure(self, options, config)
         # Pull standard doctest plugin out of config; we will do doctesting
         config.plugins.plugins = [p for p in config.plugins.plugins
                                   if p.name != 'doctest']
         self.doctest_tests = options.doctest_tests
         self.extension = tolist(options.doctestExtension)
 
         self.parser = doctest.DocTestParser()
         self.finder = DocTestFinder()
         self.checker = IPDoctestOutputChecker()
         self.globs = None
         self.extraglobs = None
 
 
     def loadTestsFromExtensionModule(self,filename):
         bpath,mod = os.path.split(filename)
         modname = os.path.splitext(mod)[0]
         try:
             sys.path.append(bpath)
             module = import_module(modname)
             tests = list(self.loadTestsFromModule(module))
         finally:
             sys.path.pop()
         return tests
 
     # NOTE: the method below is almost a copy of the original one in nose, with
     # a  few modifications to control output checking.
 
     def loadTestsFromModule(self, module):
         #print '*** ipdoctest - lTM',module  # dbg
 
         if not self.matches(module.__name__):
             log.debug("Doctest doesn't want module %s", module)
             return
 
         tests = self.finder.find(module,globs=self.globs,
                                  extraglobs=self.extraglobs)
         if not tests:
             return
 
         # always use whitespace and ellipsis options
         optionflags = doctest.NORMALIZE_WHITESPACE | doctest.ELLIPSIS
 
         tests.sort()
         module_file = module.__file__
         if module_file[-4:] in ('.pyc', '.pyo'):
             module_file = module_file[:-1]
         for test in tests:
             if not test.examples:
                 continue
             if not test.filename:
                 test.filename = module_file
 
             yield DocTestCase(test,
                               optionflags=optionflags,
                               checker=self.checker)
 
 
     def loadTestsFromFile(self, filename):
         #print "ipdoctest - from file", filename # dbg
         if is_extension_module(filename):
             for t in self.loadTestsFromExtensionModule(filename):
                 yield t
         else:
             if self.extension and anyp(filename.endswith, self.extension):
                 name = os.path.basename(filename)
                 dh = open(filename)
                 try:
                     doc = dh.read()
                 finally:
                     dh.close()
                 test = self.parser.get_doctest(
                     doc, globs={'__file__': filename}, name=name,
                     filename=filename, lineno=0)
                 if test.examples:
                     #print 'FileCase:',test.examples  # dbg
                     yield DocFileCase(test)
                 else:
                     yield False # no tests to load
 
 
 class IPythonDoctest(ExtensionDoctest):
     """Nose Plugin that supports doctests in extension modules.
     """
     name = 'ipdoctest'   # call nosetests with --with-ipdoctest
     enabled = True
 
     def makeTest(self, obj, parent):
         """Look for doctests in the given object, which will be a
         function, method or class.
         """
         #print 'Plugin analyzing:', obj, parent  # dbg
         # always use whitespace and ellipsis options
         optionflags = doctest.NORMALIZE_WHITESPACE | doctest.ELLIPSIS
 
         doctests = self.finder.find(obj, module=getmodule(parent))
         if doctests:
             for test in doctests:
                 if len(test.examples) == 0:
                     continue
 
                 yield DocTestCase(test, obj=obj,
                                   optionflags=optionflags,
                                   checker=self.checker)
 
     def options(self, parser, env=os.environ):
         #print "Options for nose plugin:", self.name # dbg
         Plugin.options(self, parser, env)
         parser.add_option('--ipdoctest-tests', action='store_true',
                           dest='ipdoctest_tests',
                           default=env.get('NOSE_IPDOCTEST_TESTS',True),
                           help="Also look for doctests in test modules. "
                           "Note that classes, methods and functions should "
                           "have either doctests or non-doctest tests, "
                           "not both. [NOSE_IPDOCTEST_TESTS]")
         parser.add_option('--ipdoctest-extension', action="append",
                           dest="ipdoctest_extension",
                           help="Also look for doctests in files with "
                           "this extension [NOSE_IPDOCTEST_EXTENSION]")
         # Set the default as a list, if given in env; otherwise
         # an additional value set on the command line will cause
         # an error.
         env_setting = env.get('NOSE_IPDOCTEST_EXTENSION')
         if env_setting is not None:
             parser.set_defaults(ipdoctest_extension=tolist(env_setting))
 
     def configure(self, options, config):
         #print "Configuring nose plugin:", self.name # dbg
         Plugin.configure(self, options, config)
         # Pull standard doctest plugin out of config; we will do doctesting
         config.plugins.plugins = [p for p in config.plugins.plugins
                                   if p.name != 'doctest']
         self.doctest_tests = options.ipdoctest_tests
         self.extension = tolist(options.ipdoctest_extension)
 
         self.parser = IPDocTestParser()
         self.finder = DocTestFinder(parser=self.parser)
         self.checker = IPDoctestOutputChecker()
         self.globs = None
         self.extraglobs = None

IPython/testing/tools.py

@@ -1,476 +1,468 b''
 """Generic testing tools.
 
 Authors
 -------
 - Fernando Perez <[e-mail]>
 """
 
 
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
 import os
 import re
 import sys
 import tempfile
 
 from contextlib import contextmanager
 from io import StringIO
 from subprocess import Popen, PIPE
 from unittest.mock import patch
 
 try:
     # These tools are used by parts of the runtime, so we make the nose
     # dependency optional at this point.  Nose is a hard dependency to run the
     # test suite, but NOT to use ipython itself.
     import nose.tools as nt
     has_nose = True
 except ImportError:
     has_nose = False
 
 from traitlets.config.loader import Config
 from IPython.utils.process import get_output_error_code
 from IPython.utils.text import list_strings
 from IPython.utils.io import temp_pyfile, Tee
 from IPython.utils import py3compat
 from IPython.utils.encoding import DEFAULT_ENCODING
 
 from . import decorators as dec
 from . import skipdoctest
 
 
 # The docstring for full_path doctests differently on win32 (different path
 # separator) so just skip the doctest there.  The example remains informative.
 doctest_deco = skipdoctest.skip_doctest if sys.platform == 'win32' else dec.null_deco
 
 @doctest_deco
 def full_path(startPath,files):
     """Make full paths for all the listed files, based on startPath.
 
     Only the base part of startPath is kept, since this routine is typically
     used with a script's ``__file__`` variable as startPath. The base of startPath
     is then prepended to all the listed files, forming the output list.
 
     Parameters
     ----------
     startPath : string
       Initial path to use as the base for the results.  This path is split
       using os.path.split() and only its first component is kept.
 
     files : string or list
       One or more files.
 
     Examples
     --------
 
     >>> full_path('/foo/bar.py',['a.txt','b.txt'])
     ['/foo/a.txt', '/foo/b.txt']
 
     >>> full_path('/foo',['a.txt','b.txt'])
     ['/a.txt', '/b.txt']
 
     If a single file is given, the output is still a list::
 
         >>> full_path('/foo','a.txt')
         ['/a.txt']
     """
 
     files = list_strings(files)
     base = os.path.split(startPath)[0]
     return [ os.path.join(base,f) for f in files ]
 
 
 def parse_test_output(txt):
     """Parse the output of a test run and return errors, failures.
 
     Parameters
     ----------
     txt : str
       Text output of a test run, assumed to contain a line of one of the
       following forms::
       
         'FAILED (errors=1)'
         'FAILED (failures=1)'
         'FAILED (errors=1, failures=1)'
 
     Returns
     -------
     nerr, nfail
       number of errors and failures.
     """
 
     err_m = re.search(r'^FAILED \(errors=(\d+)\)', txt, re.MULTILINE)
     if err_m:
         nerr = int(err_m.group(1))
         nfail = 0
         return  nerr, nfail
 
     fail_m = re.search(r'^FAILED \(failures=(\d+)\)', txt, re.MULTILINE)
     if fail_m:
         nerr = 0
         nfail = int(fail_m.group(1))
         return  nerr, nfail
 
     both_m = re.search(r'^FAILED \(errors=(\d+), failures=(\d+)\)', txt,
                        re.MULTILINE)
     if both_m:
         nerr = int(both_m.group(1))
         nfail = int(both_m.group(2))
         return  nerr, nfail
 
     # If the input didn't match any of these forms, assume no error/failures
     return 0, 0
 
 
 # So nose doesn't think this is a test
 parse_test_output.__test__ = False
 
 
 def default_argv():
     """Return a valid default argv for creating testing instances of ipython"""
 
     return ['--quick', # so no config file is loaded
             # Other defaults to minimize side effects on stdout
             '--colors=NoColor', '--no-term-title','--no-banner',
             '--autocall=0']
 
 
 def default_config():
     """Return a config object with good defaults for testing."""
     config = Config()
     config.TerminalInteractiveShell.colors = 'NoColor'
     config.TerminalTerminalInteractiveShell.term_title = False,
     config.TerminalInteractiveShell.autocall = 0
     f = tempfile.NamedTemporaryFile(suffix=u'test_hist.sqlite', delete=False)
     config.HistoryManager.hist_file = f.name
     f.close()
     config.HistoryManager.db_cache_size = 10000
     return config
 
 
 def get_ipython_cmd(as_string=False):
     """
     Return appropriate IPython command line name. By default, this will return
     a list that can be used with subprocess.Popen, for example, but passing
     `as_string=True` allows for returning the IPython command as a string.
 
     Parameters
     ----------
     as_string: bool
         Flag to allow to return the command as a string.
     """
     ipython_cmd = [sys.executable, "-m", "IPython"]
 
     if as_string:
         ipython_cmd = " ".join(ipython_cmd)
 
     return ipython_cmd
 
 def ipexec(fname, options=None, commands=()):
     """Utility to call 'ipython filename'.
 
     Starts IPython with a minimal and safe configuration to make startup as fast
     as possible.
 
     Note that this starts IPython in a subprocess!
 
     Parameters
     ----------
     fname : str
       Name of file to be executed (should have .py or .ipy extension).
 
     options : optional, list
       Extra command-line flags to be passed to IPython.
 
     commands : optional, list
       Commands to send in on stdin
 
     Returns
     -------
     (stdout, stderr) of ipython subprocess.
     """
     if options is None: options = []
 
     cmdargs = default_argv() + options
 
     test_dir = os.path.dirname(__file__)
 
     ipython_cmd = get_ipython_cmd()
     # Absolute path for filename
     full_fname = os.path.join(test_dir, fname)
     full_cmd = ipython_cmd + cmdargs + [full_fname]
     env = os.environ.copy()
     # FIXME: ignore all warnings in ipexec while we have shims
     # should we keep suppressing warnings here, even after removing shims?
     env['PYTHONWARNINGS'] = 'ignore'
     # env.pop('PYTHONWARNINGS', None)  # Avoid extraneous warnings appearing on stderr
     for k, v in env.items():
         # Debug a bizarre failure we've seen on Windows:
         # TypeError: environment can only contain strings
         if not isinstance(v, str):
             print(k, v)
     p = Popen(full_cmd, stdout=PIPE, stderr=PIPE, stdin=PIPE, env=env)
     out, err = p.communicate(input=py3compat.str_to_bytes('\n'.join(commands)) or None)
     out, err = py3compat.bytes_to_str(out), py3compat.bytes_to_str(err)
     # `import readline` causes 'ESC[?1034h' to be output sometimes,
     # so strip that out before doing comparisons
     if out:
         out = re.sub(r'\x1b\[[^h]+h', '', out)
     return out, err
 
 
 def ipexec_validate(fname, expected_out, expected_err='',
                     options=None, commands=()):
     """Utility to call 'ipython filename' and validate output/error.
 
     This function raises an AssertionError if the validation fails.
 
     Note that this starts IPython in a subprocess!
 
     Parameters
     ----------
     fname : str
       Name of the file to be executed (should have .py or .ipy extension).
 
     expected_out : str
       Expected stdout of the process.
 
     expected_err : optional, str
       Expected stderr of the process.
 
     options : optional, list
       Extra command-line flags to be passed to IPython.
 
     Returns
     -------
     None
     """
 
     import nose.tools as nt
 
     out, err = ipexec(fname, options, commands)
     #print 'OUT', out  # dbg
     #print 'ERR', err  # dbg
     # If there are any errors, we must check those befor stdout, as they may be
     # more informative than simply having an empty stdout.
     if err:
         if expected_err:
             nt.assert_equal("\n".join(err.strip().splitlines()), "\n".join(expected_err.strip().splitlines()))
         else:
             raise ValueError('Running file %r produced error: %r' %
                              (fname, err))
     # If no errors or output on stderr was expected, match stdout
     nt.assert_equal("\n".join(out.strip().splitlines()), "\n".join(expected_out.strip().splitlines()))
 
 
 class TempFileMixin(object):
     """Utility class to create temporary Python/IPython files.
 
     Meant as a mixin class for test cases."""
 
     def mktmp(self, src, ext='.py'):
         """Make a valid python temp file."""
         fname, f = temp_pyfile(src, ext)
         self.tmpfile = f
         self.fname = fname
 
     def tearDown(self):
         if hasattr(self, 'tmpfile'):
             # If the tmpfile wasn't made because of skipped tests, like in
             # win32, there's nothing to cleanup.
             self.tmpfile.close()
             try:
                 os.unlink(self.fname)
             except:
                 # On Windows, even though we close the file, we still can't
                 # delete it.  I have no clue why
                 pass
 
     def __enter__(self):
         return self
 
     def __exit__(self, exc_type, exc_value, traceback):
         self.tearDown()
 
 
 pair_fail_msg = ("Testing {0}\n\n"
                 "In:\n"
                 "  {1!r}\n"
                 "Expected:\n"
                 "  {2!r}\n"
                 "Got:\n"
                 "  {3!r}\n")
 def check_pairs(func, pairs):
     """Utility function for the common case of checking a function with a
     sequence of input/output pairs.
 
     Parameters
     ----------
     func : callable
       The function to be tested. Should accept a single argument.
     pairs : iterable
       A list of (input, expected_output) tuples.
 
     Returns
     -------
     None. Raises an AssertionError if any output does not match the expected
     value.
     """
     name = getattr(func, "func_name", getattr(func, "__name__", "<unknown>"))
     for inp, expected in pairs:
         out = func(inp)
         assert out == expected, pair_fail_msg.format(name, inp, expected, out)
 
 
-if py3compat.PY3:
-    MyStringIO = StringIO
-else:
-    # In Python 2, stdout/stderr can have either bytes or unicode written to them,
-    # so we need a class that can handle both.
-    class MyStringIO(StringIO):
-        def write(self, s):
-            s = py3compat.cast_unicode(s, encoding=DEFAULT_ENCODING)
-            super(MyStringIO, self).write(s)
+MyStringIO = StringIO
 
 _re_type = type(re.compile(r''))
 
 notprinted_msg = """Did not find {0!r} in printed output (on {1}):
 -------
 {2!s}
 -------
 """
 
 class AssertPrints(object):
     """Context manager for testing that code prints certain text.
     
     Examples
     --------
     >>> with AssertPrints("abc", suppress=False):
     ...     print("abcd")
     ...     print("def")
     ... 
     abcd
     def
     """
     def __init__(self, s, channel='stdout', suppress=True):
         self.s = s
         if isinstance(self.s, (str, _re_type)):
             self.s = [self.s]
         self.channel = channel
         self.suppress = suppress
     
     def __enter__(self):
         self.orig_stream = getattr(sys, self.channel)
         self.buffer = MyStringIO()
         self.tee = Tee(self.buffer, channel=self.channel)
         setattr(sys, self.channel, self.buffer if self.suppress else self.tee)
     
     def __exit__(self, etype, value, traceback):
         try:
             if value is not None:
                 # If an error was raised, don't check anything else
                 return False
             self.tee.flush()
             setattr(sys, self.channel, self.orig_stream)
             printed = self.buffer.getvalue()
             for s in self.s:
                 if isinstance(s, _re_type):
                     assert s.search(printed), notprinted_msg.format(s.pattern, self.channel, printed)
                 else:
                     assert s in printed, notprinted_msg.format(s, self.channel, printed)
             return False
         finally:
             self.tee.close()
 
 printed_msg = """Found {0!r} in printed output (on {1}):
 -------
 {2!s}
 -------
 """
 
 class AssertNotPrints(AssertPrints):
     """Context manager for checking that certain output *isn't* produced.
     
     Counterpart of AssertPrints"""
     def __exit__(self, etype, value, traceback):
         try:
             if value is not None:
                 # If an error was raised, don't check anything else
                 self.tee.close()
                 return False
             self.tee.flush()
             setattr(sys, self.channel, self.orig_stream)
             printed = self.buffer.getvalue()
             for s in self.s:
                 if isinstance(s, _re_type):
                     assert not s.search(printed),printed_msg.format(
                         s.pattern, self.channel, printed)
                 else:
                     assert s not in printed, printed_msg.format(
                         s, self.channel, printed)
             return False
         finally:
             self.tee.close()
 
 @contextmanager
 def mute_warn():
     from IPython.utils import warn
     save_warn = warn.warn
     warn.warn = lambda *a, **kw: None
     try:
         yield
     finally:
         warn.warn = save_warn
 
 @contextmanager
 def make_tempfile(name):
     """ Create an empty, named, temporary file for the duration of the context.
     """
     f = open(name, 'w')
     f.close()
     try:
         yield
     finally:
         os.unlink(name)
 
 def fake_input(inputs):
     """Temporarily replace the input() function to return the given values
 
     Use as a context manager:
 
     with fake_input(['result1', 'result2']):
         ...
 
     Values are returned in order. If input() is called again after the last value
     was used, EOFError is raised.
     """
     it = iter(inputs)
     def mock_input(prompt=''):
         try:
             return next(it)
         except StopIteration:
             raise EOFError('No more inputs given')
 
     return patch('builtins.input', mock_input)
 
 def help_output_test(subcommand=''):
     """test that `ipython [subcommand] -h` works"""
     cmd = get_ipython_cmd() + [subcommand, '-h']
     out, err, rc = get_output_error_code(cmd)
     nt.assert_equal(rc, 0, err)
     nt.assert_not_in("Traceback", err)
     nt.assert_in("Options", out)
     nt.assert_in("--help-all", out)
     return out, err
 
 
 def help_all_output_test(subcommand=''):
     """test that `ipython [subcommand] --help-all` works"""
     cmd = get_ipython_cmd() + [subcommand, '--help-all']
     out, err, rc = get_output_error_code(cmd)
     nt.assert_equal(rc, 0, err)
     nt.assert_not_in("Traceback", err)
     nt.assert_in("Options", out)
     nt.assert_in("Class parameters", out)
     return out, err
 

IPython/utils/tests/test_path.py

@@ -1,489 +1,482 b''
 # encoding: utf-8
 """Tests for IPython.utils.path.py"""
 
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
 import errno
 import os
 import shutil
 import sys
 import tempfile
 import warnings
 from contextlib import contextmanager
 from unittest.mock import patch
 from os.path import join, abspath, split
+from imp import reload
 
 from nose import SkipTest, with_setup
 import nose.tools as nt
 
 import IPython
 from IPython import paths
 from IPython.testing import decorators as dec
 from IPython.testing.decorators import (skip_if_not_win32, skip_win32,
                                         onlyif_unicode_paths,)
 from IPython.testing.tools import make_tempfile, AssertPrints
 from IPython.utils import path
 from IPython.utils import py3compat
 from IPython.utils.tempdir import TemporaryDirectory
 
 # Platform-dependent imports
 try:
-    import winreg as wreg  # Py 3
+    import winreg as wreg  
 except ImportError:
+    #Fake _winreg module on non-windows platforms
+    import types
+    wr_name = "winreg"
+    sys.modules[wr_name] = types.ModuleType(wr_name)
     try:
-        import _winreg as wreg  # Py 2
+        import winreg as wreg
     except ImportError:
-        #Fake _winreg module on none windows platforms
-        import types
-        wr_name = "winreg" if py3compat.PY3 else "_winreg"
-        sys.modules[wr_name] = types.ModuleType(wr_name)
-        try:
-            import winreg as wreg
-        except ImportError:
-            import _winreg as wreg
+        import _winreg as wreg
         #Add entries that needs to be stubbed by the testing code
         (wreg.OpenKey, wreg.QueryValueEx,) = (None, None)
 
-try:
-    reload
-except NameError:   # Python 3
-    from imp import reload
-
 #-----------------------------------------------------------------------------
 # Globals
 #-----------------------------------------------------------------------------
 env = os.environ
 TMP_TEST_DIR = tempfile.mkdtemp()
 HOME_TEST_DIR = join(TMP_TEST_DIR, "home_test_dir")
 #
 # Setup/teardown functions/decorators
 #
 
 def setup():
     """Setup testenvironment for the module:
 
             - Adds dummy home dir tree
     """
     # Do not mask exceptions here.  In particular, catching WindowsError is a
     # problem because that exception is only defined on Windows...
     os.makedirs(os.path.join(HOME_TEST_DIR, 'ipython'))
 
 
 def teardown():
     """Teardown testenvironment for the module:
 
             - Remove dummy home dir tree
     """
     # Note: we remove the parent test dir, which is the root of all test
     # subdirs we may have created.  Use shutil instead of os.removedirs, so
     # that non-empty directories are all recursively removed.
     shutil.rmtree(TMP_TEST_DIR)
 
 
 def setup_environment():
     """Setup testenvironment for some functions that are tested
     in this module. In particular this functions stores attributes
     and other things that we need to stub in some test functions.
     This needs to be done on a function level and not module level because
     each testfunction needs a pristine environment.
     """
     global oldstuff, platformstuff
     oldstuff = (env.copy(), os.name, sys.platform, path.get_home_dir, IPython.__file__, os.getcwd())
 
 def teardown_environment():
     """Restore things that were remembered by the setup_environment function
     """
     (oldenv, os.name, sys.platform, path.get_home_dir, IPython.__file__, old_wd) = oldstuff
     os.chdir(old_wd)
     reload(path)
 
     for key in list(env):
         if key not in oldenv:
             del env[key]
     env.update(oldenv)
     if hasattr(sys, 'frozen'):
         del sys.frozen
 
 # Build decorator that uses the setup_environment/setup_environment
 with_environment = with_setup(setup_environment, teardown_environment)
 
 @skip_if_not_win32
 @with_environment
 def test_get_home_dir_1():
     """Testcase for py2exe logic, un-compressed lib
     """
     unfrozen = path.get_home_dir()
     sys.frozen = True
 
     #fake filename for IPython.__init__
     IPython.__file__ = abspath(join(HOME_TEST_DIR, "Lib/IPython/__init__.py"))
 
     home_dir = path.get_home_dir()
     nt.assert_equal(home_dir, unfrozen)
 
 
 @skip_if_not_win32
 @with_environment
 def test_get_home_dir_2():
     """Testcase for py2exe logic, compressed lib
     """
     unfrozen = path.get_home_dir()
     sys.frozen = True
     #fake filename for IPython.__init__
     IPython.__file__ = abspath(join(HOME_TEST_DIR, "Library.zip/IPython/__init__.py")).lower()
 
     home_dir = path.get_home_dir(True)
     nt.assert_equal(home_dir, unfrozen)
 
 
 @with_environment
 def test_get_home_dir_3():
     """get_home_dir() uses $HOME if set"""
     env["HOME"] = HOME_TEST_DIR
     home_dir = path.get_home_dir(True)
     # get_home_dir expands symlinks
     nt.assert_equal(home_dir, os.path.realpath(env["HOME"]))
 
 
 @with_environment
 def test_get_home_dir_4():
     """get_home_dir() still works if $HOME is not set"""
 
     if 'HOME' in env: del env['HOME']
     # this should still succeed, but we don't care what the answer is
     home = path.get_home_dir(False)
 
 @with_environment
 def test_get_home_dir_5():
     """raise HomeDirError if $HOME is specified, but not a writable dir"""
     env['HOME'] = abspath(HOME_TEST_DIR+'garbage')
     # set os.name = posix, to prevent My Documents fallback on Windows
     os.name = 'posix'
     nt.assert_raises(path.HomeDirError, path.get_home_dir, True)
 
 # Should we stub wreg fully so we can run the test on all platforms?
 @skip_if_not_win32
 @with_environment
 def test_get_home_dir_8():
     """Using registry hack for 'My Documents', os=='nt'
 
     HOMESHARE, HOMEDRIVE, HOMEPATH, USERPROFILE and others are missing.
     """
     os.name = 'nt'
     # Remove from stub environment all keys that may be set
     for key in ['HOME', 'HOMESHARE', 'HOMEDRIVE', 'HOMEPATH', 'USERPROFILE']:
         env.pop(key, None)
 
     class key:
         def Close(self):
             pass
 
     with patch.object(wreg, 'OpenKey', return_value=key()), \
          patch.object(wreg, 'QueryValueEx', return_value=[abspath(HOME_TEST_DIR)]):
         home_dir = path.get_home_dir()
     nt.assert_equal(home_dir, abspath(HOME_TEST_DIR))
 
 @with_environment
 def test_get_xdg_dir_0():
     """test_get_xdg_dir_0, check xdg_dir"""
     reload(path)
     path._writable_dir = lambda path: True
     path.get_home_dir = lambda : 'somewhere'
     os.name = "posix"
     sys.platform = "linux2"
     env.pop('IPYTHON_DIR', None)
     env.pop('IPYTHONDIR', None)
     env.pop('XDG_CONFIG_HOME', None)
 
     nt.assert_equal(path.get_xdg_dir(), os.path.join('somewhere', '.config'))
 
 
 @with_environment
 def test_get_xdg_dir_1():
     """test_get_xdg_dir_1, check nonexistant xdg_dir"""
     reload(path)
     path.get_home_dir = lambda : HOME_TEST_DIR
     os.name = "posix"
     sys.platform = "linux2"
     env.pop('IPYTHON_DIR', None)
     env.pop('IPYTHONDIR', None)
     env.pop('XDG_CONFIG_HOME', None)
     nt.assert_equal(path.get_xdg_dir(), None)
 
 @with_environment
 def test_get_xdg_dir_2():
     """test_get_xdg_dir_2, check xdg_dir default to ~/.config"""
     reload(path)
     path.get_home_dir = lambda : HOME_TEST_DIR
     os.name = "posix"
     sys.platform = "linux2"
     env.pop('IPYTHON_DIR', None)
     env.pop('IPYTHONDIR', None)
     env.pop('XDG_CONFIG_HOME', None)
     cfgdir=os.path.join(path.get_home_dir(), '.config')
     if not os.path.exists(cfgdir):
         os.makedirs(cfgdir)
 
     nt.assert_equal(path.get_xdg_dir(), cfgdir)
 
 @with_environment
 def test_get_xdg_dir_3():
     """test_get_xdg_dir_3, check xdg_dir not used on OS X"""
     reload(path)
     path.get_home_dir = lambda : HOME_TEST_DIR
     os.name = "posix"
     sys.platform = "darwin"
     env.pop('IPYTHON_DIR', None)
     env.pop('IPYTHONDIR', None)
     env.pop('XDG_CONFIG_HOME', None)
     cfgdir=os.path.join(path.get_home_dir(), '.config')
     if not os.path.exists(cfgdir):
         os.makedirs(cfgdir)
 
     nt.assert_equal(path.get_xdg_dir(), None)
 
 def test_filefind():
     """Various tests for filefind"""
     f = tempfile.NamedTemporaryFile()
     # print 'fname:',f.name
     alt_dirs = paths.get_ipython_dir()
     t = path.filefind(f.name, alt_dirs)
     # print 'found:',t
 
 
 @dec.skip_if_not_win32
 def test_get_long_path_name_win32():
     with TemporaryDirectory() as tmpdir:
 
         # Make a long path. Expands the path of tmpdir prematurely as it may already have a long 
         # path component, so ensure we include the long form of it
         long_path = os.path.join(path.get_long_path_name(tmpdir), u'this is my long path name')
         os.makedirs(long_path)
 
         # Test to see if the short path evaluates correctly.
         short_path = os.path.join(tmpdir, u'THISIS~1')
         evaluated_path = path.get_long_path_name(short_path)
         nt.assert_equal(evaluated_path.lower(), long_path.lower())
 
 
 @dec.skip_win32
 def test_get_long_path_name():
     p = path.get_long_path_name('/usr/local')
     nt.assert_equal(p,'/usr/local')
 
 @dec.skip_win32 # can't create not-user-writable dir on win
 @with_environment
 def test_not_writable_ipdir():
     tmpdir = tempfile.mkdtemp()
     os.name = "posix"
     env.pop('IPYTHON_DIR', None)
     env.pop('IPYTHONDIR', None)
     env.pop('XDG_CONFIG_HOME', None)
     env['HOME'] = tmpdir
     ipdir = os.path.join(tmpdir, '.ipython')
     os.mkdir(ipdir, 0o555)
     try:
         open(os.path.join(ipdir, "_foo_"), 'w').close()
     except IOError:
         pass
     else:
         # I can still write to an unwritable dir,
         # assume I'm root and skip the test
         raise SkipTest("I can't create directories that I can't write to")
     with AssertPrints('is not a writable location', channel='stderr'):
         ipdir = paths.get_ipython_dir()
     env.pop('IPYTHON_DIR', None)
 
 @with_environment
 def test_get_py_filename():
     os.chdir(TMP_TEST_DIR)
     with make_tempfile('foo.py'):
         nt.assert_equal(path.get_py_filename('foo.py'), 'foo.py')
         nt.assert_equal(path.get_py_filename('foo'), 'foo.py')
     with make_tempfile('foo'):
         nt.assert_equal(path.get_py_filename('foo'), 'foo')
         nt.assert_raises(IOError, path.get_py_filename, 'foo.py')
     nt.assert_raises(IOError, path.get_py_filename, 'foo')
     nt.assert_raises(IOError, path.get_py_filename, 'foo.py')
     true_fn = 'foo with spaces.py'
     with make_tempfile(true_fn):
         nt.assert_equal(path.get_py_filename('foo with spaces'), true_fn)
         nt.assert_equal(path.get_py_filename('foo with spaces.py'), true_fn)
         nt.assert_raises(IOError, path.get_py_filename, '"foo with spaces.py"')
         nt.assert_raises(IOError, path.get_py_filename, "'foo with spaces.py'")
 
 @onlyif_unicode_paths
 def test_unicode_in_filename():
     """When a file doesn't exist, the exception raised should be safe to call
     str() on - i.e. in Python 2 it must only have ASCII characters.
     
     https://github.com/ipython/ipython/issues/875
     """
     try:
         # these calls should not throw unicode encode exceptions
         path.get_py_filename(u'fooéè.py',  force_win32=False)
     except IOError as ex:
         str(ex)
 
 
 class TestShellGlob(object):
 
     @classmethod
     def setUpClass(cls):
         cls.filenames_start_with_a = ['a0', 'a1', 'a2']
         cls.filenames_end_with_b = ['0b', '1b', '2b']
         cls.filenames = cls.filenames_start_with_a + cls.filenames_end_with_b
         cls.tempdir = TemporaryDirectory()
         td = cls.tempdir.name
 
         with cls.in_tempdir():
             # Create empty files
             for fname in cls.filenames:
                 open(os.path.join(td, fname), 'w').close()
 
     @classmethod
     def tearDownClass(cls):
         cls.tempdir.cleanup()
 
     @classmethod
     @contextmanager
     def in_tempdir(cls):
         save = os.getcwd()
         try:
             os.chdir(cls.tempdir.name)
             yield
         finally:
             os.chdir(save)
 
     def check_match(self, patterns, matches):
         with self.in_tempdir():
             # glob returns unordered list. that's why sorted is required.
             nt.assert_equal(sorted(path.shellglob(patterns)),
                             sorted(matches))
 
     def common_cases(self):
         return [
             (['*'], self.filenames),
             (['a*'], self.filenames_start_with_a),
             (['*c'], ['*c']),
             (['*', 'a*', '*b', '*c'], self.filenames
                                       + self.filenames_start_with_a
                                       + self.filenames_end_with_b
                                       + ['*c']),
             (['a[012]'], self.filenames_start_with_a),
         ]
 
     @skip_win32
     def test_match_posix(self):
         for (patterns, matches) in self.common_cases() + [
                 ([r'\*'], ['*']),
                 ([r'a\*', 'a*'], ['a*'] + self.filenames_start_with_a),
                 ([r'a\[012]'], ['a[012]']),
                 ]:
             yield (self.check_match, patterns, matches)
 
     @skip_if_not_win32
     def test_match_windows(self):
         for (patterns, matches) in self.common_cases() + [
                 # In windows, backslash is interpreted as path
                 # separator.  Therefore, you can't escape glob
                 # using it.
                 ([r'a\*', 'a*'], [r'a\*'] + self.filenames_start_with_a),
                 ([r'a\[012]'], [r'a\[012]']),
                 ]:
             yield (self.check_match, patterns, matches)
 
 
 def test_unescape_glob():
     nt.assert_equal(path.unescape_glob(r'\*\[\!\]\?'), '*[!]?')
     nt.assert_equal(path.unescape_glob(r'\\*'), r'\*')
     nt.assert_equal(path.unescape_glob(r'\\\*'), r'\*')
     nt.assert_equal(path.unescape_glob(r'\\a'), r'\a')
     nt.assert_equal(path.unescape_glob(r'\a'), r'\a')
 
 
 def test_ensure_dir_exists():
     with TemporaryDirectory() as td:
         d = os.path.join(td, u'∂ir')
         path.ensure_dir_exists(d) # create it
         assert os.path.isdir(d)
         path.ensure_dir_exists(d) # no-op
         f = os.path.join(td, u'ƒile')
         open(f, 'w').close() # touch
         with nt.assert_raises(IOError):
             path.ensure_dir_exists(f)
 
 class TestLinkOrCopy(object):
     def setUp(self):
         self.tempdir = TemporaryDirectory()
         self.src = self.dst("src")
         with open(self.src, "w") as f:
             f.write("Hello, world!")
 
     def tearDown(self):
         self.tempdir.cleanup()
 
     def dst(self, *args):
         return os.path.join(self.tempdir.name, *args)
 
     def assert_inode_not_equal(self, a, b):
         nt.assert_not_equal(os.stat(a).st_ino, os.stat(b).st_ino,
                             "%r and %r do reference the same indoes" %(a, b))
 
     def assert_inode_equal(self, a, b):
         nt.assert_equal(os.stat(a).st_ino, os.stat(b).st_ino,
                         "%r and %r do not reference the same indoes" %(a, b))
 
     def assert_content_equal(self, a, b):
         with open(a) as a_f:
             with open(b) as b_f:
                 nt.assert_equal(a_f.read(), b_f.read())
 
     @skip_win32
     def test_link_successful(self):
         dst = self.dst("target")
         path.link_or_copy(self.src, dst)
         self.assert_inode_equal(self.src, dst)
 
     @skip_win32
     def test_link_into_dir(self):
         dst = self.dst("some_dir")
         os.mkdir(dst)
         path.link_or_copy(self.src, dst)
         expected_dst = self.dst("some_dir", os.path.basename(self.src))
         self.assert_inode_equal(self.src, expected_dst)
 
     @skip_win32
     def test_target_exists(self):
         dst = self.dst("target")
         open(dst, "w").close()
         path.link_or_copy(self.src, dst)
         self.assert_inode_equal(self.src, dst)
 
     @skip_win32
     def test_no_link(self):
         real_link = os.link
         try:
             del os.link
             dst = self.dst("target")
             path.link_or_copy(self.src, dst)
             self.assert_content_equal(self.src, dst)
             self.assert_inode_not_equal(self.src, dst)
         finally:
             os.link = real_link
 
     @skip_if_not_win32
     def test_windows(self):
         dst = self.dst("target")
         path.link_or_copy(self.src, dst)
         self.assert_content_equal(self.src, dst)
 
     def test_link_twice(self):
         # Linking the same file twice shouldn't leave duplicates around.
         # See https://github.com/ipython/ipython/issues/6450
         dst = self.dst('target')
         path.link_or_copy(self.src, dst)
         path.link_or_copy(self.src, dst)
         self.assert_inode_equal(self.src, dst)
         nt.assert_equal(sorted(os.listdir(self.tempdir.name)), ['src', 'target'])