remove sys_version for Python 3

since we're now committed to using Python3, I decided to go through and
clean up the code base to remove paths which depend on sys_version
checks for Python3

IPython/core/debugger.py

@@ -1,631 +1,630 b''
 # -*- coding: utf-8 -*-
 """
 Pdb debugger class.
 
 Modified from the standard pdb.Pdb class to avoid including readline, so that
 the command line completion of other programs which include this isn't
 damaged.
 
 In the future, this class will be expanded with improvements over the standard
 pdb.
 
 The code in this file is mainly lifted out of cmd.py in Python 2.2, with minor
 changes. Licensing should therefore be under the standard Python terms.  For
 details on the PSF (Python Software Foundation) standard license, see:
 
 http://www.python.org/2.2.3/license.html"""
 
 #*****************************************************************************
 #
 #       This file is licensed under the PSF license.
 #
 #       Copyright (C) 2001 Python Software Foundation, www.python.org
 #       Copyright (C) 2005-2006 Fernando Perez. <[e-mail]>
 #
 #
 #*****************************************************************************
 from __future__ import print_function
 
 import bdb
 import functools
 import inspect
 import sys
 import warnings
 
 from IPython import get_ipython
 from IPython.utils import PyColorize, ulinecache
 from IPython.utils import coloransi, py3compat
 from IPython.core.excolors import exception_colors
 from IPython.testing.skipdoctest import skip_doctest
 
 
 prompt = 'ipdb> '
 
 #We have to check this directly from sys.argv, config struct not yet available
 from pdb import Pdb as OldPdb
 
 # Allow the set_trace code to operate outside of an ipython instance, even if
 # it does so with some limitations.  The rest of this support is implemented in
 # the Tracer constructor.
 
 def make_arrow(pad):
     """generate the leading arrow in front of traceback or debugger"""
     if pad >= 2:
         return '-'*(pad-2) + '> '
     elif pad == 1:
         return '>'
     return ''
 
 
 def BdbQuit_excepthook(et, ev, tb, excepthook=None):
     """Exception hook which handles `BdbQuit` exceptions.
 
     All other exceptions are processed using the `excepthook`
     parameter.
     """
     warnings.warn("`BdbQuit_excepthook` is deprecated since version 5.1",
                   DeprecationWarning)
     if et==bdb.BdbQuit:
         print('Exiting Debugger.')
     elif excepthook is not None:
         excepthook(et, ev, tb)
     else:
         # Backwards compatibility. Raise deprecation warning?
         BdbQuit_excepthook.excepthook_ori(et,ev,tb)
 
 
 def BdbQuit_IPython_excepthook(self,et,ev,tb,tb_offset=None):
     warnings.warn(
         "`BdbQuit_IPython_excepthook` is deprecated since version 5.1",
         DeprecationWarning)
     print('Exiting Debugger.')
 
 
 class Tracer(object):
     """
     DEPRECATED
 
     Class for local debugging, similar to pdb.set_trace.
 
     Instances of this class, when called, behave like pdb.set_trace, but
     providing IPython's enhanced capabilities.
 
     This is implemented as a class which must be initialized in your own code
     and not as a standalone function because we need to detect at runtime
     whether IPython is already active or not.  That detection is done in the
     constructor, ensuring that this code plays nicely with a running IPython,
     while functioning acceptably (though with limitations) if outside of it.
     """
 
     @skip_doctest
     def __init__(self, colors=None):
         """
         DEPRECATED
 
         Create a local debugger instance.
 
         Parameters
         ----------
 
         colors : str, optional
             The name of the color scheme to use, it must be one of IPython's
             valid color schemes.  If not given, the function will default to
             the current IPython scheme when running inside IPython, and to
             'NoColor' otherwise.
 
         Examples
         --------
         ::
 
             from IPython.core.debugger import Tracer; debug_here = Tracer()
 
         Later in your code::
 
             debug_here()  # -> will open up the debugger at that point.
 
         Once the debugger activates, you can use all of its regular commands to
         step through code, set breakpoints, etc.  See the pdb documentation
         from the Python standard library for usage details.
         """
         warnings.warn("`Tracer` is deprecated since version 5.1, directly use "
                       "`IPython.core.debugger.Pdb.set_trace()`",
                       DeprecationWarning)
 
         ip = get_ipython()
         if ip is None:
             # Outside of ipython, we set our own exception hook manually
             sys.excepthook = functools.partial(BdbQuit_excepthook,
                                                excepthook=sys.excepthook)
             def_colors = 'NoColor'
         else:
             # In ipython, we use its custom exception handler mechanism
             def_colors = ip.colors
             ip.set_custom_exc((bdb.BdbQuit,), BdbQuit_IPython_excepthook)
 
         if colors is None:
             colors = def_colors
 
         # The stdlib debugger internally uses a modified repr from the `repr`
         # module, that limits the length of printed strings to a hardcoded
         # limit of 30 characters.  That much trimming is too aggressive, let's
         # at least raise that limit to 80 chars, which should be enough for
         # most interactive uses.
         try:
             try:
                 from reprlib import aRepr  # Py 3
             except ImportError:
                 from repr import aRepr  # Py 2
             aRepr.maxstring = 80
         except:
             # This is only a user-facing convenience, so any error we encounter
             # here can be warned about but can be otherwise ignored.  These
             # printouts will tell us about problems if this API changes
             import traceback
             traceback.print_exc()
 
         self.debugger = Pdb(colors)
 
     def __call__(self):
         """Starts an interactive debugger at the point where called.
 
         This is similar to the pdb.set_trace() function from the std lib, but
         using IPython's enhanced debugger."""
 
         self.debugger.set_trace(sys._getframe().f_back)
 
 
 def decorate_fn_with_doc(new_fn, old_fn, additional_text=""):
     """Make new_fn have old_fn's doc string. This is particularly useful
     for the ``do_...`` commands that hook into the help system.
     Adapted from from a comp.lang.python posting
     by Duncan Booth."""
     def wrapper(*args, **kw):
         return new_fn(*args, **kw)
     if old_fn.__doc__:
         wrapper.__doc__ = old_fn.__doc__ + additional_text
     return wrapper
 
 
 def _file_lines(fname):
     """Return the contents of a named file as a list of lines.
 
     This function never raises an IOError exception: if the file can't be
     read, it simply returns an empty list."""
 
     try:
         outfile = open(fname)
     except IOError:
         return []
     else:
         out = outfile.readlines()
         outfile.close()
         return out
 
 
 class Pdb(OldPdb, object):
     """Modified Pdb class, does not load readline.
 
     for a standalone version that uses prompt_toolkit, see
     `IPython.terminal.debugger.TerminalPdb` and
     `IPython.terminal.debugger.set_trace()`
     """
 
     def __init__(self, color_scheme=None, completekey=None,
                  stdin=None, stdout=None, context=5):
 
         # Parent constructor:
         try:
             self.context = int(context)
             if self.context <= 0:
                 raise ValueError("Context must be a positive integer")
         except (TypeError, ValueError):
                 raise ValueError("Context must be a positive integer")
 
         OldPdb.__init__(self, completekey, stdin, stdout)
 
         # IPython changes...
         self.shell = get_ipython()
 
         if self.shell is None:
             save_main = sys.modules['__main__']
             # No IPython instance running, we must create one
             from IPython.terminal.interactiveshell import \
                 TerminalInteractiveShell
             self.shell = TerminalInteractiveShell.instance()
             # needed by any code which calls __import__("__main__") after
             # the debugger was entered. See also #9941.
             sys.modules['__main__'] = save_main 
 
         if color_scheme is not None:
             warnings.warn(
                 "The `color_scheme` argument is deprecated since version 5.1",
                 DeprecationWarning, stacklevel=2)
         else:
             color_scheme = self.shell.colors
 
         self.aliases = {}
 
         # Create color table: we copy the default one from the traceback
         # module and add a few attributes needed for debugging
         self.color_scheme_table = exception_colors()
 
         # shorthands
         C = coloransi.TermColors
         cst = self.color_scheme_table
 
         cst['NoColor'].colors.prompt = C.NoColor
         cst['NoColor'].colors.breakpoint_enabled = C.NoColor
         cst['NoColor'].colors.breakpoint_disabled = C.NoColor
 
         cst['Linux'].colors.prompt = C.Green
         cst['Linux'].colors.breakpoint_enabled = C.LightRed
         cst['Linux'].colors.breakpoint_disabled = C.Red
 
         cst['LightBG'].colors.prompt = C.Blue
         cst['LightBG'].colors.breakpoint_enabled = C.LightRed
         cst['LightBG'].colors.breakpoint_disabled = C.Red
 
         cst['Neutral'].colors.prompt = C.Blue
         cst['Neutral'].colors.breakpoint_enabled = C.LightRed
         cst['Neutral'].colors.breakpoint_disabled = C.Red
 
 
         # Add a python parser so we can syntax highlight source while
         # debugging.
         self.parser = PyColorize.Parser(style=color_scheme)
         self.set_colors(color_scheme)
 
         # Set the prompt - the default prompt is '(Pdb)'
         self.prompt = prompt
 
     def set_colors(self, scheme):
         """Shorthand access to the color table scheme selector method."""
         self.color_scheme_table.set_active_scheme(scheme)
         self.parser.style = scheme
 
     def interaction(self, frame, traceback):
         try:
             OldPdb.interaction(self, frame, traceback)
         except KeyboardInterrupt:
             sys.stdout.write('\n' + self.shell.get_exception_only())
 
     def parseline(self, line):
         if line.startswith("!!"):
             # Force standard behavior.
             return super(Pdb, self).parseline(line[2:])
         # "Smart command mode" from pdb++: don't execute commands if a variable
         # with the same name exists.
         cmd, arg, newline = super(Pdb, self).parseline(line)
         # Fix for #9611: Do not trigger smart command if the command is `exit`
         # or `quit` and it would resolve to their *global* value (the
         # `ExitAutocall` object).  Just checking that it is not present in the
         # locals dict is not enough as locals and globals match at the
         # toplevel.
         if ((cmd in self.curframe.f_locals or cmd in self.curframe.f_globals)
                 and not (cmd in ["exit", "quit"]
                          and (self.curframe.f_locals is self.curframe.f_globals
                               or cmd not in self.curframe.f_locals))):
             return super(Pdb, self).parseline("!" + line)
         return super(Pdb, self).parseline(line)
 
     def new_do_up(self, arg):
         OldPdb.do_up(self, arg)
     do_u = do_up = decorate_fn_with_doc(new_do_up, OldPdb.do_up)
 
     def new_do_down(self, arg):
         OldPdb.do_down(self, arg)
 
     do_d = do_down = decorate_fn_with_doc(new_do_down, OldPdb.do_down)
 
     def new_do_frame(self, arg):
         OldPdb.do_frame(self, arg)
 
     def new_do_quit(self, arg):
 
         if hasattr(self, 'old_all_completions'):
             self.shell.Completer.all_completions=self.old_all_completions
 
         return OldPdb.do_quit(self, arg)
 
     do_q = do_quit = decorate_fn_with_doc(new_do_quit, OldPdb.do_quit)
 
     def new_do_restart(self, arg):
         """Restart command. In the context of ipython this is exactly the same
         thing as 'quit'."""
         self.msg("Restart doesn't make sense here. Using 'quit' instead.")
         return self.do_quit(arg)
 
     def print_stack_trace(self, context=None):
         if context is None:
             context = self.context
         try:
             context=int(context)
             if context <= 0:
                 raise ValueError("Context must be a positive integer")
         except (TypeError, ValueError):
                 raise ValueError("Context must be a positive integer")
         try:
             for frame_lineno in self.stack:
                 self.print_stack_entry(frame_lineno, context=context)
         except KeyboardInterrupt:
             pass
 
     def print_stack_entry(self,frame_lineno, prompt_prefix='\n-> ',
                           context=None):
         if context is None:
             context = self.context
         try:
             context=int(context)
             if context <= 0:
                 raise ValueError("Context must be a positive integer")
         except (TypeError, ValueError):
                 raise ValueError("Context must be a positive integer")
         print(self.format_stack_entry(frame_lineno, '', context))
 
         # vds: >>
         frame, lineno = frame_lineno
         filename = frame.f_code.co_filename
         self.shell.hooks.synchronize_with_editor(filename, lineno, 0)
         # vds: <<
 
     def format_stack_entry(self, frame_lineno, lprefix=': ', context=None):
         if context is None:
             context = self.context
         try:
             context=int(context)
             if context <= 0:
                 print("Context must be a positive integer")
         except (TypeError, ValueError):
                 print("Context must be a positive integer")
         try:
             import reprlib  # Py 3
         except ImportError:
             import repr as reprlib  # Py 2
 
         ret = []
 
         Colors = self.color_scheme_table.active_colors
         ColorsNormal = Colors.Normal
         tpl_link = u'%s%%s%s' % (Colors.filenameEm, ColorsNormal)
         tpl_call = u'%s%%s%s%%s%s' % (Colors.vName, Colors.valEm, ColorsNormal)
         tpl_line = u'%%s%s%%s %s%%s' % (Colors.lineno, ColorsNormal)
         tpl_line_em = u'%%s%s%%s %s%%s%s' % (Colors.linenoEm, Colors.line,
                                             ColorsNormal)
 
         frame, lineno = frame_lineno
 
         return_value = ''
         if '__return__' in frame.f_locals:
             rv = frame.f_locals['__return__']
             #return_value += '->'
             return_value += reprlib.repr(rv) + '\n'
         ret.append(return_value)
 
         #s = filename + '(' + `lineno` + ')'
         filename = self.canonic(frame.f_code.co_filename)
         link = tpl_link % py3compat.cast_unicode(filename)
 
         if frame.f_code.co_name:
             func = frame.f_code.co_name
         else:
             func = "<lambda>"
 
         call = ''
         if func != '?':
             if '__args__' in frame.f_locals:
                 args = reprlib.repr(frame.f_locals['__args__'])
             else:
                 args = '()'
             call = tpl_call % (func, args)
 
         # The level info should be generated in the same format pdb uses, to
         # avoid breaking the pdbtrack functionality of python-mode in *emacs.
         if frame is self.curframe:
             ret.append('> ')
         else:
             ret.append('  ')
         ret.append(u'%s(%s)%s\n' % (link,lineno,call))
 
         start = lineno - 1 - context//2
         lines = ulinecache.getlines(filename)
         start = min(start, len(lines) - context)
         start = max(start, 0)
         lines = lines[start : start + context]
 
         for i,line in enumerate(lines):
             show_arrow = (start + 1 + i == lineno)
             linetpl = (frame is self.curframe or show_arrow) \
                       and tpl_line_em \
                       or tpl_line
             ret.append(self.__format_line(linetpl, filename,
                                           start + 1 + i, line,
                                           arrow = show_arrow) )
         return ''.join(ret)
 
     def __format_line(self, tpl_line, filename, lineno, line, arrow = False):
         bp_mark = ""
         bp_mark_color = ""
 
         new_line, err = self.parser.format2(line, 'str')
         if not err:
             line = new_line
 
         bp = None
         if lineno in self.get_file_breaks(filename):
             bps = self.get_breaks(filename, lineno)
             bp = bps[-1]
 
         if bp:
             Colors = self.color_scheme_table.active_colors
             bp_mark = str(bp.number)
             bp_mark_color = Colors.breakpoint_enabled
             if not bp.enabled:
                 bp_mark_color = Colors.breakpoint_disabled
 
         numbers_width = 7
         if arrow:
             # This is the line with the error
             pad = numbers_width - len(str(lineno)) - len(bp_mark)
             num = '%s%s' % (make_arrow(pad), str(lineno))
         else:
             num = '%*s' % (numbers_width - len(bp_mark), str(lineno))
 
         return tpl_line % (bp_mark_color + bp_mark, num, line)
 
 
     def print_list_lines(self, filename, first, last):
         """The printing (as opposed to the parsing part of a 'list'
         command."""
         try:
             Colors = self.color_scheme_table.active_colors
             ColorsNormal = Colors.Normal
             tpl_line = '%%s%s%%s %s%%s' % (Colors.lineno, ColorsNormal)
             tpl_line_em = '%%s%s%%s %s%%s%s' % (Colors.linenoEm, Colors.line, ColorsNormal)
             src = []
             if filename == "<string>" and hasattr(self, "_exec_filename"):
                 filename = self._exec_filename
 
             for lineno in range(first, last+1):
                 line = ulinecache.getline(filename, lineno)
                 if not line:
                     break
 
                 if lineno == self.curframe.f_lineno:
                     line = self.__format_line(tpl_line_em, filename, lineno, line, arrow = True)
                 else:
                     line = self.__format_line(tpl_line, filename, lineno, line, arrow = False)
 
                 src.append(line)
                 self.lineno = lineno
 
             print(''.join(src))
 
         except KeyboardInterrupt:
             pass
 
     def do_list(self, arg):
         self.lastcmd = 'list'
         last = None
         if arg:
             try:
                 x = eval(arg, {}, {})
                 if type(x) == type(()):
                     first, last = x
                     first = int(first)
                     last = int(last)
                     if last < first:
                         # Assume it's a count
                         last = first + last
                 else:
                     first = max(1, int(x) - 5)
             except:
                 print('*** Error in argument:', repr(arg))
                 return
         elif self.lineno is None:
             first = max(1, self.curframe.f_lineno - 5)
         else:
             first = self.lineno + 1
         if last is None:
             last = first + 10
         self.print_list_lines(self.curframe.f_code.co_filename, first, last)
 
         # vds: >>
         lineno = first
         filename = self.curframe.f_code.co_filename
         self.shell.hooks.synchronize_with_editor(filename, lineno, 0)
         # vds: <<
 
     do_l = do_list
 
     def getsourcelines(self, obj):
         lines, lineno = inspect.findsource(obj)
         if inspect.isframe(obj) and obj.f_globals is obj.f_locals:
             # must be a module frame: do not try to cut a block out of it
             return lines, 1
         elif inspect.ismodule(obj):
             return lines, 1
         return inspect.getblock(lines[lineno:]), lineno+1
 
     def do_longlist(self, arg):
         self.lastcmd = 'longlist'
         try:
             lines, lineno = self.getsourcelines(self.curframe)
         except OSError as err:
             self.error(err)
             return
         last = lineno + len(lines)
         self.print_list_lines(self.curframe.f_code.co_filename, lineno, last)
     do_ll = do_longlist
 
     def do_pdef(self, arg):
         """Print the call signature for any callable object.
 
         The debugger interface to %pdef"""
         namespaces = [('Locals', self.curframe.f_locals),
                       ('Globals', self.curframe.f_globals)]
         self.shell.find_line_magic('pdef')(arg, namespaces=namespaces)
 
     def do_pdoc(self, arg):
         """Print the docstring for an object.
 
         The debugger interface to %pdoc."""
         namespaces = [('Locals', self.curframe.f_locals),
                       ('Globals', self.curframe.f_globals)]
         self.shell.find_line_magic('pdoc')(arg, namespaces=namespaces)
 
     def do_pfile(self, arg):
         """Print (or run through pager) the file where an object is defined.
 
         The debugger interface to %pfile.
         """
         namespaces = [('Locals', self.curframe.f_locals),
                       ('Globals', self.curframe.f_globals)]
         self.shell.find_line_magic('pfile')(arg, namespaces=namespaces)
 
     def do_pinfo(self, arg):
         """Provide detailed information about an object.
 
         The debugger interface to %pinfo, i.e., obj?."""
         namespaces = [('Locals', self.curframe.f_locals),
                       ('Globals', self.curframe.f_globals)]
         self.shell.find_line_magic('pinfo')(arg, namespaces=namespaces)
 
     def do_pinfo2(self, arg):
         """Provide extra detailed information about an object.
 
         The debugger interface to %pinfo2, i.e., obj??."""
         namespaces = [('Locals', self.curframe.f_locals),
                       ('Globals', self.curframe.f_globals)]
         self.shell.find_line_magic('pinfo2')(arg, namespaces=namespaces)
 
     def do_psource(self, arg):
         """Print (or run through pager) the source code for an object."""
         namespaces = [('Locals', self.curframe.f_locals),
                       ('Globals', self.curframe.f_globals)]
         self.shell.find_line_magic('psource')(arg, namespaces=namespaces)
 
-    if sys.version_info > (3, ):
-        def do_where(self, arg):
-            """w(here)
-            Print a stack trace, with the most recent frame at the bottom.
-            An arrow indicates the "current frame", which determines the
-            context of most commands. 'bt' is an alias for this command.
-
-            Take a number as argument as an (optional) number of context line to
-            print"""
-            if arg:
-                context = int(arg)
-                self.print_stack_trace(context)
-            else:
-                self.print_stack_trace()
+    def do_where(self, arg):
+        """w(here)
+        Print a stack trace, with the most recent frame at the bottom.
+        An arrow indicates the "current frame", which determines the
+        context of most commands. 'bt' is an alias for this command.
+
+        Take a number as argument as an (optional) number of context line to
+        print"""
+        if arg:
+            context = int(arg)
+            self.print_stack_trace(context)
+        else:
+            self.print_stack_trace()
 
-        do_w = do_where
+    do_w = do_where
 
 
 def set_trace(frame=None):
     """
     Start debugging from `frame`.
 
     If frame is not specified, debugging starts from caller's frame.
     """
     Pdb().set_trace(frame or sys._getframe().f_back)

IPython/core/tests/test_completer.py

@@ -1,806 +1,799 b''
 # encoding: utf-8
 """Tests for the IPython tab-completion machinery."""
 
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
 import os
 import sys
 import unittest
 
 from contextlib import contextmanager
 
 import nose.tools as nt
 
 from traitlets.config.loader import Config
 from IPython import get_ipython
 from IPython.core import completer
 from IPython.external.decorators import knownfailureif
 from IPython.utils.tempdir import TemporaryDirectory, TemporaryWorkingDirectory
 from IPython.utils.generics import complete_object
 from IPython.utils.py3compat import string_types, unicode_type
 from IPython.testing import decorators as dec
 
 #-----------------------------------------------------------------------------
 # Test functions
 #-----------------------------------------------------------------------------
 
 @contextmanager
 def greedy_completion():
     ip = get_ipython()
     greedy_original = ip.Completer.greedy
     try:
         ip.Completer.greedy = True
         yield
     finally:
         ip.Completer.greedy = greedy_original
 
 def test_protect_filename():
     if sys.platform == 'win32':
         pairs = [('abc','abc'),
                  (' abc','" abc"'),
                  ('a bc','"a bc"'),
                  ('a  bc','"a  bc"'),
                  ('  bc','"  bc"'),
                  ]
     else:
         pairs = [('abc','abc'),
                  (' abc',r'\ abc'),
                  ('a bc',r'a\ bc'),
                  ('a  bc',r'a\ \ bc'),
                  ('  bc',r'\ \ bc'),
                  # On posix, we also protect parens and other special characters.
                  ('a(bc',r'a\(bc'),
                  ('a)bc',r'a\)bc'),
                  ('a( )bc',r'a\(\ \)bc'),
                  ('a[1]bc', r'a\[1\]bc'),
                  ('a{1}bc', r'a\{1\}bc'),
                  ('a#bc', r'a\#bc'),
                  ('a?bc', r'a\?bc'),
                  ('a=bc', r'a\=bc'),
                  ('a\\bc', r'a\\bc'),
                  ('a|bc', r'a\|bc'),
                  ('a;bc', r'a\;bc'),
                  ('a:bc', r'a\:bc'),
                  ("a'bc", r"a\'bc"),
                  ('a*bc', r'a\*bc'),
                  ('a"bc', r'a\"bc'),
                  ('a^bc', r'a\^bc'),
                  ('a&bc', r'a\&bc'),
                  ]
     # run the actual tests
     for s1, s2 in pairs:
         s1p = completer.protect_filename(s1)
         nt.assert_equal(s1p, s2)
 
 
 def check_line_split(splitter, test_specs):
     for part1, part2, split in test_specs:
         cursor_pos = len(part1)
         line = part1+part2
         out = splitter.split_line(line, cursor_pos)
         nt.assert_equal(out, split)
 
 
 def test_line_split():
     """Basic line splitter test with default specs."""
     sp = completer.CompletionSplitter()
     # The format of the test specs is: part1, part2, expected answer.  Parts 1
     # and 2 are joined into the 'line' sent to the splitter, as if the cursor
     # was at the end of part1.  So an empty part2 represents someone hitting
     # tab at the end of the line, the most common case.
     t = [('run some/scrip', '', 'some/scrip'),
          ('run scripts/er', 'ror.py foo', 'scripts/er'),
          ('echo $HOM', '', 'HOM'),
          ('print sys.pa', '', 'sys.pa'),
          ('print(sys.pa', '', 'sys.pa'),
          ("execfile('scripts/er", '', 'scripts/er'),
          ('a[x.', '', 'x.'),
          ('a[x.', 'y', 'x.'),
          ('cd "some_file/', '', 'some_file/'),
          ]
     check_line_split(sp, t)
     # Ensure splitting works OK with unicode by re-running the tests with
     # all inputs turned into unicode
     check_line_split(sp, [ map(unicode_type, p) for p in t] )
 
 
 def test_custom_completion_error():
     """Test that errors from custom attribute completers are silenced."""
     ip = get_ipython()
     class A(object): pass
     ip.user_ns['a'] = A()
     
     @complete_object.when_type(A)
     def complete_A(a, existing_completions):
         raise TypeError("this should be silenced")
     
     ip.complete("a.")
 
 
 def test_unicode_completions():
     ip = get_ipython()
     # Some strings that trigger different types of completion.  Check them both
     # in str and unicode forms
     s = ['ru', '%ru', 'cd /', 'floa', 'float(x)/']
     for t in s + list(map(unicode_type, s)):
         # We don't need to check exact completion values (they may change
         # depending on the state of the namespace, but at least no exceptions
         # should be thrown and the return value should be a pair of text, list
         # values.
         text, matches = ip.complete(t)
         nt.assert_true(isinstance(text, string_types))
         nt.assert_true(isinstance(matches, list))
 
 def test_latex_completions():
     from IPython.core.latex_symbols import latex_symbols
     import random
     ip = get_ipython()
     # Test some random unicode symbols
     keys = random.sample(latex_symbols.keys(), 10)
     for k in keys:
         text, matches = ip.complete(k)
         nt.assert_equal(len(matches),1)
         nt.assert_equal(text, k)
         nt.assert_equal(matches[0], latex_symbols[k])
     # Test a more complex line
     text, matches = ip.complete(u'print(\\alpha')
     nt.assert_equals(text, u'\\alpha')
     nt.assert_equals(matches[0], latex_symbols['\\alpha'])
     # Test multiple matching latex symbols
     text, matches = ip.complete(u'\\al')
     nt.assert_in('\\alpha', matches)
     nt.assert_in('\\aleph', matches)
 
 
 
 
[e-mail](sys.version_info[0] >= 3, 'This test only apply on python3')
 def test_back_latex_completion():
     ip = get_ipython()
 
     # do not return more than 1 matches fro \beta, only the latex one.
     name, matches = ip.complete('\\β')
     nt.assert_equal(len(matches), 1)
     nt.assert_equal(matches[0], '\\beta')
 
[e-mail](sys.version_info[0] >= 3, 'This test only apply on python3')
 def test_back_unicode_completion():
     ip = get_ipython()
     
     name, matches = ip.complete('\\Ⅴ')
     nt.assert_equal(len(matches), 1)
     nt.assert_equal(matches[0], '\\ROMAN NUMERAL FIVE')
 
 
[e-mail](sys.version_info[0] >= 3, 'This test only apply on python3')
 def test_forward_unicode_completion():
     ip = get_ipython()
     
     name, matches = ip.complete('\\ROMAN NUMERAL FIVE')
     nt.assert_equal(len(matches), 1)
     nt.assert_equal(matches[0], 'Ⅴ')
 
[e-mail](sys.version_info[0] >= 3, 'This test only apply on python3')
 @dec.knownfailureif(sys.platform == 'win32', 'Fails if there is a C:\\j... path')
 def test_no_ascii_back_completion():
     ip = get_ipython()
     with TemporaryWorkingDirectory():  # Avoid any filename completions
         # single ascii letter that don't have yet completions
         for letter in 'jJ' :
             name, matches = ip.complete('\\'+letter)
             nt.assert_equal(matches, [])
 
 
 
 
 class CompletionSplitterTestCase(unittest.TestCase):
     def setUp(self):
         self.sp = completer.CompletionSplitter()
 
     def test_delim_setting(self):
         self.sp.delims = ' '
         nt.assert_equal(self.sp.delims, ' ')
         nt.assert_equal(self.sp._delim_expr, '[\ ]')
 
     def test_spaces(self):
         """Test with only spaces as split chars."""
         self.sp.delims = ' '
         t = [('foo', '', 'foo'),
              ('run foo', '', 'foo'),
              ('run foo', 'bar', 'foo'),
              ]
         check_line_split(self.sp, t)
 
 
 def test_has_open_quotes1():
     for s in ["'", "'''", "'hi' '"]:
         nt.assert_equal(completer.has_open_quotes(s), "'")
 
 
 def test_has_open_quotes2():
     for s in ['"', '"""', '"hi" "']:
         nt.assert_equal(completer.has_open_quotes(s), '"')
 
 
 def test_has_open_quotes3():
     for s in ["''", "''' '''", "'hi' 'ipython'"]:
         nt.assert_false(completer.has_open_quotes(s))
 
 
 def test_has_open_quotes4():
     for s in ['""', '""" """', '"hi" "ipython"']:
         nt.assert_false(completer.has_open_quotes(s))
 
 
 @knownfailureif(sys.platform == 'win32', "abspath completions fail on Windows")
 def test_abspath_file_completions():
     ip = get_ipython()
     with TemporaryDirectory() as tmpdir:
         prefix = os.path.join(tmpdir, 'foo')
         suffixes = ['1', '2']
         names = [prefix+s for s in suffixes]
         for n in names:
             open(n, 'w').close()
 
         # Check simple completion
         c = ip.complete(prefix)[1]
         nt.assert_equal(c, names)
 
         # Now check with a function call
         cmd = 'a = f("%s' % prefix
         c = ip.complete(prefix, cmd)[1]
         comp = [prefix+s for s in suffixes]
         nt.assert_equal(c, comp)
 
 
 def test_local_file_completions():
     ip = get_ipython()
     with TemporaryWorkingDirectory():
         prefix = './foo'
         suffixes = ['1', '2']
         names = [prefix+s for s in suffixes]
         for n in names:
             open(n, 'w').close()
 
         # Check simple completion
         c = ip.complete(prefix)[1]
         nt.assert_equal(c, names)
 
         # Now check with a function call
         cmd = 'a = f("%s' % prefix
         c = ip.complete(prefix, cmd)[1]
         comp = set(prefix+s for s in suffixes)
         nt.assert_true(comp.issubset(set(c)))
 
 
 def test_greedy_completions():
     ip = get_ipython()
     ip.ex('a=list(range(5))')
     _,c = ip.complete('.',line='a[0].')
     nt.assert_false('.real' in c,
                     "Shouldn't have completed on a[0]: %s"%c)
     with greedy_completion():
         def _(line, cursor_pos, expect, message):
             _,c = ip.complete('.', line=line, cursor_pos=cursor_pos)
             nt.assert_in(expect, c, message%c)
 
         yield _, 'a[0].', 5, 'a[0].real', "Should have completed on a[0].: %s"
         yield _, 'a[0].r', 6, 'a[0].real', "Should have completed on a[0].r: %s"
         
         if sys.version_info > (3,4):
             yield _, 'a[0].from_', 10, 'a[0].from_bytes', "Should have completed on a[0].from_: %s"
 
 
 
 def test_omit__names():
     # also happens to test IPCompleter as a configurable
     ip = get_ipython()
     ip._hidden_attr = 1
     ip._x = {}
     c = ip.Completer
     ip.ex('ip=get_ipython()')
     cfg = Config()
     cfg.IPCompleter.omit__names = 0
     c.update_config(cfg)
     s,matches = c.complete('ip.')
     nt.assert_in('ip.__str__', matches)
     nt.assert_in('ip._hidden_attr', matches)
     cfg = Config()
     cfg.IPCompleter.omit__names = 1
     c.update_config(cfg)
     s,matches = c.complete('ip.')
     nt.assert_not_in('ip.__str__', matches)
     nt.assert_in('ip._hidden_attr', matches)
     cfg = Config()
     cfg.IPCompleter.omit__names = 2
     c.update_config(cfg)
     s,matches = c.complete('ip.')
     nt.assert_not_in('ip.__str__', matches)
     nt.assert_not_in('ip._hidden_attr', matches)
     s,matches = c.complete('ip._x.')
     nt.assert_in('ip._x.keys', matches)
     del ip._hidden_attr
 
 
 def test_limit_to__all__False_ok():
     ip = get_ipython()
     c = ip.Completer
     ip.ex('class D: x=24')
     ip.ex('d=D()')
     cfg = Config()
     cfg.IPCompleter.limit_to__all__ = False
     c.update_config(cfg)
     s, matches = c.complete('d.')
     nt.assert_in('d.x', matches)
 
 
 def test_get__all__entries_ok():
     class A(object):
         __all__ = ['x', 1]
     words = completer.get__all__entries(A())
     nt.assert_equal(words, ['x'])
 
 
 def test_get__all__entries_no__all__ok():
     class A(object):
         pass
     words = completer.get__all__entries(A())
     nt.assert_equal(words, [])
 
 
 def test_func_kw_completions():
     ip = get_ipython()
     c = ip.Completer
     ip.ex('def myfunc(a=1,b=2): return a+b')
     s, matches = c.complete(None, 'myfunc(1,b')
     nt.assert_in('b=', matches)
     # Simulate completing with cursor right after b (pos==10):
     s, matches = c.complete(None, 'myfunc(1,b)', 10)
     nt.assert_in('b=', matches)
     s, matches = c.complete(None, 'myfunc(a="escaped\\")string",b')
     nt.assert_in('b=', matches)
     #builtin function
     s, matches = c.complete(None, 'min(k, k')
     nt.assert_in('key=', matches)
 
 
 def test_default_arguments_from_docstring():
     ip = get_ipython()
     c = ip.Completer
     kwd = c._default_arguments_from_docstring(
         'min(iterable[, key=func]) -> value')
     nt.assert_equal(kwd, ['key'])
     #with cython type etc
     kwd = c._default_arguments_from_docstring(
         'Minuit.migrad(self, int ncall=10000, resume=True, int nsplit=1)\n')
     nt.assert_equal(kwd, ['ncall', 'resume', 'nsplit'])
     #white spaces
     kwd = c._default_arguments_from_docstring(
         '\n Minuit.migrad(self, int ncall=10000, resume=True, int nsplit=1)\n')
     nt.assert_equal(kwd, ['ncall', 'resume', 'nsplit'])
 
 def test_line_magics():
     ip = get_ipython()
     c = ip.Completer
     s, matches = c.complete(None, 'lsmag')
     nt.assert_in('%lsmagic', matches)
     s, matches = c.complete(None, '%lsmag')
     nt.assert_in('%lsmagic', matches)
 
 
 def test_cell_magics():
     from IPython.core.magic import register_cell_magic
 
     @register_cell_magic
     def _foo_cellm(line, cell):
         pass
     
     ip = get_ipython()
     c = ip.Completer
 
     s, matches = c.complete(None, '_foo_ce')
     nt.assert_in('%%_foo_cellm', matches)
     s, matches = c.complete(None, '%%_foo_ce')
     nt.assert_in('%%_foo_cellm', matches)
 
 
 def test_line_cell_magics():
     from IPython.core.magic import register_line_cell_magic
 
     @register_line_cell_magic
     def _bar_cellm(line, cell):
         pass
     
     ip = get_ipython()
     c = ip.Completer
 
     # The policy here is trickier, see comments in completion code.  The
     # returned values depend on whether the user passes %% or not explicitly,
     # and this will show a difference if the same name is both a line and cell
     # magic.
     s, matches = c.complete(None, '_bar_ce')
     nt.assert_in('%_bar_cellm', matches)
     nt.assert_in('%%_bar_cellm', matches)
     s, matches = c.complete(None, '%_bar_ce')
     nt.assert_in('%_bar_cellm', matches)
     nt.assert_in('%%_bar_cellm', matches)
     s, matches = c.complete(None, '%%_bar_ce')
     nt.assert_not_in('%_bar_cellm', matches)
     nt.assert_in('%%_bar_cellm', matches)
 
 
 def test_magic_completion_order():
 
     ip = get_ipython()
     c = ip.Completer
 
     # Test ordering of magics and non-magics with the same name
     # We want the non-magic first
 
     # Before importing matplotlib, there should only be one option:
 
     text, matches = c.complete('mat')
     nt.assert_equal(matches, ["%matplotlib"])
 
 
     ip.run_cell("matplotlib = 1")  # introduce name into namespace
 
     # After the import, there should be two options, ordered like this:
     text, matches = c.complete('mat')
     nt.assert_equal(matches, ["matplotlib", "%matplotlib"])
 
 
     ip.run_cell("timeit = 1")  # define a user variable called 'timeit'
 
     # Order of user variable and line and cell magics with same name:
     text, matches = c.complete('timeit')
     nt.assert_equal(matches, ["timeit", "%timeit","%%timeit"])
 
 
 def test_dict_key_completion_string():
     """Test dictionary key completion for string keys"""
     ip = get_ipython()
     complete = ip.Completer.complete
 
     ip.user_ns['d'] = {'abc': None}
 
     # check completion at different stages
     _, matches = complete(line_buffer="d[")
     nt.assert_in("'abc'", matches)
     nt.assert_not_in("'abc']", matches)
 
     _, matches = complete(line_buffer="d['")
     nt.assert_in("abc", matches)
     nt.assert_not_in("abc']", matches)
 
     _, matches = complete(line_buffer="d['a")
     nt.assert_in("abc", matches)
     nt.assert_not_in("abc']", matches)
 
     # check use of different quoting
     _, matches = complete(line_buffer="d[\"")
     nt.assert_in("abc", matches)
     nt.assert_not_in('abc\"]', matches)
 
     _, matches = complete(line_buffer="d[\"a")
     nt.assert_in("abc", matches)
     nt.assert_not_in('abc\"]', matches)
 
     # check sensitivity to following context
     _, matches = complete(line_buffer="d[]", cursor_pos=2)
     nt.assert_in("'abc'", matches)
 
     _, matches = complete(line_buffer="d['']", cursor_pos=3)
     nt.assert_in("abc", matches)
     nt.assert_not_in("abc'", matches)
     nt.assert_not_in("abc']", matches)
 
     # check multiple solutions are correctly returned and that noise is not
     ip.user_ns['d'] = {'abc': None, 'abd': None, 'bad': None, object(): None,
                        5: None}
 
     _, matches = complete(line_buffer="d['a")
     nt.assert_in("abc", matches)
     nt.assert_in("abd", matches)
     nt.assert_not_in("bad", matches)
     assert not any(m.endswith((']', '"', "'")) for m in matches), matches
 
     # check escaping and whitespace
     ip.user_ns['d'] = {'a\nb': None, 'a\'b': None, 'a"b': None, 'a word': None}
     _, matches = complete(line_buffer="d['a")
     nt.assert_in("a\\nb", matches)
     nt.assert_in("a\\'b", matches)
     nt.assert_in("a\"b", matches)
     nt.assert_in("a word", matches)
     assert not any(m.endswith((']', '"', "'")) for m in matches), matches
 
     # - can complete on non-initial word of the string
     _, matches = complete(line_buffer="d['a w")
     nt.assert_in("word", matches)
 
     # - understands quote escaping
     _, matches = complete(line_buffer="d['a\\'")
     nt.assert_in("b", matches)
 
     # - default quoting should work like repr
     _, matches = complete(line_buffer="d[")
     nt.assert_in("\"a'b\"", matches)
 
     # - when opening quote with ", possible to match with unescaped apostrophe
     _, matches = complete(line_buffer="d[\"a'")
     nt.assert_in("b", matches)
 
     # need to not split at delims that readline won't split at
     if '-' not in ip.Completer.splitter.delims:
         ip.user_ns['d'] = {'before-after': None}
         _, matches = complete(line_buffer="d['before-af")
         nt.assert_in('before-after', matches)
 
 def test_dict_key_completion_contexts():
     """Test expression contexts in which dict key completion occurs"""
     ip = get_ipython()
     complete = ip.Completer.complete
     d = {'abc': None}
     ip.user_ns['d'] = d
 
     class C:
         data = d
     ip.user_ns['C'] = C
     ip.user_ns['get'] = lambda: d
 
     def assert_no_completion(**kwargs):
         _, matches = complete(**kwargs)
         nt.assert_not_in('abc', matches)
         nt.assert_not_in('abc\'', matches)
         nt.assert_not_in('abc\']', matches)
         nt.assert_not_in('\'abc\'', matches)
         nt.assert_not_in('\'abc\']', matches)
 
     def assert_completion(**kwargs):
         _, matches = complete(**kwargs)
         nt.assert_in("'abc'", matches)
         nt.assert_not_in("'abc']", matches)
 
     # no completion after string closed, even if reopened
     assert_no_completion(line_buffer="d['a'")
     assert_no_completion(line_buffer="d[\"a\"")
     assert_no_completion(line_buffer="d['a' + ")
     assert_no_completion(line_buffer="d['a' + '")
 
     # completion in non-trivial expressions
     assert_completion(line_buffer="+ d[")
     assert_completion(line_buffer="(d[")
     assert_completion(line_buffer="C.data[")
 
     # greedy flag
     def assert_completion(**kwargs):
         _, matches = complete(**kwargs)
         nt.assert_in("get()['abc']", matches)
     
     assert_no_completion(line_buffer="get()[")
     with greedy_completion():
         assert_completion(line_buffer="get()[")
         assert_completion(line_buffer="get()['")
         assert_completion(line_buffer="get()['a")
         assert_completion(line_buffer="get()['ab")
         assert_completion(line_buffer="get()['abc")
 
 
 
[e-mail](sys.version_info[0] >= 3, 'This test only applies in Py>=3')
 def test_dict_key_completion_bytes():
     """Test handling of bytes in dict key completion"""
     ip = get_ipython()
     complete = ip.Completer.complete
 
     ip.user_ns['d'] = {'abc': None, b'abd': None}
 
     _, matches = complete(line_buffer="d[")
     nt.assert_in("'abc'", matches)
     nt.assert_in("b'abd'", matches)
 
     if False:  # not currently implemented
         _, matches = complete(line_buffer="d[b")
         nt.assert_in("b'abd'", matches)
         nt.assert_not_in("b'abc'", matches)
 
         _, matches = complete(line_buffer="d[b'")
         nt.assert_in("abd", matches)
         nt.assert_not_in("abc", matches)
 
         _, matches = complete(line_buffer="d[B'")
         nt.assert_in("abd", matches)
         nt.assert_not_in("abc", matches)
 
         _, matches = complete(line_buffer="d['")
         nt.assert_in("abc", matches)
         nt.assert_not_in("abd", matches)
 
 
[e-mail](sys.version_info[0] < 3, 'This test only applies in Py<3')
 def test_dict_key_completion_unicode_py2():
     """Test handling of unicode in dict key completion"""
     ip = get_ipython()
     complete = ip.Completer.complete
 
     ip.user_ns['d'] = {u'abc': None,
                        u'a\u05d0b': None}
 
     _, matches = complete(line_buffer="d[")
     nt.assert_in("u'abc'", matches)
     nt.assert_in("u'a\\u05d0b'", matches)
 
     _, matches = complete(line_buffer="d['a")
     nt.assert_in("abc", matches)
     nt.assert_not_in("a\\u05d0b", matches)
 
     _, matches = complete(line_buffer="d[u'a")
     nt.assert_in("abc", matches)
     nt.assert_in("a\\u05d0b", matches)
 
     _, matches = complete(line_buffer="d[U'a")
     nt.assert_in("abc", matches)
     nt.assert_in("a\\u05d0b", matches)
 
     # query using escape
     if sys.platform != 'win32':
         # Known failure on Windows
         _, matches = complete(line_buffer=u"d[u'a\\u05d0")
         nt.assert_in("u05d0b", matches)  # tokenized after \\
 
     # query using character
     _, matches = complete(line_buffer=u"d[u'a\u05d0")
     nt.assert_in(u"a\u05d0b", matches)
     
     with greedy_completion():
         _, matches = complete(line_buffer="d[")
         nt.assert_in("d[u'abc']", matches)
         nt.assert_in("d[u'a\\u05d0b']", matches)
 
         _, matches = complete(line_buffer="d['a")
         nt.assert_in("d['abc']", matches)
         nt.assert_not_in("d[u'a\\u05d0b']", matches)
 
         _, matches = complete(line_buffer="d[u'a")
         nt.assert_in("d[u'abc']", matches)
         nt.assert_in("d[u'a\\u05d0b']", matches)
 
         _, matches = complete(line_buffer="d[U'a")
         nt.assert_in("d[U'abc']", matches)
         nt.assert_in("d[U'a\\u05d0b']", matches)
 
         # query using escape
         _, matches = complete(line_buffer=u"d[u'a\\u05d0")
         nt.assert_in("d[u'a\\u05d0b']", matches)  # tokenized after \\
 
         # query using character
         _, matches = complete(line_buffer=u"d[u'a\u05d0")
         nt.assert_in(u"d[u'a\u05d0b']", matches)
 
 
[e-mail](sys.version_info[0] >= 3, 'This test only applies in Py>=3')
 def test_dict_key_completion_unicode_py3():
     """Test handling of unicode in dict key completion"""
     ip = get_ipython()
     complete = ip.Completer.complete
 
     ip.user_ns['d'] = {u'a\u05d0': None}
 
     # query using escape
     if sys.platform != 'win32':
         # Known failure on Windows
         _, matches = complete(line_buffer="d['a\\u05d0")
         nt.assert_in("u05d0", matches)  # tokenized after \\
 
     # query using character
     _, matches = complete(line_buffer="d['a\u05d0")
     nt.assert_in(u"a\u05d0", matches)
     
     with greedy_completion():
         # query using escape
         _, matches = complete(line_buffer="d['a\\u05d0")
         nt.assert_in("d['a\\u05d0']", matches)  # tokenized after \\
 
         # query using character
         _, matches = complete(line_buffer="d['a\u05d0")
         nt.assert_in(u"d['a\u05d0']", matches)
         
 
 
 @dec.skip_without('numpy')
 def test_struct_array_key_completion():
     """Test dict key completion applies to numpy struct arrays"""
     import numpy
     ip = get_ipython()
     complete = ip.Completer.complete
     ip.user_ns['d'] = numpy.array([], dtype=[('hello', 'f'), ('world', 'f')])
     _, matches = complete(line_buffer="d['")
     nt.assert_in("hello", matches)
     nt.assert_in("world", matches)
     # complete on the numpy struct itself
     dt = numpy.dtype([('my_head', [('my_dt', '>u4'), ('my_df', '>u4')]),
                       ('my_data', '>f4', 5)])
     x = numpy.zeros(2, dtype=dt)
     ip.user_ns['d'] = x[1]
     _, matches = complete(line_buffer="d['")
     nt.assert_in("my_head", matches)
     nt.assert_in("my_data", matches)
     # complete on a nested level
     with greedy_completion():
         ip.user_ns['d'] = numpy.zeros(2, dtype=dt)
         _, matches = complete(line_buffer="d[1]['my_head']['")
         nt.assert_true(any(["my_dt" in m for m in matches]))
         nt.assert_true(any(["my_df" in m for m in matches]))
 
 
 @dec.skip_without('pandas')
 def test_dataframe_key_completion():
     """Test dict key completion applies to pandas DataFrames"""
     import pandas
     ip = get_ipython()
     complete = ip.Completer.complete
     ip.user_ns['d'] = pandas.DataFrame({'hello': [1], 'world': [2]})
     _, matches = complete(line_buffer="d['")
     nt.assert_in("hello", matches)
     nt.assert_in("world", matches)
 
 
 def test_dict_key_completion_invalids():
     """Smoke test cases dict key completion can't handle"""
     ip = get_ipython()
     complete = ip.Completer.complete
 
     ip.user_ns['no_getitem'] = None
     ip.user_ns['no_keys'] = []
     ip.user_ns['cant_call_keys'] = dict
     ip.user_ns['empty'] = {}
     ip.user_ns['d'] = {'abc': 5}
 
     _, matches = complete(line_buffer="no_getitem['")
     _, matches = complete(line_buffer="no_keys['")
     _, matches = complete(line_buffer="cant_call_keys['")
     _, matches = complete(line_buffer="empty['")
     _, matches = complete(line_buffer="name_error['")
     _, matches = complete(line_buffer="d['\\")  # incomplete escape
 
 class KeyCompletable(object):
     def __init__(self, things=()):
         self.things = things
 
     def _ipython_key_completions_(self):
         return list(self.things)
 
 def test_object_key_completion():
     ip = get_ipython()
     ip.user_ns['key_completable'] = KeyCompletable(['qwerty', 'qwick'])
 
     _, matches = ip.Completer.complete(line_buffer="key_completable['qw")
     nt.assert_in('qwerty', matches)
     nt.assert_in('qwick', matches)
 
 
 def test_aimport_module_completer():
     ip = get_ipython()
     _, matches = ip.complete('i', '%aimport i')
     nt.assert_in('io', matches)
     nt.assert_not_in('int', matches)
 
 def test_nested_import_module_completer():
     ip = get_ipython()
     _, matches = ip.complete(None, 'import IPython.co', 17)
     nt.assert_in('IPython.core', matches)
     nt.assert_not_in('import IPython.core', matches)
     nt.assert_not_in('IPython.display', matches)
 
 def test_import_module_completer():
     ip = get_ipython()
     _, matches = ip.complete('i', 'import i')
     nt.assert_in('io', matches)
     nt.assert_not_in('int', matches)
 
 def test_from_module_completer():
     ip = get_ipython()
     _, matches = ip.complete('B', 'from io import B', 16)
     nt.assert_in('BytesIO', matches)
     nt.assert_not_in('BaseException', matches)

IPython/core/tests/test_interactiveshell.py

@@ -1,950 +1,918 b''
 # -*- coding: utf-8 -*-
 """Tests for the key interactiveshell module.
 
 Historically the main classes in interactiveshell have been under-tested.  This
 module should grow as many single-method tests as possible to trap many of the
 recurring bugs we seem to encounter with high-level interaction.
 """
 
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
 import ast
 import os
 import signal
 import shutil
 import sys
 import tempfile
 import unittest
 try:
     from unittest import mock
 except ImportError:
     import mock
 from os.path import join
 
 import nose.tools as nt
 
 from IPython.core.error import InputRejected
 from IPython.core.inputtransformer import InputTransformer
 from IPython.testing.decorators import (
     skipif, skip_win32, onlyif_unicode_paths, onlyif_cmds_exist,
 )
 from IPython.testing import tools as tt
 from IPython.utils.process import find_cmd
 from IPython.utils import py3compat
 from IPython.utils.py3compat import unicode_type, PY3
 
 if PY3:
     from io import StringIO
 else:
     from StringIO import StringIO
 
 #-----------------------------------------------------------------------------
 # Globals
 #-----------------------------------------------------------------------------
 # This is used by every single test, no point repeating it ad nauseam
 ip = get_ipython()
 
 #-----------------------------------------------------------------------------
 # Tests
 #-----------------------------------------------------------------------------
 
 class DerivedInterrupt(KeyboardInterrupt):
     pass
 
 class InteractiveShellTestCase(unittest.TestCase):
     def test_naked_string_cells(self):
         """Test that cells with only naked strings are fully executed"""
         # First, single-line inputs
         ip.run_cell('"a"\n')
         self.assertEqual(ip.user_ns['_'], 'a')
         # And also multi-line cells
         ip.run_cell('"""a\nb"""\n')
         self.assertEqual(ip.user_ns['_'], 'a\nb')
 
     def test_run_empty_cell(self):
         """Just make sure we don't get a horrible error with a blank
         cell of input. Yes, I did overlook that."""
         old_xc = ip.execution_count
         res = ip.run_cell('')
         self.assertEqual(ip.execution_count, old_xc)
         self.assertEqual(res.execution_count, None)
 
     def test_run_cell_multiline(self):
         """Multi-block, multi-line cells must execute correctly.
         """
         src = '\n'.join(["x=1",
                          "y=2",
                          "if 1:",
                          "    x += 1",
                          "    y += 1",])
         res = ip.run_cell(src)
         self.assertEqual(ip.user_ns['x'], 2)
         self.assertEqual(ip.user_ns['y'], 3)
         self.assertEqual(res.success, True)
         self.assertEqual(res.result, None)
 
     def test_multiline_string_cells(self):
         "Code sprinkled with multiline strings should execute (GH-306)"
         ip.run_cell('tmp=0')
         self.assertEqual(ip.user_ns['tmp'], 0)
         res = ip.run_cell('tmp=1;"""a\nb"""\n')
         self.assertEqual(ip.user_ns['tmp'], 1)
         self.assertEqual(res.success, True)
         self.assertEqual(res.result, "a\nb")
 
     def test_dont_cache_with_semicolon(self):
         "Ending a line with semicolon should not cache the returned object (GH-307)"
         oldlen = len(ip.user_ns['Out'])
         for cell in ['1;', '1;1;']:
             res = ip.run_cell(cell, store_history=True)
             newlen = len(ip.user_ns['Out'])
             self.assertEqual(oldlen, newlen)
             self.assertIsNone(res.result)
         i = 0
         #also test the default caching behavior
         for cell in ['1', '1;1']:
             ip.run_cell(cell, store_history=True)
             newlen = len(ip.user_ns['Out'])
             i += 1
             self.assertEqual(oldlen+i, newlen)
 
     def test_syntax_error(self):
         res = ip.run_cell("raise = 3")
         self.assertIsInstance(res.error_before_exec, SyntaxError)
 
     def test_In_variable(self):
         "Verify that In variable grows with user input (GH-284)"
         oldlen = len(ip.user_ns['In'])
         ip.run_cell('1;', store_history=True)
         newlen = len(ip.user_ns['In'])
         self.assertEqual(oldlen+1, newlen)
         self.assertEqual(ip.user_ns['In'][-1],'1;')
         
     def test_magic_names_in_string(self):
         ip.run_cell('a = """\n%exit\n"""')
         self.assertEqual(ip.user_ns['a'], '\n%exit\n')
     
     def test_trailing_newline(self):
         """test that running !(command) does not raise a SyntaxError"""
         ip.run_cell('!(true)\n', False)
         ip.run_cell('!(true)\n\n\n', False)
     
     def test_gh_597(self):
         """Pretty-printing lists of objects with non-ascii reprs may cause
         problems."""
         class Spam(object):
           def __repr__(self):
             return "\xe9"*50
         import IPython.core.formatters
         f = IPython.core.formatters.PlainTextFormatter()
         f([Spam(),Spam()])
     
 
     def test_future_flags(self):
         """Check that future flags are used for parsing code (gh-777)"""
         ip.run_cell('from __future__ import print_function')
         try:
             ip.run_cell('prfunc_return_val = print(1,2, sep=" ")')
             assert 'prfunc_return_val' in ip.user_ns
         finally:
             # Reset compiler flags so we don't mess up other tests.
             ip.compile.reset_compiler_flags()
 
     def test_future_unicode(self):
         """Check that unicode_literals is imported from __future__ (gh #786)"""
         try:
             ip.run_cell(u'byte_str = "a"')
             assert isinstance(ip.user_ns['byte_str'], str) # string literals are byte strings by default
             ip.run_cell('from __future__ import unicode_literals')
             ip.run_cell(u'unicode_str = "a"')
             assert isinstance(ip.user_ns['unicode_str'], unicode_type) # strings literals are now unicode
         finally:
             # Reset compiler flags so we don't mess up other tests.
             ip.compile.reset_compiler_flags()
     
     def test_can_pickle(self):
         "Can we pickle objects defined interactively (GH-29)"
         ip = get_ipython()
         ip.reset()
         ip.run_cell(("class Mylist(list):\n"
                      "    def __init__(self,x=[]):\n"
                      "        list.__init__(self,x)"))
         ip.run_cell("w=Mylist([1,2,3])")
         
         from pickle import dumps
         
         # We need to swap in our main module - this is only necessary
         # inside the test framework, because IPython puts the interactive module
         # in place (but the test framework undoes this).
         _main = sys.modules['__main__']
         sys.modules['__main__'] = ip.user_module
         try:
             res = dumps(ip.user_ns["w"])
         finally:
             sys.modules['__main__'] = _main
         self.assertTrue(isinstance(res, bytes))
         
     def test_global_ns(self):
         "Code in functions must be able to access variables outside them."
         ip = get_ipython()
         ip.run_cell("a = 10")
         ip.run_cell(("def f(x):\n"
                      "    return x + a"))
         ip.run_cell("b = f(12)")
         self.assertEqual(ip.user_ns["b"], 22)
 
     def test_bad_custom_tb(self):
         """Check that InteractiveShell is protected from bad custom exception handlers"""
         ip.set_custom_exc((IOError,), lambda etype,value,tb: 1/0)
         self.assertEqual(ip.custom_exceptions, (IOError,))
         with tt.AssertPrints("Custom TB Handler failed", channel='stderr'):
             ip.run_cell(u'raise IOError("foo")')
         self.assertEqual(ip.custom_exceptions, ())
 
     def test_bad_custom_tb_return(self):
         """Check that InteractiveShell is protected from bad return types in custom exception handlers"""
         ip.set_custom_exc((NameError,),lambda etype,value,tb, tb_offset=None: 1)
         self.assertEqual(ip.custom_exceptions, (NameError,))
         with tt.AssertPrints("Custom TB Handler failed", channel='stderr'):
             ip.run_cell(u'a=abracadabra')
         self.assertEqual(ip.custom_exceptions, ())
 
     def test_drop_by_id(self):
         myvars = {"a":object(), "b":object(), "c": object()}
         ip.push(myvars, interactive=False)
         for name in myvars:
             assert name in ip.user_ns, name
             assert name in ip.user_ns_hidden, name
         ip.user_ns['b'] = 12
         ip.drop_by_id(myvars)
         for name in ["a", "c"]:
             assert name not in ip.user_ns, name
             assert name not in ip.user_ns_hidden, name
         assert ip.user_ns['b'] == 12
         ip.reset()
 
     def test_var_expand(self):
         ip.user_ns['f'] = u'Ca\xf1o'
         self.assertEqual(ip.var_expand(u'echo $f'), u'echo Ca\xf1o')
         self.assertEqual(ip.var_expand(u'echo {f}'), u'echo Ca\xf1o')
         self.assertEqual(ip.var_expand(u'echo {f[:-1]}'), u'echo Ca\xf1')
         self.assertEqual(ip.var_expand(u'echo {1*2}'), u'echo 2')
 
         ip.user_ns['f'] = b'Ca\xc3\xb1o'
         # This should not raise any exception:
         ip.var_expand(u'echo $f')
     
     def test_var_expand_local(self):
         """Test local variable expansion in !system and %magic calls"""
         # !system
         ip.run_cell('def test():\n'
                     '    lvar = "ttt"\n'
                     '    ret = !echo {lvar}\n'
                     '    return ret[0]\n')
         res = ip.user_ns['test']()
         nt.assert_in('ttt', res)
         
         # %magic
         ip.run_cell('def makemacro():\n'
                     '    macroname = "macro_var_expand_locals"\n'
                     '    %macro {macroname} codestr\n')
         ip.user_ns['codestr'] = "str(12)"
         ip.run_cell('makemacro()')
         nt.assert_in('macro_var_expand_locals', ip.user_ns)
     
     def test_var_expand_self(self):
         """Test variable expansion with the name 'self', which was failing.
         
         See https://github.com/ipython/ipython/issues/1878#issuecomment-7698218
         """
         ip.run_cell('class cTest:\n'
                     '  classvar="see me"\n'
                     '  def test(self):\n'
                     '    res = !echo Variable: {self.classvar}\n'
                     '    return res[0]\n')
         nt.assert_in('see me', ip.user_ns['cTest']().test())
     
     def test_bad_var_expand(self):
         """var_expand on invalid formats shouldn't raise"""
         # SyntaxError
         self.assertEqual(ip.var_expand(u"{'a':5}"), u"{'a':5}")
         # NameError
         self.assertEqual(ip.var_expand(u"{asdf}"), u"{asdf}")
         # ZeroDivisionError
         self.assertEqual(ip.var_expand(u"{1/0}"), u"{1/0}")
     
     def test_silent_postexec(self):
         """run_cell(silent=True) doesn't invoke pre/post_run_cell callbacks"""
         pre_explicit = mock.Mock()
         pre_always = mock.Mock()
         post_explicit = mock.Mock()
         post_always = mock.Mock()
         
         ip.events.register('pre_run_cell', pre_explicit)
         ip.events.register('pre_execute', pre_always)
         ip.events.register('post_run_cell', post_explicit)
         ip.events.register('post_execute', post_always)
         
         try:
             ip.run_cell("1", silent=True)
             assert pre_always.called
             assert not pre_explicit.called
             assert post_always.called
             assert not post_explicit.called
             # double-check that non-silent exec did what we expected
             # silent to avoid
             ip.run_cell("1")
             assert pre_explicit.called
             assert post_explicit.called
         finally:
             # remove post-exec
             ip.events.unregister('pre_run_cell', pre_explicit)
             ip.events.unregister('pre_execute', pre_always)
             ip.events.unregister('post_run_cell', post_explicit)
             ip.events.unregister('post_execute', post_always)
     
     def test_silent_noadvance(self):
         """run_cell(silent=True) doesn't advance execution_count"""
         ec = ip.execution_count
         # silent should force store_history=False
         ip.run_cell("1", store_history=True, silent=True)
         
         self.assertEqual(ec, ip.execution_count)
         # double-check that non-silent exec did what we expected
         # silent to avoid
         ip.run_cell("1", store_history=True)
         self.assertEqual(ec+1, ip.execution_count)
     
     def test_silent_nodisplayhook(self):
         """run_cell(silent=True) doesn't trigger displayhook"""
         d = dict(called=False)
         
         trap = ip.display_trap
         save_hook = trap.hook
         
         def failing_hook(*args, **kwargs):
             d['called'] = True
         
         try:
             trap.hook = failing_hook
             res = ip.run_cell("1", silent=True)
             self.assertFalse(d['called'])
             self.assertIsNone(res.result)
             # double-check that non-silent exec did what we expected
             # silent to avoid
             ip.run_cell("1")
             self.assertTrue(d['called'])
         finally:
             trap.hook = save_hook
 
-    @skipif(sys.version_info[0] >= 3, "softspace removed in py3")
-    def test_print_softspace(self):
-        """Verify that softspace is handled correctly when executing multiple
-        statements.
-
-        In [1]: print 1; print 2
-        1
-        2
-
-        In [2]: print 1,; print 2
-        1 2
-        """
-        
     def test_ofind_line_magic(self):
         from IPython.core.magic import register_line_magic
         
         @register_line_magic
         def lmagic(line):
             "A line magic"
 
         # Get info on line magic
         lfind = ip._ofind('lmagic')
         info = dict(found=True, isalias=False, ismagic=True,
                     namespace = 'IPython internal', obj= lmagic.__wrapped__,
                     parent = None)
         nt.assert_equal(lfind, info)
         
     def test_ofind_cell_magic(self):
         from IPython.core.magic import register_cell_magic
         
         @register_cell_magic
         def cmagic(line, cell):
             "A cell magic"
 
         # Get info on cell magic
         find = ip._ofind('cmagic')
         info = dict(found=True, isalias=False, ismagic=True,
                     namespace = 'IPython internal', obj= cmagic.__wrapped__,
                     parent = None)
         nt.assert_equal(find, info)
 
     def test_ofind_property_with_error(self):
         class A(object):
             @property
             def foo(self):
                 raise NotImplementedError()
         a = A()
 
         found = ip._ofind('a.foo', [('locals', locals())])
         info = dict(found=True, isalias=False, ismagic=False,
                     namespace='locals', obj=A.foo, parent=a)
         nt.assert_equal(found, info)
 
     def test_ofind_multiple_attribute_lookups(self):
         class A(object):
             @property
             def foo(self):
                 raise NotImplementedError()
 
         a = A()
         a.a = A()
         a.a.a = A()
 
         found = ip._ofind('a.a.a.foo', [('locals', locals())])
         info = dict(found=True, isalias=False, ismagic=False,
                     namespace='locals', obj=A.foo, parent=a.a.a)
         nt.assert_equal(found, info)
 
     def test_ofind_slotted_attributes(self):
         class A(object):
             __slots__ = ['foo']
             def __init__(self):
                 self.foo = 'bar'
 
         a = A()
         found = ip._ofind('a.foo', [('locals', locals())])
         info = dict(found=True, isalias=False, ismagic=False,
                     namespace='locals', obj=a.foo, parent=a)
         nt.assert_equal(found, info)
 
         found = ip._ofind('a.bar', [('locals', locals())])
         info = dict(found=False, isalias=False, ismagic=False,
                     namespace=None, obj=None, parent=a)
         nt.assert_equal(found, info)
 
     def test_ofind_prefers_property_to_instance_level_attribute(self):
         class A(object):
             @property
             def foo(self):
                 return 'bar'
         a = A()
         a.__dict__['foo'] = 'baz'
         nt.assert_equal(a.foo, 'bar')
         found = ip._ofind('a.foo', [('locals', locals())])
         nt.assert_is(found['obj'], A.foo)
 
     def test_custom_syntaxerror_exception(self):
         called = []
         def my_handler(shell, etype, value, tb, tb_offset=None):
             called.append(etype)
             shell.showtraceback((etype, value, tb), tb_offset=tb_offset)
 
         ip.set_custom_exc((SyntaxError,), my_handler)
         try:
             ip.run_cell("1f")
             # Check that this was called, and only once.
             self.assertEqual(called, [SyntaxError])
         finally:
             # Reset the custom exception hook
             ip.set_custom_exc((), None)
 
     def test_custom_exception(self):
         called = []
         def my_handler(shell, etype, value, tb, tb_offset=None):
             called.append(etype)
             shell.showtraceback((etype, value, tb), tb_offset=tb_offset)
         
         ip.set_custom_exc((ValueError,), my_handler)
         try:
             res = ip.run_cell("raise ValueError('test')")
             # Check that this was called, and only once.
             self.assertEqual(called, [ValueError])
             # Check that the error is on the result object
             self.assertIsInstance(res.error_in_exec, ValueError)
         finally:
             # Reset the custom exception hook
             ip.set_custom_exc((), None)
     
-    @skipif(sys.version_info[0] >= 3, "no differences with __future__ in py3")
-    def test_future_environment(self):
-        "Can we run code with & without the shell's __future__ imports?"
-        ip.run_cell("from __future__ import division")
-        ip.run_cell("a = 1/2", shell_futures=True)
-        self.assertEqual(ip.user_ns['a'], 0.5)
-        ip.run_cell("b = 1/2", shell_futures=False)
-        self.assertEqual(ip.user_ns['b'], 0)
-        
-        ip.compile.reset_compiler_flags()
-        # This shouldn't leak to the shell's compiler
-        ip.run_cell("from __future__ import division \nc=1/2", shell_futures=False)
-        self.assertEqual(ip.user_ns['c'], 0.5)
-        ip.run_cell("d = 1/2", shell_futures=True)
-        self.assertEqual(ip.user_ns['d'], 0)
-
     def test_mktempfile(self):
         filename = ip.mktempfile()
         # Check that we can open the file again on Windows
         with open(filename, 'w') as f:
             f.write('abc')
 
         filename = ip.mktempfile(data='blah')
         with open(filename, 'r') as f:
             self.assertEqual(f.read(), 'blah')
 
     def test_new_main_mod(self):
         # Smoketest to check that this accepts a unicode module name
         name = u'jiefmw'
         mod = ip.new_main_mod(u'%s.py' % name, name)
         self.assertEqual(mod.__name__, name)
 
     def test_get_exception_only(self):
         try:
             raise KeyboardInterrupt
         except KeyboardInterrupt:
             msg = ip.get_exception_only()
         self.assertEqual(msg, 'KeyboardInterrupt\n')
 
         try:
             raise DerivedInterrupt("foo")
         except KeyboardInterrupt:
             msg = ip.get_exception_only()
-        if sys.version_info[0] <= 2:
-            self.assertEqual(msg, 'DerivedInterrupt: foo\n')
-        else:
-            self.assertEqual(msg, 'IPython.core.tests.test_interactiveshell.DerivedInterrupt: foo\n')
+        self.assertEqual(msg, 'IPython.core.tests.test_interactiveshell.DerivedInterrupt: foo\n')
 
     def test_inspect_text(self):
         ip.run_cell('a = 5')
         text = ip.object_inspect_text('a')
         self.assertIsInstance(text, unicode_type)
 
 
 class TestSafeExecfileNonAsciiPath(unittest.TestCase):
 
     @onlyif_unicode_paths
     def setUp(self):
         self.BASETESTDIR = tempfile.mkdtemp()
         self.TESTDIR = join(self.BASETESTDIR, u"åäö")
         os.mkdir(self.TESTDIR)
         with open(join(self.TESTDIR, u"åäötestscript.py"), "w") as sfile:
             sfile.write("pass\n")
         self.oldpath = py3compat.getcwd()
         os.chdir(self.TESTDIR)
         self.fname = u"åäötestscript.py"
 
     def tearDown(self):
         os.chdir(self.oldpath)
         shutil.rmtree(self.BASETESTDIR)
 
     @onlyif_unicode_paths
     def test_1(self):
         """Test safe_execfile with non-ascii path
         """
         ip.safe_execfile(self.fname, {}, raise_exceptions=True)
 
 class ExitCodeChecks(tt.TempFileMixin):
     def test_exit_code_ok(self):
         self.system('exit 0')
         self.assertEqual(ip.user_ns['_exit_code'], 0)
 
     def test_exit_code_error(self):
         self.system('exit 1')
         self.assertEqual(ip.user_ns['_exit_code'], 1)
     
     @skipif(not hasattr(signal, 'SIGALRM'))
     def test_exit_code_signal(self):
         self.mktmp("import signal, time\n"
                    "signal.setitimer(signal.ITIMER_REAL, 0.1)\n"
                    "time.sleep(1)\n")
         self.system("%s %s" % (sys.executable, self.fname))
         self.assertEqual(ip.user_ns['_exit_code'], -signal.SIGALRM)
     
     @onlyif_cmds_exist("csh")
     def test_exit_code_signal_csh(self):
         SHELL = os.environ.get('SHELL', None)
         os.environ['SHELL'] = find_cmd("csh")
         try:
             self.test_exit_code_signal()
         finally:
             if SHELL is not None:
                 os.environ['SHELL'] = SHELL
             else:
                 del os.environ['SHELL']
 
 class TestSystemRaw(unittest.TestCase, ExitCodeChecks):
     system = ip.system_raw
 
     @onlyif_unicode_paths
     def test_1(self):
         """Test system_raw with non-ascii cmd
         """
         cmd = u'''python -c "'åäö'"   '''
         ip.system_raw(cmd)
 
     @mock.patch('subprocess.call', side_effect=KeyboardInterrupt)
     @mock.patch('os.system', side_effect=KeyboardInterrupt)
     def test_control_c(self, *mocks):
         try:
             self.system("sleep 1 # wont happen")
         except KeyboardInterrupt:
             self.fail("system call should intercept "
                       "keyboard interrupt from subprocess.call")
         self.assertEqual(ip.user_ns['_exit_code'], -signal.SIGINT)
 
 # TODO: Exit codes are currently ignored on Windows.
 class TestSystemPipedExitCode(unittest.TestCase, ExitCodeChecks):
     system = ip.system_piped
 
     @skip_win32
     def test_exit_code_ok(self):
         ExitCodeChecks.test_exit_code_ok(self)
 
     @skip_win32
     def test_exit_code_error(self):
         ExitCodeChecks.test_exit_code_error(self)
 
     @skip_win32
     def test_exit_code_signal(self):
         ExitCodeChecks.test_exit_code_signal(self)
 
 class TestModules(unittest.TestCase, tt.TempFileMixin):
     def test_extraneous_loads(self):
         """Test we're not loading modules on startup that we shouldn't.
         """
         self.mktmp("import sys\n"
                    "print('numpy' in sys.modules)\n"
                    "print('ipyparallel' in sys.modules)\n"
                    "print('ipykernel' in sys.modules)\n"
                    )
         out = "False\nFalse\nFalse\n"
         tt.ipexec_validate(self.fname, out)
 
 class Negator(ast.NodeTransformer):
     """Negates all number literals in an AST."""
     def visit_Num(self, node):
         node.n = -node.n
         return node
 
 class TestAstTransform(unittest.TestCase):
     def setUp(self):
         self.negator = Negator()
         ip.ast_transformers.append(self.negator)
     
     def tearDown(self):
         ip.ast_transformers.remove(self.negator)
     
     def test_run_cell(self):
         with tt.AssertPrints('-34'):
             ip.run_cell('print (12 + 22)')
         
         # A named reference to a number shouldn't be transformed.
         ip.user_ns['n'] = 55
         with tt.AssertNotPrints('-55'):
             ip.run_cell('print (n)')
     
     def test_timeit(self):
         called = set()
         def f(x):
             called.add(x)
         ip.push({'f':f})
         
         with tt.AssertPrints("average of "):
             ip.run_line_magic("timeit", "-n1 f(1)")
         self.assertEqual(called, {-1})
         called.clear()
 
         with tt.AssertPrints("average of "):
             ip.run_cell_magic("timeit", "-n1 f(2)", "f(3)")
         self.assertEqual(called, {-2, -3})
     
     def test_time(self):
         called = []
         def f(x):
             called.append(x)
         ip.push({'f':f})
         
         # Test with an expression
         with tt.AssertPrints("Wall time: "):
             ip.run_line_magic("time", "f(5+9)")
         self.assertEqual(called, [-14])
         called[:] = []
         
         # Test with a statement (different code path)
         with tt.AssertPrints("Wall time: "):
             ip.run_line_magic("time", "a = f(-3 + -2)")
         self.assertEqual(called, [5])
     
     def test_macro(self):
         ip.push({'a':10})
         # The AST transformation makes this do a+=-1
         ip.define_macro("amacro", "a+=1\nprint(a)")
         
         with tt.AssertPrints("9"):
             ip.run_cell("amacro")
         with tt.AssertPrints("8"):
             ip.run_cell("amacro")
 
 class IntegerWrapper(ast.NodeTransformer):
     """Wraps all integers in a call to Integer()"""
     def visit_Num(self, node):
         if isinstance(node.n, int):
             return ast.Call(func=ast.Name(id='Integer', ctx=ast.Load()),
                             args=[node], keywords=[])
         return node
 
 class TestAstTransform2(unittest.TestCase):
     def setUp(self):
         self.intwrapper = IntegerWrapper()
         ip.ast_transformers.append(self.intwrapper)
         
         self.calls = []
         def Integer(*args):
             self.calls.append(args)
             return args
         ip.push({"Integer": Integer})
     
     def tearDown(self):
         ip.ast_transformers.remove(self.intwrapper)
         del ip.user_ns['Integer']
     
     def test_run_cell(self):
         ip.run_cell("n = 2")
         self.assertEqual(self.calls, [(2,)])
         
         # This shouldn't throw an error
         ip.run_cell("o = 2.0")
         self.assertEqual(ip.user_ns['o'], 2.0)
     
     def test_timeit(self):
         called = set()
         def f(x):
             called.add(x)
         ip.push({'f':f})
 
         with tt.AssertPrints("average of "):
             ip.run_line_magic("timeit", "-n1 f(1)")
         self.assertEqual(called, {(1,)})
         called.clear()
 
         with tt.AssertPrints("average of "):
             ip.run_cell_magic("timeit", "-n1 f(2)", "f(3)")
         self.assertEqual(called, {(2,), (3,)})
 
 class ErrorTransformer(ast.NodeTransformer):
     """Throws an error when it sees a number."""
     def visit_Num(self, node):
         raise ValueError("test")
 
 class TestAstTransformError(unittest.TestCase):
     def test_unregistering(self):
         err_transformer = ErrorTransformer()
         ip.ast_transformers.append(err_transformer)
         
         with tt.AssertPrints("unregister", channel='stderr'):
             ip.run_cell("1 + 2")
         
         # This should have been removed.
         nt.assert_not_in(err_transformer, ip.ast_transformers)
 
 
 class StringRejector(ast.NodeTransformer):
     """Throws an InputRejected when it sees a string literal.
 
     Used to verify that NodeTransformers can signal that a piece of code should
     not be executed by throwing an InputRejected.
     """
 
     def visit_Str(self, node):
         raise InputRejected("test")
 
 
 class TestAstTransformInputRejection(unittest.TestCase):
 
     def setUp(self):
         self.transformer = StringRejector()
         ip.ast_transformers.append(self.transformer)
 
     def tearDown(self):
         ip.ast_transformers.remove(self.transformer)
 
     def test_input_rejection(self):
         """Check that NodeTransformers can reject input."""
 
         expect_exception_tb = tt.AssertPrints("InputRejected: test")
         expect_no_cell_output = tt.AssertNotPrints("'unsafe'", suppress=False)
 
         # Run the same check twice to verify that the transformer is not
         # disabled after raising.
         with expect_exception_tb, expect_no_cell_output:
             ip.run_cell("'unsafe'")
 
         with expect_exception_tb, expect_no_cell_output:
             res = ip.run_cell("'unsafe'")
 
         self.assertIsInstance(res.error_before_exec, InputRejected)
 
 def test__IPYTHON__():
     # This shouldn't raise a NameError, that's all
     __IPYTHON__
 
 
 class DummyRepr(object):
     def __repr__(self):
         return "DummyRepr"
     
     def _repr_html_(self):
         return "<b>dummy</b>"
     
     def _repr_javascript_(self):
         return "console.log('hi');", {'key': 'value'}
     
 
 def test_user_variables():
     # enable all formatters
     ip.display_formatter.active_types = ip.display_formatter.format_types
     
     ip.user_ns['dummy'] = d = DummyRepr()
     keys = {'dummy', 'doesnotexist'}
     r = ip.user_expressions({ key:key for key in keys})
 
     nt.assert_equal(keys, set(r.keys()))
     dummy = r['dummy']
     nt.assert_equal({'status', 'data', 'metadata'}, set(dummy.keys()))
     nt.assert_equal(dummy['status'], 'ok')
     data = dummy['data']
     metadata = dummy['metadata']
     nt.assert_equal(data.get('text/html'), d._repr_html_())
     js, jsmd = d._repr_javascript_()
     nt.assert_equal(data.get('application/javascript'), js)
     nt.assert_equal(metadata.get('application/javascript'), jsmd)
     
     dne = r['doesnotexist']
     nt.assert_equal(dne['status'], 'error')
     nt.assert_equal(dne['ename'], 'NameError')
     
     # back to text only
     ip.display_formatter.active_types = ['text/plain']
     
 def test_user_expression():
     # enable all formatters
     ip.display_formatter.active_types = ip.display_formatter.format_types
     query = {
         'a' : '1 + 2',
         'b' : '1/0',
     }
     r = ip.user_expressions(query)
     import pprint
     pprint.pprint(r)
     nt.assert_equal(set(r.keys()), set(query.keys()))
     a = r['a']
     nt.assert_equal({'status', 'data', 'metadata'}, set(a.keys()))
     nt.assert_equal(a['status'], 'ok')
     data = a['data']
     metadata = a['metadata']
     nt.assert_equal(data.get('text/plain'), '3')
     
     b = r['b']
     nt.assert_equal(b['status'], 'error')
     nt.assert_equal(b['ename'], 'ZeroDivisionError')
     
     # back to text only
     ip.display_formatter.active_types = ['text/plain']
     
 
 
 
 
 class TestSyntaxErrorTransformer(unittest.TestCase):
     """Check that SyntaxError raised by an input transformer is handled by run_cell()"""
 
     class SyntaxErrorTransformer(InputTransformer):
 
         def push(self, line):
             pos = line.find('syntaxerror')
             if pos >= 0:
                 e = SyntaxError('input contains "syntaxerror"')
                 e.text = line
                 e.offset = pos + 1
                 raise e
             return line
 
         def reset(self):
             pass
 
     def setUp(self):
         self.transformer = TestSyntaxErrorTransformer.SyntaxErrorTransformer()
         ip.input_splitter.python_line_transforms.append(self.transformer)
         ip.input_transformer_manager.python_line_transforms.append(self.transformer)
 
     def tearDown(self):
         ip.input_splitter.python_line_transforms.remove(self.transformer)
         ip.input_transformer_manager.python_line_transforms.remove(self.transformer)
 
     def test_syntaxerror_input_transformer(self):
         with tt.AssertPrints('1234'):
             ip.run_cell('1234')
         with tt.AssertPrints('SyntaxError: invalid syntax'):
             ip.run_cell('1 2 3')   # plain python syntax error
         with tt.AssertPrints('SyntaxError: input contains "syntaxerror"'):
             ip.run_cell('2345  # syntaxerror')  # input transformer syntax error
         with tt.AssertPrints('3456'):
             ip.run_cell('3456')
 
 
 
 def test_warning_suppression():
     ip.run_cell("import warnings")
     try:
         with tt.AssertPrints("UserWarning: asdf", channel="stderr"):
             ip.run_cell("warnings.warn('asdf')")
         # Here's the real test -- if we run that again, we should get the
         # warning again. Traditionally, each warning was only issued once per
         # IPython session (approximately), even if the user typed in new and
         # different code that should have also triggered the warning, leading
         # to much confusion.
         with tt.AssertPrints("UserWarning: asdf", channel="stderr"):
             ip.run_cell("warnings.warn('asdf')")
     finally:
         ip.run_cell("del warnings")
 
 
 def test_deprecation_warning():
     ip.run_cell("""
 import warnings
 def wrn():
     warnings.warn(
         "I AM  A WARNING",
         DeprecationWarning
     )
         """)
     try:
         with tt.AssertPrints("I AM  A WARNING", channel="stderr"):
             ip.run_cell("wrn()")
     finally:
         ip.run_cell("del warnings")
         ip.run_cell("del wrn")
 
 
 class TestImportNoDeprecate(tt.TempFileMixin):
 
     def setup(self):
         """Make a valid python temp file."""
         self.mktmp("""
 import warnings
 def wrn():
     warnings.warn(
         "I AM  A WARNING",
         DeprecationWarning
     )
 """)
 
     def test_no_dep(self):
         """
         No deprecation warning should be raised from imported functions
         """
         ip.run_cell("from {} import wrn".format(self.fname))
 
         with tt.AssertNotPrints("I AM  A WARNING"):
             ip.run_cell("wrn()")
         ip.run_cell("del wrn")

IPython/core/tests/test_magic.py

@@ -1,1011 +1,988 b''
 # -*- coding: utf-8 -*-
 """Tests for various magic functions.
 
 Needs to be run by nose (to make ipython session available).
 """
 from __future__ import absolute_import
 
 import io
 import os
 import sys
 import warnings
 from unittest import TestCase
 
 try:
     from importlib import invalidate_caches   # Required from Python 3.3
 except ImportError:
     def invalidate_caches():
         pass
 
 import nose.tools as nt
 
 from IPython import get_ipython
 from IPython.core import magic
 from IPython.core.error import UsageError
 from IPython.core.magic import (Magics, magics_class, line_magic,
                                 cell_magic,
                                 register_line_magic, register_cell_magic)
 from IPython.core.magics import execution, script, code
 from IPython.testing import decorators as dec
 from IPython.testing import tools as tt
 from IPython.utils import py3compat
 from IPython.utils.io import capture_output
 from IPython.utils.tempdir import TemporaryDirectory
 from IPython.utils.process import find_cmd
 
 if py3compat.PY3:
     from io import StringIO
 else:
     from StringIO import StringIO
 
 
 _ip = get_ipython()
 
 @magic.magics_class
 class DummyMagics(magic.Magics): pass
 
 def test_extract_code_ranges():
     instr = "1 3 5-6 7-9 10:15 17: :10 10- -13 :"
     expected = [(0, 1), 
                 (2, 3),
                 (4, 6),
                 (6, 9),
                 (9, 14),
                 (16, None),
                 (None, 9),
                 (9, None),
                 (None, 13),
                 (None, None)]
     actual = list(code.extract_code_ranges(instr))
     nt.assert_equal(actual, expected)
 
 def test_extract_symbols():
     source = """import foo\na = 10\ndef b():\n    return 42\n\n\nclass A: pass\n\n\n"""
     symbols_args = ["a", "b", "A", "A,b", "A,a", "z"]
     expected = [([], ['a']),
                 (["def b():\n    return 42\n"], []),
                 (["class A: pass\n"], []),
                 (["class A: pass\n", "def b():\n    return 42\n"], []),
                 (["class A: pass\n"], ['a']),
                 ([], ['z'])]
     for symbols, exp in zip(symbols_args, expected):
         nt.assert_equal(code.extract_symbols(source, symbols), exp)
 
 
 def test_extract_symbols_raises_exception_with_non_python_code():
     source = ("=begin A Ruby program :)=end\n"
               "def hello\n"
               "puts 'Hello world'\n"
               "end")
     with nt.assert_raises(SyntaxError):
         code.extract_symbols(source, "hello")
 
 def test_config():
     """ test that config magic does not raise
     can happen if Configurable init is moved too early into
     Magics.__init__ as then a Config object will be registerd as a
     magic.
     """
     ## should not raise.
     _ip.magic('config')
 
 def test_rehashx():
     # clear up everything
     _ip.alias_manager.clear_aliases()
     del _ip.db['syscmdlist']
     
     _ip.magic('rehashx')
     # Practically ALL ipython development systems will have more than 10 aliases
 
     nt.assert_true(len(_ip.alias_manager.aliases) > 10)
     for name, cmd in _ip.alias_manager.aliases:
         # we must strip dots from alias names
         nt.assert_not_in('.', name)
 
     # rehashx must fill up syscmdlist
     scoms = _ip.db['syscmdlist']
     nt.assert_true(len(scoms) > 10)
 
 
 def test_magic_parse_options():
     """Test that we don't mangle paths when parsing magic options."""
     ip = get_ipython()
     path = 'c:\\x'
     m = DummyMagics(ip)
     opts = m.parse_options('-f %s' % path,'f:')[0]
     # argv splitting is os-dependent
     if os.name == 'posix':
         expected = 'c:x'
     else:
         expected = path
     nt.assert_equal(opts['f'], expected)
 
 def test_magic_parse_long_options():
     """Magic.parse_options can handle --foo=bar long options"""
     ip = get_ipython()
     m = DummyMagics(ip)
     opts, _ = m.parse_options('--foo --bar=bubble', 'a', 'foo', 'bar=')
     nt.assert_in('foo', opts)
     nt.assert_in('bar', opts)
     nt.assert_equal(opts['bar'], "bubble")
 
 
 @dec.skip_without('sqlite3')
 def doctest_hist_f():
     """Test %hist -f with temporary filename.
 
     In [9]: import tempfile
 
     In [10]: tfile = tempfile.mktemp('.py','tmp-ipython-')
 
     In [11]: %hist -nl -f $tfile 3
 
     In [13]: import os; os.unlink(tfile)
     """
 
 
 @dec.skip_without('sqlite3')
 def doctest_hist_r():
     """Test %hist -r
 
     XXX - This test is not recording the output correctly.  For some reason, in
     testing mode the raw history isn't getting populated.  No idea why.
     Disabling the output checking for now, though at least we do run it.
 
     In [1]: 'hist' in _ip.lsmagic()
     Out[1]: True
 
     In [2]: x=1
 
     In [3]: %hist -rl 2
     x=1 # random
     %hist -r 2
     """
 
 
 @dec.skip_without('sqlite3')
 def doctest_hist_op():
     """Test %hist -op
 
     In [1]: class b(float):
        ...:     pass
        ...: 
 
     In [2]: class s(object):
        ...:     def __str__(self):
        ...:         return 's'
        ...: 
 
     In [3]: 
 
     In [4]: class r(b):
        ...:     def __repr__(self):
        ...:         return 'r'
        ...: 
 
     In [5]: class sr(s,r): pass
        ...: 
 
     In [6]: 
 
     In [7]: bb=b()
 
     In [8]: ss=s()
 
     In [9]: rr=r()
 
     In [10]: ssrr=sr()
 
     In [11]: 4.5
     Out[11]: 4.5
 
     In [12]: str(ss)
     Out[12]: 's'
 
     In [13]: 
 
     In [14]: %hist -op
     >>> class b:
     ...     pass
     ... 
     >>> class s(b):
     ...     def __str__(self):
     ...         return 's'
     ... 
     >>> 
     >>> class r(b):
     ...     def __repr__(self):
     ...         return 'r'
     ... 
     >>> class sr(s,r): pass
     >>> 
     >>> bb=b()
     >>> ss=s()
     >>> rr=r()
     >>> ssrr=sr()
     >>> 4.5
     4.5
     >>> str(ss)
     's'
     >>> 
     """
 
 def test_hist_pof():
     ip = get_ipython()
     ip.run_cell(u"1+2", store_history=True)
     #raise Exception(ip.history_manager.session_number)
     #raise Exception(list(ip.history_manager._get_range_session()))
     with TemporaryDirectory() as td:
         tf = os.path.join(td, 'hist.py')
         ip.run_line_magic('history', '-pof %s' % tf)
         assert os.path.isfile(tf)
 
 
 @dec.skip_without('sqlite3')
 def test_macro():
     ip = get_ipython()
     ip.history_manager.reset()   # Clear any existing history.
     cmds = ["a=1", "def b():\n  return a**2", "print(a,b())"]
     for i, cmd in enumerate(cmds, start=1):
         ip.history_manager.store_inputs(i, cmd)
     ip.magic("macro test 1-3")
     nt.assert_equal(ip.user_ns["test"].value, "\n".join(cmds)+"\n")
     
     # List macros
     nt.assert_in("test", ip.magic("macro"))
 
 
 @dec.skip_without('sqlite3')
 def test_macro_run():
     """Test that we can run a multi-line macro successfully."""
     ip = get_ipython()
     ip.history_manager.reset()
     cmds = ["a=10", "a+=1", py3compat.doctest_refactor_print("print a"),
                                                             "%macro test 2-3"]
     for cmd in cmds:
         ip.run_cell(cmd, store_history=True)
     nt.assert_equal(ip.user_ns["test"].value,
                             py3compat.doctest_refactor_print("a+=1\nprint a\n"))
     with tt.AssertPrints("12"):
         ip.run_cell("test")
     with tt.AssertPrints("13"):
         ip.run_cell("test")
 
 
 def test_magic_magic():
     """Test %magic"""
     ip = get_ipython()
     with capture_output() as captured:
         ip.magic("magic")
     
     stdout = captured.stdout
     nt.assert_in('%magic', stdout)
     nt.assert_in('IPython', stdout)
     nt.assert_in('Available', stdout)
 
 
 @dec.skipif_not_numpy
 def test_numpy_reset_array_undec():
     "Test '%reset array' functionality"
     _ip.ex('import numpy as np')
     _ip.ex('a = np.empty(2)')
     nt.assert_in('a', _ip.user_ns)
     _ip.magic('reset -f array')
     nt.assert_not_in('a', _ip.user_ns)
 
 def test_reset_out():
     "Test '%reset out' magic"
     _ip.run_cell("parrot = 'dead'", store_history=True)
     # test '%reset -f out', make an Out prompt
     _ip.run_cell("parrot", store_history=True)
     nt.assert_true('dead' in [_ip.user_ns[x] for x in ('_','__','___')])
     _ip.magic('reset -f out')
     nt.assert_false('dead' in [_ip.user_ns[x] for x in ('_','__','___')])
     nt.assert_equal(len(_ip.user_ns['Out']), 0)
 
 def test_reset_in():
     "Test '%reset in' magic"
     # test '%reset -f in'
     _ip.run_cell("parrot", store_history=True)
     nt.assert_true('parrot' in [_ip.user_ns[x] for x in ('_i','_ii','_iii')])
     _ip.magic('%reset -f in')
     nt.assert_false('parrot' in [_ip.user_ns[x] for x in ('_i','_ii','_iii')])
     nt.assert_equal(len(set(_ip.user_ns['In'])), 1)
 
 def test_reset_dhist():
     "Test '%reset dhist' magic"
     _ip.run_cell("tmp = [d for d in _dh]") # copy before clearing
     _ip.magic('cd ' + os.path.dirname(nt.__file__))
     _ip.magic('cd -')
     nt.assert_true(len(_ip.user_ns['_dh']) > 0)
     _ip.magic('reset -f dhist')
     nt.assert_equal(len(_ip.user_ns['_dh']), 0)
     _ip.run_cell("_dh = [d for d in tmp]") #restore
 
 def test_reset_in_length():
     "Test that '%reset in' preserves In[] length"
     _ip.run_cell("print 'foo'")
     _ip.run_cell("reset -f in")
     nt.assert_equal(len(_ip.user_ns['In']), _ip.displayhook.prompt_count+1)
 
 def test_tb_syntaxerror():
     """test %tb after a SyntaxError"""
     ip = get_ipython()
     ip.run_cell("for")
     
     # trap and validate stdout
     save_stdout = sys.stdout
     try:
         sys.stdout = StringIO()
         ip.run_cell("%tb")
         out = sys.stdout.getvalue()
     finally:
         sys.stdout = save_stdout
     # trim output, and only check the last line
     last_line = out.rstrip().splitlines()[-1].strip()
     nt.assert_equal(last_line, "SyntaxError: invalid syntax")
 
 
 def test_time():
     ip = get_ipython()
     
     with tt.AssertPrints("Wall time: "):
         ip.run_cell("%time None")
     
     ip.run_cell("def f(kmjy):\n"
                 "    %time print (2*kmjy)")
     
     with tt.AssertPrints("Wall time: "):
         with tt.AssertPrints("hihi", suppress=False):
             ip.run_cell("f('hi')")
 
 
 @dec.skip_win32
 def test_time2():
     ip = get_ipython()
     
     with tt.AssertPrints("CPU times: user "):
         ip.run_cell("%time None")
 
 def test_time3():
     """Erroneous magic function calls, issue gh-3334"""
     ip = get_ipython()
     ip.user_ns.pop('run', None)
     
     with tt.AssertNotPrints("not found", channel='stderr'):
         ip.run_cell("%%time\n"
                     "run = 0\n"
                     "run += 1")
 
[e-mail](sys.version_info[0] >= 3, "no differences with __future__ in py3")
-def test_time_futures():
-    "Test %time with __future__ environments"
-    ip = get_ipython()
-    ip.autocall = 0
-    ip.run_cell("from __future__ import division")
-    with tt.AssertPrints('0.25'):
-        ip.run_line_magic('time', 'print(1/4)')
-    ip.compile.reset_compiler_flags()
-    with tt.AssertNotPrints('0.25'):
-        ip.run_line_magic('time', 'print(1/4)')
-
 def test_doctest_mode():
     "Toggle doctest_mode twice, it should be a no-op and run without error"
     _ip.magic('doctest_mode')
     _ip.magic('doctest_mode')
 
 
 def test_parse_options():
     """Tests for basic options parsing in magics."""
     # These are only the most minimal of tests, more should be added later.  At
     # the very least we check that basic text/unicode calls work OK.
     m = DummyMagics(_ip)
     nt.assert_equal(m.parse_options('foo', '')[1], 'foo')
     nt.assert_equal(m.parse_options(u'foo', '')[1], u'foo')
 
     
 def test_dirops():
     """Test various directory handling operations."""
     # curpath = lambda :os.path.splitdrive(py3compat.getcwd())[1].replace('\\','/')
     curpath = py3compat.getcwd
     startdir = py3compat.getcwd()
     ipdir = os.path.realpath(_ip.ipython_dir)
     try:
         _ip.magic('cd "%s"' % ipdir)
         nt.assert_equal(curpath(), ipdir)
         _ip.magic('cd -')
         nt.assert_equal(curpath(), startdir)
         _ip.magic('pushd "%s"' % ipdir)
         nt.assert_equal(curpath(), ipdir)
         _ip.magic('popd')
         nt.assert_equal(curpath(), startdir)
     finally:
         os.chdir(startdir)
 
 
 def test_xmode():
     # Calling xmode three times should be a no-op
     xmode = _ip.InteractiveTB.mode
     for i in range(3):
         _ip.magic("xmode")
     nt.assert_equal(_ip.InteractiveTB.mode, xmode)
     
 def test_reset_hard():
     monitor = []
     class A(object):
         def __del__(self):
             monitor.append(1)
         def __repr__(self):
             return "<A instance>"
             
     _ip.user_ns["a"] = A()
     _ip.run_cell("a")
     
     nt.assert_equal(monitor, [])
     _ip.magic("reset -f")
     nt.assert_equal(monitor, [1])
     
 class TestXdel(tt.TempFileMixin):
     def test_xdel(self):
         """Test that references from %run are cleared by xdel."""
         src = ("class A(object):\n"
                "    monitor = []\n"
                "    def __del__(self):\n"
                "        self.monitor.append(1)\n"
                "a = A()\n")
         self.mktmp(src)
         # %run creates some hidden references...
         _ip.magic("run %s" % self.fname)
         # ... as does the displayhook.
         _ip.run_cell("a")
         
         monitor = _ip.user_ns["A"].monitor
         nt.assert_equal(monitor, [])
         
         _ip.magic("xdel a")
         
         # Check that a's __del__ method has been called.
         nt.assert_equal(monitor, [1])
 
 def doctest_who():
     """doctest for %who
     
     In [1]: %reset -f
     
     In [2]: alpha = 123
     
     In [3]: beta = 'beta'
     
     In [4]: %who int
     alpha
     
     In [5]: %who str
     beta
     
     In [6]: %whos
     Variable   Type    Data/Info
     ----------------------------
     alpha      int     123
     beta       str     beta
     
     In [7]: %who_ls
     Out[7]: ['alpha', 'beta']
     """
 
 def test_whos():
     """Check that whos is protected against objects where repr() fails."""
     class A(object):
         def __repr__(self):
             raise Exception()
     _ip.user_ns['a'] = A()
     _ip.magic("whos")
 
 @py3compat.u_format
 def doctest_precision():
     """doctest for %precision
     
     In [1]: f = get_ipython().display_formatter.formatters['text/plain']
     
     In [2]: %precision 5
     Out[2]: {u}'%.5f'
     
     In [3]: f.float_format
     Out[3]: {u}'%.5f'
     
     In [4]: %precision %e
     Out[4]: {u}'%e'
     
     In [5]: f(3.1415927)
     Out[5]: {u}'3.141593e+00'
     """
 
 def test_psearch():
     with tt.AssertPrints("dict.fromkeys"):
         _ip.run_cell("dict.fr*?")
 
 def test_timeit_shlex():
     """test shlex issues with timeit (#1109)"""
     _ip.ex("def f(*a,**kw): pass")
     _ip.magic('timeit -n1 "this is a bug".count(" ")')
     _ip.magic('timeit -r1 -n1 f(" ", 1)')
     _ip.magic('timeit -r1 -n1 f(" ", 1, " ", 2, " ")')
     _ip.magic('timeit -r1 -n1 ("a " + "b")')
     _ip.magic('timeit -r1 -n1 f("a " + "b")')
     _ip.magic('timeit -r1 -n1 f("a " + "b ")')
 
 
 def test_timeit_arguments():
     "Test valid timeit arguments, should not cause SyntaxError (GH #1269)"
     _ip.magic("timeit ('#')")
 
 
 def test_timeit_special_syntax():
     "Test %%timeit with IPython special syntax"
     @register_line_magic
     def lmagic(line):
         ip = get_ipython()
         ip.user_ns['lmagic_out'] = line
 
     # line mode test
     _ip.run_line_magic('timeit', '-n1 -r1 %lmagic my line')
     nt.assert_equal(_ip.user_ns['lmagic_out'], 'my line')
     # cell mode test
     _ip.run_cell_magic('timeit', '-n1 -r1', '%lmagic my line2')
     nt.assert_equal(_ip.user_ns['lmagic_out'], 'my line2')
 
 def test_timeit_return():
     """
     test wether timeit -o return object
     """
 
     res = _ip.run_line_magic('timeit','-n10 -r10 -o 1')
     assert(res is not None)
 
 def test_timeit_quiet():
     """
     test quiet option of timeit magic
     """
     with tt.AssertNotPrints("loops"):
         _ip.run_cell("%timeit -n1 -r1 -q 1")
 
 def test_timeit_return_quiet():
     with tt.AssertNotPrints("loops"):
         res = _ip.run_line_magic('timeit', '-n1 -r1 -q -o 1')
     assert (res is not None)
 
[e-mail](sys.version_info[0] >= 3, "no differences with __future__ in py3")
-def test_timeit_futures():
-    "Test %timeit with __future__ environments"
-    ip = get_ipython()
-    ip.run_cell("from __future__ import division")
-    with tt.AssertPrints('0.25'):
-        ip.run_line_magic('timeit', '-n1 -r1 print(1/4)')
-    ip.compile.reset_compiler_flags()
-    with tt.AssertNotPrints('0.25'):
-        ip.run_line_magic('timeit', '-n1 -r1 print(1/4)')
-
 @dec.skipif(execution.profile is None)
 def test_prun_special_syntax():
     "Test %%prun with IPython special syntax"
     @register_line_magic
     def lmagic(line):
         ip = get_ipython()
         ip.user_ns['lmagic_out'] = line
 
     # line mode test
     _ip.run_line_magic('prun', '-q %lmagic my line')
     nt.assert_equal(_ip.user_ns['lmagic_out'], 'my line')
     # cell mode test
     _ip.run_cell_magic('prun', '-q', '%lmagic my line2')
     nt.assert_equal(_ip.user_ns['lmagic_out'], 'my line2')
 
 @dec.skipif(execution.profile is None)
 def test_prun_quotes():
     "Test that prun does not clobber string escapes (GH #1302)"
     _ip.magic(r"prun -q x = '\t'")
     nt.assert_equal(_ip.user_ns['x'], '\t')
 
 def test_extension():
     # Debugging information for failures of this test
     print('sys.path:')
     for p in sys.path:
         print(' ', p)
     print('CWD', os.getcwd())
 
     nt.assert_raises(ImportError, _ip.magic, "load_ext daft_extension")
     daft_path = os.path.join(os.path.dirname(__file__), "daft_extension")
     sys.path.insert(0, daft_path)
     try:
         _ip.user_ns.pop('arq', None)
         invalidate_caches()   # Clear import caches
         _ip.magic("load_ext daft_extension")
         nt.assert_equal(_ip.user_ns['arq'], 185)
         _ip.magic("unload_ext daft_extension")
         assert 'arq' not in _ip.user_ns
     finally:
         sys.path.remove(daft_path)
 
 
 def test_notebook_export_json():
     _ip = get_ipython()
     _ip.history_manager.reset()   # Clear any existing history.
     cmds = [u"a=1", u"def b():\n  return a**2", u"print('noël, été', b())"]
     for i, cmd in enumerate(cmds, start=1):
         _ip.history_manager.store_inputs(i, cmd)
     with TemporaryDirectory() as td:
         outfile = os.path.join(td, "nb.ipynb")
         _ip.magic("notebook -e %s" % outfile)
 
 
 class TestEnv(TestCase):
 
     def test_env(self):
         env = _ip.magic("env")
         self.assertTrue(isinstance(env, dict))
 
     def test_env_get_set_simple(self):
         env = _ip.magic("env var val1")
         self.assertEqual(env, None)
         self.assertEqual(os.environ['var'], 'val1')
         self.assertEqual(_ip.magic("env var"), 'val1')
         env = _ip.magic("env var=val2")
         self.assertEqual(env, None)
         self.assertEqual(os.environ['var'], 'val2')
 
     def test_env_get_set_complex(self):
         env = _ip.magic("env var 'val1 '' 'val2")
         self.assertEqual(env, None)
         self.assertEqual(os.environ['var'], "'val1 '' 'val2")
         self.assertEqual(_ip.magic("env var"), "'val1 '' 'val2")
         env = _ip.magic('env var=val2 val3="val4')
         self.assertEqual(env, None)
         self.assertEqual(os.environ['var'], 'val2 val3="val4')
 
     def test_env_set_bad_input(self):
         self.assertRaises(UsageError, lambda: _ip.magic("set_env var"))
 
     def test_env_set_whitespace(self):
         self.assertRaises(UsageError, lambda: _ip.magic("env var A=B"))
 
 
 class CellMagicTestCase(TestCase):
 
     def check_ident(self, magic):
         # Manually called, we get the result
         out = _ip.run_cell_magic(magic, 'a', 'b')
         nt.assert_equal(out, ('a','b'))
         # Via run_cell, it goes into the user's namespace via displayhook
         _ip.run_cell('%%' + magic +' c\nd')
         nt.assert_equal(_ip.user_ns['_'], ('c','d'))
 
     def test_cell_magic_func_deco(self):
         "Cell magic using simple decorator"
         @register_cell_magic
         def cellm(line, cell):
             return line, cell
 
         self.check_ident('cellm')
 
     def test_cell_magic_reg(self):
         "Cell magic manually registered"
         def cellm(line, cell):
             return line, cell
 
         _ip.register_magic_function(cellm, 'cell', 'cellm2')
         self.check_ident('cellm2')
 
     def test_cell_magic_class(self):
         "Cell magics declared via a class"
         @magics_class
         class MyMagics(Magics):
 
             @cell_magic
             def cellm3(self, line, cell):
                 return line, cell
 
         _ip.register_magics(MyMagics)
         self.check_ident('cellm3')
 
     def test_cell_magic_class2(self):
         "Cell magics declared via a class, #2"
         @magics_class
         class MyMagics2(Magics):
 
             @cell_magic('cellm4')
             def cellm33(self, line, cell):
                 return line, cell
             
         _ip.register_magics(MyMagics2)
         self.check_ident('cellm4')
         # Check that nothing is registered as 'cellm33'
         c33 = _ip.find_cell_magic('cellm33')
         nt.assert_equal(c33, None)
 
 def test_file():
     """Basic %%file"""
     ip = get_ipython()
     with TemporaryDirectory() as td:
         fname = os.path.join(td, 'file1')
         ip.run_cell_magic("file", fname, u'\n'.join([
             'line1',
             'line2',
         ]))
         with open(fname) as f:
             s = f.read()
         nt.assert_in('line1\n', s)
         nt.assert_in('line2', s)
 
 def test_file_var_expand():
     """%%file $filename"""
     ip = get_ipython()
     with TemporaryDirectory() as td:
         fname = os.path.join(td, 'file1')
         ip.user_ns['filename'] = fname
         ip.run_cell_magic("file", '$filename', u'\n'.join([
             'line1',
             'line2',
         ]))
         with open(fname) as f:
             s = f.read()
         nt.assert_in('line1\n', s)
         nt.assert_in('line2', s)
 
 def test_file_unicode():
     """%%file with unicode cell"""
     ip = get_ipython()
     with TemporaryDirectory() as td:
         fname = os.path.join(td, 'file1')
         ip.run_cell_magic("file", fname, u'\n'.join([
             u'liné1',
             u'liné2',
         ]))
         with io.open(fname, encoding='utf-8') as f:
             s = f.read()
         nt.assert_in(u'liné1\n', s)
         nt.assert_in(u'liné2', s)
 
 def test_file_amend():
     """%%file -a amends files"""
     ip = get_ipython()
     with TemporaryDirectory() as td:
         fname = os.path.join(td, 'file2')
         ip.run_cell_magic("file", fname, u'\n'.join([
             'line1',
             'line2',
         ]))
         ip.run_cell_magic("file", "-a %s" % fname, u'\n'.join([
             'line3',
             'line4',
         ]))
         with open(fname) as f:
             s = f.read()
         nt.assert_in('line1\n', s)
         nt.assert_in('line3\n', s)
         
     
 def test_script_config():
     ip = get_ipython()
     ip.config.ScriptMagics.script_magics = ['whoda']
     sm = script.ScriptMagics(shell=ip)
     nt.assert_in('whoda', sm.magics['cell'])
 
 @dec.skip_win32
 def test_script_out():
     ip = get_ipython()
     ip.run_cell_magic("script", "--out output sh", "echo 'hi'")
     nt.assert_equal(ip.user_ns['output'], 'hi\n')
 
 @dec.skip_win32
 def test_script_err():
     ip = get_ipython()
     ip.run_cell_magic("script", "--err error sh", "echo 'hello' >&2")
     nt.assert_equal(ip.user_ns['error'], 'hello\n')
 
 @dec.skip_win32
 def test_script_out_err():
     ip = get_ipython()
     ip.run_cell_magic("script", "--out output --err error sh", "echo 'hi'\necho 'hello' >&2")
     nt.assert_equal(ip.user_ns['output'], 'hi\n')
     nt.assert_equal(ip.user_ns['error'], 'hello\n')
 
 @dec.skip_win32
 def test_script_bg_out():
     ip = get_ipython()
     ip.run_cell_magic("script", "--bg --out output sh", "echo 'hi'")
     nt.assert_equal(ip.user_ns['output'].read(), b'hi\n')
 
 @dec.skip_win32
 def test_script_bg_err():
     ip = get_ipython()
     ip.run_cell_magic("script", "--bg --err error sh", "echo 'hello' >&2")
     nt.assert_equal(ip.user_ns['error'].read(), b'hello\n')
 
 @dec.skip_win32
 def test_script_bg_out_err():
     ip = get_ipython()
     ip.run_cell_magic("script", "--bg --out output --err error sh", "echo 'hi'\necho 'hello' >&2")
     nt.assert_equal(ip.user_ns['output'].read(), b'hi\n')
     nt.assert_equal(ip.user_ns['error'].read(), b'hello\n')
 
 def test_script_defaults():
     ip = get_ipython()
     for cmd in ['sh', 'bash', 'perl', 'ruby']:
         try:
             find_cmd(cmd)
         except Exception:
             pass
         else:
             nt.assert_in(cmd, ip.magics_manager.magics['cell'])
 
 
 @magics_class
 class FooFoo(Magics):
     """class with both %foo and %%foo magics"""
     @line_magic('foo')
     def line_foo(self, line):
         "I am line foo"
         pass
 
     @cell_magic("foo")
     def cell_foo(self, line, cell):
         "I am cell foo, not line foo"
         pass
 
 def test_line_cell_info():
     """%%foo and %foo magics are distinguishable to inspect"""
     ip = get_ipython()
     ip.magics_manager.register(FooFoo)
     oinfo = ip.object_inspect('foo')
     nt.assert_true(oinfo['found'])
     nt.assert_true(oinfo['ismagic'])
     
     oinfo = ip.object_inspect('%%foo')
     nt.assert_true(oinfo['found'])
     nt.assert_true(oinfo['ismagic'])
     nt.assert_equal(oinfo['docstring'], FooFoo.cell_foo.__doc__)
 
     oinfo = ip.object_inspect('%foo')
     nt.assert_true(oinfo['found'])
     nt.assert_true(oinfo['ismagic'])
     nt.assert_equal(oinfo['docstring'], FooFoo.line_foo.__doc__)
 
 def test_multiple_magics():
     ip = get_ipython()
     foo1 = FooFoo(ip)
     foo2 = FooFoo(ip)
     mm = ip.magics_manager
     mm.register(foo1)
     nt.assert_true(mm.magics['line']['foo'].__self__ is foo1)
     mm.register(foo2)
     nt.assert_true(mm.magics['line']['foo'].__self__ is foo2)
 
 def test_alias_magic():
     """Test %alias_magic."""
     ip = get_ipython()
     mm = ip.magics_manager
 
     # Basic operation: both cell and line magics are created, if possible.
     ip.run_line_magic('alias_magic', 'timeit_alias timeit')
     nt.assert_in('timeit_alias', mm.magics['line'])
     nt.assert_in('timeit_alias', mm.magics['cell'])
 
     # --cell is specified, line magic not created.
     ip.run_line_magic('alias_magic', '--cell timeit_cell_alias timeit')
     nt.assert_not_in('timeit_cell_alias', mm.magics['line'])
     nt.assert_in('timeit_cell_alias', mm.magics['cell'])
 
     # Test that line alias is created successfully.
     ip.run_line_magic('alias_magic', '--line env_alias env')
     nt.assert_equal(ip.run_line_magic('env', ''),
                     ip.run_line_magic('env_alias', ''))
 
 def test_save():
     """Test %save."""
     ip = get_ipython()
     ip.history_manager.reset()   # Clear any existing history.
     cmds = [u"a=1", u"def b():\n  return a**2", u"print(a, b())"]
     for i, cmd in enumerate(cmds, start=1):
         ip.history_manager.store_inputs(i, cmd)
     with TemporaryDirectory() as tmpdir:
         file = os.path.join(tmpdir, "testsave.py")
         ip.run_line_magic("save", "%s 1-10" % file)
         with open(file) as f:
             content = f.read()
             nt.assert_equal(content.count(cmds[0]), 1)
             nt.assert_in('coding: utf-8', content)
         ip.run_line_magic("save", "-a %s 1-10" % file)
         with open(file) as f:
             content = f.read()
             nt.assert_equal(content.count(cmds[0]), 2)
             nt.assert_in('coding: utf-8', content)
 
 
 def test_store():
     """Test %store."""
     ip = get_ipython()
     ip.run_line_magic('load_ext', 'storemagic')
     
     # make sure the storage is empty
     ip.run_line_magic('store', '-z')
     ip.user_ns['var'] = 42
     ip.run_line_magic('store', 'var')
     ip.user_ns['var'] = 39
     ip.run_line_magic('store', '-r')
     nt.assert_equal(ip.user_ns['var'], 42)
 
     ip.run_line_magic('store', '-d var')
     ip.user_ns['var'] = 39
     ip.run_line_magic('store' , '-r')
     nt.assert_equal(ip.user_ns['var'], 39)
 
 
 def _run_edit_test(arg_s, exp_filename=None,
                         exp_lineno=-1,
                         exp_contents=None,
                         exp_is_temp=None):
     ip = get_ipython()
     M = code.CodeMagics(ip)
     last_call = ['','']
     opts,args = M.parse_options(arg_s,'prxn:')
     filename, lineno, is_temp = M._find_edit_target(ip, args, opts, last_call)
         
     if exp_filename is not None:
         nt.assert_equal(exp_filename, filename)
     if exp_contents is not None:
         with io.open(filename, 'r', encoding='utf-8') as f:
             contents = f.read()
         nt.assert_equal(exp_contents, contents)
     if exp_lineno != -1:
         nt.assert_equal(exp_lineno, lineno)
     if exp_is_temp is not None:
         nt.assert_equal(exp_is_temp, is_temp)
 
 
 def test_edit_interactive():
     """%edit on interactively defined objects"""
     ip = get_ipython()
     n = ip.execution_count
     ip.run_cell(u"def foo(): return 1", store_history=True)
     
     try:
         _run_edit_test("foo")
     except code.InteractivelyDefined as e:
         nt.assert_equal(e.index, n)
     else:
         raise AssertionError("Should have raised InteractivelyDefined")
 
 
 def test_edit_cell():
     """%edit [cell id]"""
     ip = get_ipython()
     
     ip.run_cell(u"def foo(): return 1", store_history=True)
     
     # test
     _run_edit_test("1", exp_contents=ip.user_ns['In'][1], exp_is_temp=True)
 
 def test_bookmark():
     ip = get_ipython()
     ip.run_line_magic('bookmark', 'bmname')
     with tt.AssertPrints('bmname'):
         ip.run_line_magic('bookmark', '-l')
     ip.run_line_magic('bookmark', '-d bmname')
 
 def test_ls_magic():
     ip = get_ipython()
     json_formatter = ip.display_formatter.formatters['application/json']
     json_formatter.enabled = True
     lsmagic = ip.magic('lsmagic')
     with warnings.catch_warnings(record=True) as w:
         j = json_formatter(lsmagic)
     nt.assert_equal(sorted(j), ['cell', 'line'])
     nt.assert_equal(w, []) # no warnings
 
 def test_strip_initial_indent():
     def sii(s):
         lines = s.splitlines()
         return '\n'.join(code.strip_initial_indent(lines))
 
     nt.assert_equal(sii("  a = 1\nb = 2"), "a = 1\nb = 2")
     nt.assert_equal(sii("  a\n    b\nc"), "a\n  b\nc")
     nt.assert_equal(sii("a\n  b"), "a\n  b")

IPython/core/tests/test_magic_terminal.py

@@ -1,210 +1,203 b''
 """Tests for various magic functions specific to the terminal frontend.
 
 Needs to be run by nose (to make ipython session available).
 """
 from __future__ import absolute_import
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 import sys
 from unittest import TestCase
 
 import nose.tools as nt
 
 from IPython.testing import tools as tt
 from IPython.utils.py3compat import PY3
 
 if PY3:
     from io import StringIO
 else:
     from StringIO import StringIO
 
 #-----------------------------------------------------------------------------
 # Globals
 #-----------------------------------------------------------------------------
 ip = get_ipython()
 
 #-----------------------------------------------------------------------------
 # Test functions begin
 #-----------------------------------------------------------------------------
 
 def check_cpaste(code, should_fail=False):
     """Execute code via 'cpaste' and ensure it was executed, unless
     should_fail is set.
     """
     ip.user_ns['code_ran'] = False
 
     src = StringIO()
     if not hasattr(src, 'encoding'):
         # IPython expects stdin to have an encoding attribute
         src.encoding = None
     src.write(code)
     src.write('\n--\n')
     src.seek(0)
 
     stdin_save = sys.stdin
     sys.stdin = src
 
     try:
         context = tt.AssertPrints if should_fail else tt.AssertNotPrints
         with context("Traceback (most recent call last)"):
                 ip.magic('cpaste')
 
         if not should_fail:
             assert ip.user_ns['code_ran'], "%r failed" % code
     finally:
         sys.stdin = stdin_save
 
-PY31 = sys.version_info[:2] == (3,1)
-
 def test_cpaste():
     """Test cpaste magic"""
 
     def runf():
         """Marker function: sets a flag when executed.
         """
         ip.user_ns['code_ran'] = True
         return 'runf' # return string so '+ runf()' doesn't result in success
 
     tests = {'pass': ["runf()",
                       "In [1]: runf()",
                       "In [1]: if 1:\n   ...:     runf()",
                       "> > > runf()",
                       ">>> runf()",
                       "   >>> runf()",
                       ],
 
              'fail': ["1 + runf()",
+                      "++ runf()",
              ]}
-    
-    # I don't know why this is failing specifically on Python 3.1. I've
-    # checked it manually interactively, but we don't care enough about 3.1
-    # to spend time fiddling with the tests, so we just skip it.
-    if not PY31:
-        tests['fail'].append("++ runf()")
 
     ip.user_ns['runf'] = runf
 
     for code in tests['pass']:
         check_cpaste(code)
 
     for code in tests['fail']:
         check_cpaste(code, should_fail=True)
 
 
 class PasteTestCase(TestCase):
     """Multiple tests for clipboard pasting"""
 
     def paste(self, txt, flags='-q'):
         """Paste input text, by default in quiet mode"""
         ip.hooks.clipboard_get = lambda : txt
         ip.magic('paste '+flags)
 
     def setUp(self):
         # Inject fake clipboard hook but save original so we can restore it later
         self.original_clip = ip.hooks.clipboard_get
 
     def tearDown(self): 
         # Restore original hook
         ip.hooks.clipboard_get = self.original_clip
        
     def test_paste(self):
         ip.user_ns.pop('x', None)
         self.paste('x = 1')
         nt.assert_equal(ip.user_ns['x'], 1)
         ip.user_ns.pop('x')
 
     def test_paste_pyprompt(self):
         ip.user_ns.pop('x', None)
         self.paste('>>> x=2')
         nt.assert_equal(ip.user_ns['x'], 2)
         ip.user_ns.pop('x')
 
     def test_paste_py_multi(self):
         self.paste("""
         >>> x = [1,2,3]
         >>> y = []
         >>> for i in x:
         ...     y.append(i**2)
         ... 
         """)
         nt.assert_equal(ip.user_ns['x'], [1,2,3])
         nt.assert_equal(ip.user_ns['y'], [1,4,9])
 
     def test_paste_py_multi_r(self):
         "Now, test that self.paste -r works"
         self.test_paste_py_multi()
         nt.assert_equal(ip.user_ns.pop('x'), [1,2,3])
         nt.assert_equal(ip.user_ns.pop('y'), [1,4,9])
         nt.assert_false('x' in ip.user_ns)
         ip.magic('paste -r')
         nt.assert_equal(ip.user_ns['x'], [1,2,3])
         nt.assert_equal(ip.user_ns['y'], [1,4,9])
 
     def test_paste_email(self):
         "Test pasting of email-quoted contents"
         self.paste("""\
         >> def foo(x):
         >>     return x + 1
         >> xx = foo(1.1)""")
         nt.assert_equal(ip.user_ns['xx'], 2.1)
 
     def test_paste_email2(self):
         "Email again; some programs add a space also at each quoting level"
         self.paste("""\
         > > def foo(x):
         > >     return x + 1
         > > yy = foo(2.1)     """)
         nt.assert_equal(ip.user_ns['yy'], 3.1)
 
     def test_paste_email_py(self):
         "Email quoting of interactive input"
         self.paste("""\
         >> >>> def f(x):
         >> ...   return x+1
         >> ... 
         >> >>> zz = f(2.5)      """)
         nt.assert_equal(ip.user_ns['zz'], 3.5)
 
     def test_paste_echo(self):
         "Also test self.paste echoing, by temporarily faking the writer"
         w = StringIO()
         writer = ip.write
         ip.write = w.write
         code = """
         a = 100
         b = 200"""
         try:
             self.paste(code,'')
             out = w.getvalue()
         finally:
             ip.write = writer
         nt.assert_equal(ip.user_ns['a'], 100)
         nt.assert_equal(ip.user_ns['b'], 200)
         nt.assert_equal(out, code+"\n## -- End pasted text --\n")
 
     def test_paste_leading_commas(self):
         "Test multiline strings with leading commas"
         tm = ip.magics_manager.registry['TerminalMagics']
         s = '''\
 a = """
 ,1,2,3
 """'''
         ip.user_ns.pop('foo', None)
         tm.store_or_execute(s, 'foo')
         nt.assert_in('foo', ip.user_ns)
 
 
     def test_paste_trailing_question(self):
         "Test pasting sources with trailing question marks"
         tm = ip.magics_manager.registry['TerminalMagics']
         s = '''\
 def funcfoo():
    if True: #am i true?
        return 'fooresult'
 '''
         ip.user_ns.pop('funcfoo', None)
         self.paste(s)
         nt.assert_equal(ip.user_ns['funcfoo'](), 'fooresult')

IPython/core/tests/test_oinspect.py

@@ -1,456 +1,455 b''
 """Tests for the object inspection functionality.
 """
 
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
 from __future__ import print_function
 
 import os
 import re
 import sys
 
 import nose.tools as nt
 
 from .. import oinspect
 from IPython.core.magic import (Magics, magics_class, line_magic,
                                 cell_magic, line_cell_magic,
                                 register_line_magic, register_cell_magic,
                                 register_line_cell_magic)
 from decorator import decorator
 from IPython.testing.decorators import skipif
 from IPython.testing.tools import AssertPrints
 from IPython.utils.path import compress_user
 from IPython.utils import py3compat
 from IPython.utils.signatures import Signature, Parameter
 
 
 #-----------------------------------------------------------------------------
 # Globals and constants
 #-----------------------------------------------------------------------------
 
 inspector = oinspect.Inspector()
 ip = get_ipython()
 
 #-----------------------------------------------------------------------------
 # Local utilities
 #-----------------------------------------------------------------------------
 
 # WARNING: since this test checks the line number where a function is
 # defined, if any code is inserted above, the following line will need to be
 # updated.  Do NOT insert any whitespace between the next line and the function
 # definition below.
 THIS_LINE_NUMBER = 43  # Put here the actual number of this line
 
 from unittest import TestCase
 
 class Test(TestCase):
 
     def test_find_source_lines(self):
         self.assertEqual(oinspect.find_source_lines(Test.test_find_source_lines), 
                     THIS_LINE_NUMBER+6)
 
 
 # A couple of utilities to ensure these tests work the same from a source or a
 # binary install
 def pyfile(fname):
     return os.path.normcase(re.sub('.py[co]$', '.py', fname))
 
 
 def match_pyfiles(f1, f2):
     nt.assert_equal(pyfile(f1), pyfile(f2))
 
 
 def test_find_file():
     match_pyfiles(oinspect.find_file(test_find_file), os.path.abspath(__file__))
 
 
 def test_find_file_decorated1():
 
     @decorator
     def noop1(f):
         def wrapper():
             return f(*a, **kw)
         return wrapper
 
     @noop1
     def f(x):
         "My docstring"
     
     match_pyfiles(oinspect.find_file(f), os.path.abspath(__file__))
     nt.assert_equal(f.__doc__, "My docstring")
 
 
 def test_find_file_decorated2():
 
     @decorator
     def noop2(f, *a, **kw):
         return f(*a, **kw)
 
     @noop2
     @noop2
     @noop2
     def f(x):
         "My docstring 2"
     
     match_pyfiles(oinspect.find_file(f), os.path.abspath(__file__))
     nt.assert_equal(f.__doc__, "My docstring 2")
     
 
 def test_find_file_magic():
     run = ip.find_line_magic('run')
     nt.assert_not_equal(oinspect.find_file(run), None)
 
 
 # A few generic objects we can then inspect in the tests below
 
 class Call(object):
     """This is the class docstring."""
 
     def __init__(self, x, y=1):
         """This is the constructor docstring."""
 
     def __call__(self, *a, **kw):
         """This is the call docstring."""
 
     def method(self, x, z=2):
         """Some method's docstring"""
 
 class HasSignature(object):
     """This is the class docstring."""
     __signature__ = Signature([Parameter('test', Parameter.POSITIONAL_OR_KEYWORD)])
 
     def __init__(self, *args):
         """This is the init docstring"""
 
 
 class SimpleClass(object):
     def method(self, x, z=2):
         """Some method's docstring"""
 
 
 class OldStyle:
     """An old-style class for testing."""
     pass
 
 
 def f(x, y=2, *a, **kw):
     """A simple function."""
 
 
 def g(y, z=3, *a, **kw):
     pass  # no docstring
 
 
 @register_line_magic
 def lmagic(line):
     "A line magic"
 
 
 @register_cell_magic
 def cmagic(line, cell):
     "A cell magic"
 
 
 @register_line_cell_magic
 def lcmagic(line, cell=None):
     "A line/cell magic"
 
 
 @magics_class
 class SimpleMagics(Magics):
     @line_magic
     def Clmagic(self, cline):
         "A class-based line magic"
         
     @cell_magic
     def Ccmagic(self, cline, ccell):
         "A class-based cell magic"
         
     @line_cell_magic
     def Clcmagic(self, cline, ccell=None):
         "A class-based line/cell magic"
 
 
 class Awkward(object):
     def __getattr__(self, name):
         raise Exception(name)
 
 class NoBoolCall:
     """
     callable with `__bool__` raising should still be inspect-able.
     """
 
     def __call__(self):
         """does nothing"""
         pass
 
     def __bool__(self):
         """just raise NotImplemented"""
         raise NotImplementedError('Must be implemented')
 
 
 class SerialLiar(object):
     """Attribute accesses always get another copy of the same class.
 
     unittest.mock.call does something similar, but it's not ideal for testing
     as the failure mode is to eat all your RAM. This gives up after 10k levels.
     """
     def __init__(self, max_fibbing_twig, lies_told=0):
         if lies_told > 10000:
             raise RuntimeError('Nose too long, honesty is the best policy')
         self.max_fibbing_twig = max_fibbing_twig
         self.lies_told = lies_told
         max_fibbing_twig[0] = max(max_fibbing_twig[0], lies_told)
 
     def __getattr__(self, item):
         return SerialLiar(self.max_fibbing_twig, self.lies_told + 1)
 
 
 def check_calltip(obj, name, call, docstring):
     """Generic check pattern all calltip tests will use"""
     info = inspector.info(obj, name)
     call_line, ds = oinspect.call_tip(info)
     nt.assert_equal(call_line, call)
     nt.assert_equal(ds, docstring)
 
 #-----------------------------------------------------------------------------
 # Tests
 #-----------------------------------------------------------------------------
 
 def test_calltip_class():
     check_calltip(Call, 'Call', 'Call(x, y=1)', Call.__init__.__doc__)
 
 
 def test_calltip_instance():
     c = Call(1)
     check_calltip(c, 'c', 'c(*a, **kw)', c.__call__.__doc__)
 
 
 def test_calltip_method():
     c = Call(1)
     check_calltip(c.method, 'c.method', 'c.method(x, z=2)', c.method.__doc__)
 
 
 def test_calltip_function():
     check_calltip(f, 'f', 'f(x, y=2, *a, **kw)', f.__doc__)
 
 
 def test_calltip_function2():
     check_calltip(g, 'g', 'g(y, z=3, *a, **kw)', '<no docstring>')
 
 
 @skipif(sys.version_info >= (3, 5))
 def test_calltip_builtin():
     check_calltip(sum, 'sum', None, sum.__doc__)
 
 
 def test_calltip_line_magic():
     check_calltip(lmagic, 'lmagic', 'lmagic(line)', "A line magic")
 
         
 def test_calltip_cell_magic():
     check_calltip(cmagic, 'cmagic', 'cmagic(line, cell)', "A cell magic")
 
         
 def test_calltip_line_cell_magic():
     check_calltip(lcmagic, 'lcmagic', 'lcmagic(line, cell=None)', 
                   "A line/cell magic")
         
 
 def test_class_magics():
     cm = SimpleMagics(ip)
     ip.register_magics(cm)
     check_calltip(cm.Clmagic, 'Clmagic', 'Clmagic(cline)',
                   "A class-based line magic")
     check_calltip(cm.Ccmagic, 'Ccmagic', 'Ccmagic(cline, ccell)',
                   "A class-based cell magic")
     check_calltip(cm.Clcmagic, 'Clcmagic', 'Clcmagic(cline, ccell=None)',
                   "A class-based line/cell magic")
     
 
 def test_info():
     "Check that Inspector.info fills out various fields as expected."
     i = inspector.info(Call, oname='Call')
     nt.assert_equal(i['type_name'], 'type')
     expted_class = str(type(type))  # <class 'type'> (Python 3) or <type 'type'>
     nt.assert_equal(i['base_class'], expted_class)
-    if sys.version_info > (3,):
-        nt.assert_regex(i['string_form'], "<class 'IPython.core.tests.test_oinspect.Call'( at 0x[0-9a-f]{1,9})?>")
+    nt.assert_regex(i['string_form'], "<class 'IPython.core.tests.test_oinspect.Call'( at 0x[0-9a-f]{1,9})?>")
     fname = __file__
     if fname.endswith(".pyc"):
         fname = fname[:-1]
     # case-insensitive comparison needed on some filesystems
     # e.g. Windows:
     nt.assert_equal(i['file'].lower(), compress_user(fname).lower())
     nt.assert_equal(i['definition'], None)
     nt.assert_equal(i['docstring'], Call.__doc__)
     nt.assert_equal(i['source'], None)
     nt.assert_true(i['isclass'])
     _self_py2 = '' if py3compat.PY3 else 'self, '
     nt.assert_equal(i['init_definition'], "Call(%sx, y=1)" % _self_py2)
     nt.assert_equal(i['init_docstring'], Call.__init__.__doc__)
 
     i = inspector.info(Call, detail_level=1)
     nt.assert_not_equal(i['source'], None)
     nt.assert_equal(i['docstring'], None)
 
     c = Call(1)
     c.__doc__ = "Modified instance docstring"
     i = inspector.info(c)
     nt.assert_equal(i['type_name'], 'Call')
     nt.assert_equal(i['docstring'], "Modified instance docstring")
     nt.assert_equal(i['class_docstring'], Call.__doc__)
     nt.assert_equal(i['init_docstring'], Call.__init__.__doc__)
     nt.assert_equal(i['call_docstring'], Call.__call__.__doc__)
 
     # Test old-style classes, which for example may not have an __init__ method.
     if not py3compat.PY3:
         i = inspector.info(OldStyle)
         nt.assert_equal(i['type_name'], 'classobj')
 
         i = inspector.info(OldStyle())
         nt.assert_equal(i['type_name'], 'instance')
         nt.assert_equal(i['docstring'], OldStyle.__doc__)
 
 def test_class_signature():
     info = inspector.info(HasSignature, 'HasSignature')
     nt.assert_equal(info['init_definition'], "HasSignature(test)")
     nt.assert_equal(info['init_docstring'], HasSignature.__init__.__doc__)
 
 def test_info_awkward():
     # Just test that this doesn't throw an error.
     inspector.info(Awkward())
 
 def test_bool_raise():
     inspector.info(NoBoolCall())
 
 def test_info_serialliar():
     fib_tracker = [0]
     inspector.info(SerialLiar(fib_tracker))
 
     # Nested attribute access should be cut off at 100 levels deep to avoid
     # infinite loops: https://github.com/ipython/ipython/issues/9122
     nt.assert_less(fib_tracker[0], 9000)
 
 def test_calldef_none():
     # We should ignore __call__ for all of these.
     for obj in [f, SimpleClass().method, any, str.upper]:
         print(obj)
         i = inspector.info(obj)
         nt.assert_is(i['call_def'], None)
 
 def f_kwarg(pos, *, kwonly):
     pass
 
 def test_definition_kwonlyargs():
     i = inspector.info(f_kwarg, oname='f_kwarg')  # analysis:ignore
     nt.assert_equal(i['definition'], "f_kwarg(pos, *, kwonly)")
 
 def test_getdoc():
     class A(object):
         """standard docstring"""
         pass
 
     class B(object):
         """standard docstring"""
         def getdoc(self):
             return "custom docstring"
     
     class C(object):
         """standard docstring"""
         def getdoc(self):
             return None
     
     a = A()
     b = B()
     c = C()
     
     nt.assert_equal(oinspect.getdoc(a), "standard docstring")
     nt.assert_equal(oinspect.getdoc(b), "custom docstring")
     nt.assert_equal(oinspect.getdoc(c), "standard docstring")
 
 
 def test_empty_property_has_no_source():
     i = inspector.info(property(), detail_level=1)
     nt.assert_is(i['source'], None)
 
 
 def test_property_sources():
     import zlib
 
     class A(object):
         @property
         def foo(self):
             return 'bar'
 
         foo = foo.setter(lambda self, v: setattr(self, 'bar', v))
 
         id = property(id)
         compress = property(zlib.compress)
 
     i = inspector.info(A.foo, detail_level=1)
     nt.assert_in('def foo(self):', i['source'])
     nt.assert_in('lambda self, v:', i['source'])
 
     i = inspector.info(A.id, detail_level=1)
     nt.assert_in('fget = <function id>', i['source'])
 
     i = inspector.info(A.compress, detail_level=1)
     nt.assert_in('fget = <function zlib.compress>', i['source'])
 
 
 def test_property_docstring_is_in_info_for_detail_level_0():
     class A(object):
         @property
         def foobar(self):
             """This is `foobar` property."""
             pass
 
     ip.user_ns['a_obj'] = A()
     nt.assert_equals(
         'This is `foobar` property.',
         ip.object_inspect('a_obj.foobar', detail_level=0)['docstring'])
 
     ip.user_ns['a_cls'] = A
     nt.assert_equals(
         'This is `foobar` property.',
         ip.object_inspect('a_cls.foobar', detail_level=0)['docstring'])
 
 
 def test_pdef():
     # See gh-1914
     def foo(): pass
     inspector.pdef(foo, 'foo')
 
 
 def test_pinfo_nonascii():
     # See gh-1177
     from . import nonascii2
     ip.user_ns['nonascii2'] = nonascii2
     ip._inspect('pinfo', 'nonascii2', detail_level=1)
 
 
 def test_pinfo_magic():
     with AssertPrints('Docstring:'):
         ip._inspect('pinfo', 'lsmagic', detail_level=0)
 
     with AssertPrints('Source:'):
         ip._inspect('pinfo', 'lsmagic', detail_level=1)
 
 
 def test_init_colors():
     # ensure colors are not present in signature info
     info = inspector.info(HasSignature)
     init_def = info['init_definition']
     nt.assert_not_in('[0m', init_def)
 
 
 def test_builtin_init():
     info = inspector.info(list)
     init_def = info['init_definition']
     # Python < 3.4 can't get init definition from builtins,
     # but still exercise the inspection in case of error-raising bugs.
     if sys.version_info >= (3,4):
         nt.assert_is_not_none(init_def)
 

IPython/core/tests/test_ultratb.py

@@ -1,356 +1,355 b''
 # encoding: utf-8
 """Tests for IPython.core.ultratb
 """
 import io
 import sys
 import os.path
 from textwrap import dedent
 import traceback
 import unittest
 
 try:
     from unittest import mock
 except ImportError:
     import mock    # Python 2
 
 from ..ultratb import ColorTB, VerboseTB, find_recursion
 
 
 from IPython.testing import tools as tt
 from IPython.testing.decorators import onlyif_unicode_paths
 from IPython.utils.syspathcontext import prepended_to_syspath
 from IPython.utils.tempdir import TemporaryDirectory
 from IPython.utils.py3compat import PY3
 
 ip = get_ipython()
 
 file_1 = """1
 2
 3
 def f():
   1/0
 """
 
 file_2 = """def f():
   1/0
 """
 
 class ChangedPyFileTest(unittest.TestCase):
     def test_changing_py_file(self):
         """Traceback produced if the line where the error occurred is missing?
         
         https://github.com/ipython/ipython/issues/1456
         """
         with TemporaryDirectory() as td:
             fname = os.path.join(td, "foo.py")
             with open(fname, "w") as f:
                 f.write(file_1)
             
             with prepended_to_syspath(td):
                 ip.run_cell("import foo")
             
             with tt.AssertPrints("ZeroDivisionError"):
                 ip.run_cell("foo.f()")
             
             # Make the file shorter, so the line of the error is missing.
             with open(fname, "w") as f:
                 f.write(file_2)
             
             # For some reason, this was failing on the *second* call after
             # changing the file, so we call f() twice.
             with tt.AssertNotPrints("Internal Python error", channel='stderr'):
                 with tt.AssertPrints("ZeroDivisionError"):
                     ip.run_cell("foo.f()")
                 with tt.AssertPrints("ZeroDivisionError"):
                     ip.run_cell("foo.f()")
 
 iso_8859_5_file = u'''# coding: iso-8859-5
 
 def fail():
     """дбИЖ"""
     1/0     # дбИЖ
 '''
 
 class NonAsciiTest(unittest.TestCase):
     @onlyif_unicode_paths
     def test_nonascii_path(self):
         # Non-ascii directory name as well.
         with TemporaryDirectory(suffix=u'é') as td:
             fname = os.path.join(td, u"fooé.py")
             with open(fname, "w") as f:
                 f.write(file_1)
             
             with prepended_to_syspath(td):
                 ip.run_cell("import foo")
             
             with tt.AssertPrints("ZeroDivisionError"):
                 ip.run_cell("foo.f()")
     
     def test_iso8859_5(self):
         with TemporaryDirectory() as td:
             fname = os.path.join(td, 'dfghjkl.py')
 
             with io.open(fname, 'w', encoding='iso-8859-5') as f:
                 f.write(iso_8859_5_file)
             
             with prepended_to_syspath(td):
                 ip.run_cell("from dfghjkl import fail")
             
             with tt.AssertPrints("ZeroDivisionError"):
                 with tt.AssertPrints(u'дбИЖ', suppress=False):
                     ip.run_cell('fail()')
     
     def test_nonascii_msg(self):
         cell = u"raise Exception('é')"
         expected = u"Exception('é')"
         ip.run_cell("%xmode plain")
         with tt.AssertPrints(expected):
             ip.run_cell(cell)
 
         ip.run_cell("%xmode verbose")
         with tt.AssertPrints(expected):
             ip.run_cell(cell)
 
         ip.run_cell("%xmode context")
         with tt.AssertPrints(expected):
             ip.run_cell(cell)
 
 
 class NestedGenExprTestCase(unittest.TestCase):
     """
     Regression test for the following issues:
     https://github.com/ipython/ipython/issues/8293
     https://github.com/ipython/ipython/issues/8205
     """
     def test_nested_genexpr(self):
         code = dedent(
             """\
             class SpecificException(Exception):
                 pass
 
             def foo(x):
                 raise SpecificException("Success!")
 
             sum(sum(foo(x) for _ in [0]) for x in [0])
             """
         )
         with tt.AssertPrints('SpecificException: Success!', suppress=False):
             ip.run_cell(code)
 
 
 indentationerror_file = """if True:
 zoon()
 """
 
 class IndentationErrorTest(unittest.TestCase):
     def test_indentationerror_shows_line(self):
         # See issue gh-2398
         with tt.AssertPrints("IndentationError"):
             with tt.AssertPrints("zoon()", suppress=False):
                 ip.run_cell(indentationerror_file)
         
         with TemporaryDirectory() as td:
             fname = os.path.join(td, "foo.py")
             with open(fname, "w") as f:
                 f.write(indentationerror_file)
             
             with tt.AssertPrints("IndentationError"):
                 with tt.AssertPrints("zoon()", suppress=False):
                     ip.magic('run %s' % fname)
 
 se_file_1 = """1
 2
 7/
 """
 
 se_file_2 = """7/
 """
 
 class SyntaxErrorTest(unittest.TestCase):
     def test_syntaxerror_without_lineno(self):
         with tt.AssertNotPrints("TypeError"):
             with tt.AssertPrints("line unknown"):
                 ip.run_cell("raise SyntaxError()")
 
     def test_changing_py_file(self):
         with TemporaryDirectory() as td:
             fname = os.path.join(td, "foo.py")
             with open(fname, 'w') as f:
                 f.write(se_file_1)
 
             with tt.AssertPrints(["7/", "SyntaxError"]):
                 ip.magic("run " + fname)
 
             # Modify the file
             with open(fname, 'w') as f:
                 f.write(se_file_2)
 
             # The SyntaxError should point to the correct line
             with tt.AssertPrints(["7/", "SyntaxError"]):
                 ip.magic("run " + fname)
 
     def test_non_syntaxerror(self):
         # SyntaxTB may be called with an error other than a SyntaxError
         # See e.g. gh-4361
         try:
             raise ValueError('QWERTY')
         except ValueError:
             with tt.AssertPrints('QWERTY'):
                 ip.showsyntaxerror()
 
 
 class Python3ChainedExceptionsTest(unittest.TestCase):
     DIRECT_CAUSE_ERROR_CODE = """
 try:
     x = 1 + 2
     print(not_defined_here)
 except Exception as e:
     x += 55
     x - 1
     y = {}
     raise KeyError('uh') from e
     """
 
     EXCEPTION_DURING_HANDLING_CODE = """
 try:
     x = 1 + 2
     print(not_defined_here)
 except Exception as e:
     x += 55
     x - 1
     y = {}
     raise KeyError('uh')
     """
 
     SUPPRESS_CHAINING_CODE = """
 try:
     1/0
 except Exception:
     raise ValueError("Yikes") from None
     """
 
     def test_direct_cause_error(self):
         if PY3:
             with tt.AssertPrints(["KeyError", "NameError", "direct cause"]):
                 ip.run_cell(self.DIRECT_CAUSE_ERROR_CODE)
 
     def test_exception_during_handling_error(self):
         if PY3:
             with tt.AssertPrints(["KeyError", "NameError", "During handling"]):
                 ip.run_cell(self.EXCEPTION_DURING_HANDLING_CODE)
 
     def test_suppress_exception_chaining(self):
         if PY3:
             with tt.AssertNotPrints("ZeroDivisionError"), \
                     tt.AssertPrints("ValueError", suppress=False):
                 ip.run_cell(self.SUPPRESS_CHAINING_CODE)
 
 
 class RecursionTest(unittest.TestCase):
     DEFINITIONS = """
 def non_recurs():
     1/0
 
 def r1():
     r1()
 
 def r3a():
     r3b()
 
 def r3b():
     r3c()
 
 def r3c():
     r3a()
 
 def r3o1():
     r3a()
 
 def r3o2():
     r3o1()
 """
     def setUp(self):
         ip.run_cell(self.DEFINITIONS)
 
     def test_no_recursion(self):
         with tt.AssertNotPrints("frames repeated"):
             ip.run_cell("non_recurs()")
 
     def test_recursion_one_frame(self):
         with tt.AssertPrints("1 frames repeated"):
             ip.run_cell("r1()")
 
     def test_recursion_three_frames(self):
         with tt.AssertPrints("3 frames repeated"):
             ip.run_cell("r3o2()")
 
     def test_find_recursion(self):
         captured = []
         def capture_exc(*args, **kwargs):
             captured.append(sys.exc_info())
         with mock.patch.object(ip, 'showtraceback', capture_exc):
             ip.run_cell("r3o2()")
 
         self.assertEqual(len(captured), 1)
         etype, evalue, tb = captured[0]
         self.assertIn("recursion", str(evalue))
 
         records = ip.InteractiveTB.get_records(tb, 3, ip.InteractiveTB.tb_offset)
         for r in records[:10]:
             print(r[1:4])
 
         # The outermost frames should be:
         # 0: the 'cell' that was running when the exception came up
         # 1: r3o2()
         # 2: r3o1()
         # 3: r3a()
         # Then repeating r3b, r3c, r3a
         last_unique, repeat_length = find_recursion(etype, evalue, records)
         self.assertEqual(last_unique, 2)
         self.assertEqual(repeat_length, 3)
 
 
 #----------------------------------------------------------------------------
 
 # module testing (minimal)
-if sys.version_info > (3,):
-    def test_handlers():
-        def spam(c, d_e):
-            (d, e) = d_e
-            x = c + d
-            y = c * d
-            foo(x, y)
-
-        def foo(a, b, bar=1):
-            eggs(a, b + bar)
-
-        def eggs(f, g, z=globals()):
-            h = f + g
-            i = f - g
-            return h / i
-        
-        buff = io.StringIO()
-
-        buff.write('')
-        buff.write('*** Before ***')
-        try:
-            buff.write(spam(1, (2, 3)))
-        except:
-            traceback.print_exc(file=buff)
-
-        handler = ColorTB(ostream=buff)
-        buff.write('*** ColorTB ***')
-        try:
-            buff.write(spam(1, (2, 3)))
-        except:
-            handler(*sys.exc_info())
-        buff.write('')
-
-        handler = VerboseTB(ostream=buff)
-        buff.write('*** VerboseTB ***')
-        try:
-            buff.write(spam(1, (2, 3)))
-        except:
-            handler(*sys.exc_info())
-        buff.write('')
+def test_handlers():
+    def spam(c, d_e):
+        (d, e) = d_e
+        x = c + d
+        y = c * d
+        foo(x, y)
+
+    def foo(a, b, bar=1):
+        eggs(a, b + bar)
+
+    def eggs(f, g, z=globals()):
+        h = f + g
+        i = f - g
+        return h / i
+
+    buff = io.StringIO()
+
+    buff.write('')
+    buff.write('*** Before ***')
+    try:
+        buff.write(spam(1, (2, 3)))
+    except:
+        traceback.print_exc(file=buff)
+
+    handler = ColorTB(ostream=buff)
+    buff.write('*** ColorTB ***')
+    try:
+        buff.write(spam(1, (2, 3)))
+    except:
+        handler(*sys.exc_info())
+    buff.write('')
+
+    handler = VerboseTB(ostream=buff)
+    buff.write('*** VerboseTB ***')
+    try:
+        buff.write(spam(1, (2, 3)))
+    except:
+        handler(*sys.exc_info())
+    buff.write('')
 

IPython/testing/iptest.py

@@ -1,433 +1,432 b''
 # -*- coding: utf-8 -*-
 """IPython Test Suite Runner.
 
 This module provides a main entry point to a user script to test IPython
 itself from the command line. There are two ways of running this script:
 
 1. With the syntax `iptest all`.  This runs our entire test suite by
    calling this script (with different arguments) recursively.  This
    causes modules and package to be tested in different processes, using nose
    or trial where appropriate.
 2. With the regular nose syntax, like `iptest -vvs IPython`.  In this form
    the script simply calls nose, but with special command line flags and
    plugins loaded.
 
 """
 
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
 from __future__ import print_function
 
 import glob
 from io import BytesIO
 import os
 import os.path as path
 import sys
 from threading import Thread, Lock, Event
 import warnings
 
 import nose.plugins.builtin
 from nose.plugins.xunit import Xunit
 from nose import SkipTest
 from nose.core import TestProgram
 from nose.plugins import Plugin
 from nose.util import safe_str
 
 from IPython import version_info
 from IPython.utils.py3compat import bytes_to_str
 from IPython.utils.importstring import import_item
 from IPython.testing.plugin.ipdoctest import IPythonDoctest
 from IPython.external.decorators import KnownFailure, knownfailureif
 
 pjoin = path.join
 
 
 # Enable printing all warnings raise by IPython's modules
 warnings.filterwarnings('ignore', message='.*Matplotlib is building the font cache.*', category=UserWarning, module='.*')
-if sys.version_info > (3,0):
-    warnings.filterwarnings('error', message='.*', category=ResourceWarning, module='.*')
+warnings.filterwarnings('error', message='.*', category=ResourceWarning, module='.*')
 warnings.filterwarnings('error', message=".*{'config': True}.*", category=DeprecationWarning, module='IPy.*')
 warnings.filterwarnings('default', message='.*', category=Warning, module='IPy.*')
 
 if version_info < (6,):
     # nose.tools renames all things from `camelCase` to `snake_case` which raise an
     # warning with the runner they also import from standard import library. (as of Dec 2015)
     # Ignore, let's revisit that in a couple of years for IPython 6.
     warnings.filterwarnings('ignore', message='.*Please use assertEqual instead', category=Warning, module='IPython.*')
 
 
 # ------------------------------------------------------------------------------
 # Monkeypatch Xunit to count known failures as skipped.
 # ------------------------------------------------------------------------------
 def monkeypatch_xunit():
     try:
         knownfailureif(True)(lambda: None)()
     except Exception as e:
         KnownFailureTest = type(e)
 
     def addError(self, test, err, capt=None):
         if issubclass(err[0], KnownFailureTest):
             err = (SkipTest,) + err[1:]
         return self.orig_addError(test, err, capt)
 
     Xunit.orig_addError = Xunit.addError
     Xunit.addError = addError
 
 #-----------------------------------------------------------------------------
 # Check which dependencies are installed and greater than minimum version.
 #-----------------------------------------------------------------------------
 def extract_version(mod):
     return mod.__version__
 
 def test_for(item, min_version=None, callback=extract_version):
     """Test to see if item is importable, and optionally check against a minimum
     version.
 
     If min_version is given, the default behavior is to check against the
     `__version__` attribute of the item, but specifying `callback` allows you to
     extract the value you are interested in. e.g::
 
         In [1]: import sys
 
         In [2]: from IPython.testing.iptest import test_for
 
         In [3]: test_for('sys', (2,6), callback=lambda sys: sys.version_info)
         Out[3]: True
 
     """
     try:
         check = import_item(item)
     except (ImportError, RuntimeError):
         # GTK reports Runtime error if it can't be initialized even if it's
         # importable.
         return False
     else:
         if min_version:
             if callback:
                 # extra processing step to get version to compare
                 check = callback(check)
 
             return check >= min_version
         else:
             return True
 
 # Global dict where we can store information on what we have and what we don't
 # have available at test run time
 have = {'matplotlib': test_for('matplotlib'),
         'pygments': test_for('pygments'),
         'sqlite3': test_for('sqlite3')}
 
 #-----------------------------------------------------------------------------
 # Test suite definitions
 #-----------------------------------------------------------------------------
 
 test_group_names = ['core',
                     'extensions', 'lib', 'terminal', 'testing', 'utils',
                    ]
 
 class TestSection(object):
     def __init__(self, name, includes):
         self.name = name
         self.includes = includes
         self.excludes = []
         self.dependencies = []
         self.enabled = True
     
     def exclude(self, module):
         if not module.startswith('IPython'):
             module = self.includes[0] + "." + module
         self.excludes.append(module.replace('.', os.sep))
     
     def requires(self, *packages):
         self.dependencies.extend(packages)
     
     @property
     def will_run(self):
         return self.enabled and all(have[p] for p in self.dependencies)
 
 # Name -> (include, exclude, dependencies_met)
 test_sections = {n:TestSection(n, ['IPython.%s' % n]) for n in test_group_names}
 
 
 # Exclusions and dependencies
 # ---------------------------
 
 # core:
 sec = test_sections['core']
 if not have['sqlite3']:
     sec.exclude('tests.test_history')
     sec.exclude('history')
 if not have['matplotlib']:
     sec.exclude('pylabtools'),
     sec.exclude('tests.test_pylabtools')
 
 # lib:
 sec = test_sections['lib']
 sec.exclude('kernel')
 if not have['pygments']:
     sec.exclude('tests.test_lexers')
 # We do this unconditionally, so that the test suite doesn't import
 # gtk, changing the default encoding and masking some unicode bugs.
 sec.exclude('inputhookgtk')
 # We also do this unconditionally, because wx can interfere with Unix signals.
 # There are currently no tests for it anyway.
 sec.exclude('inputhookwx')
 # Testing inputhook will need a lot of thought, to figure out
 # how to have tests that don't lock up with the gui event
 # loops in the picture
 sec.exclude('inputhook')
 
 # testing:
 sec = test_sections['testing']
 # These have to be skipped on win32 because they use echo, rm, cd, etc.
 # See ticket https://github.com/ipython/ipython/issues/87
 if sys.platform == 'win32':
     sec.exclude('plugin.test_exampleip')
     sec.exclude('plugin.dtexample')
 
 # don't run jupyter_console tests found via shim
 test_sections['terminal'].exclude('console')
 
 # extensions:
 sec = test_sections['extensions']
 # This is deprecated in favour of rpy2
 sec.exclude('rmagic')
 # autoreload does some strange stuff, so move it to its own test section
 sec.exclude('autoreload')
 sec.exclude('tests.test_autoreload')
 test_sections['autoreload'] = TestSection('autoreload',
         ['IPython.extensions.autoreload', 'IPython.extensions.tests.test_autoreload'])
 test_group_names.append('autoreload')
 
 
 #-----------------------------------------------------------------------------
 # Functions and classes
 #-----------------------------------------------------------------------------
 
 def check_exclusions_exist():
     from IPython.paths import get_ipython_package_dir
     from warnings import warn
     parent = os.path.dirname(get_ipython_package_dir())
     for sec in test_sections:
         for pattern in sec.exclusions:
             fullpath = pjoin(parent, pattern)
             if not os.path.exists(fullpath) and not glob.glob(fullpath + '.*'):
                 warn("Excluding nonexistent file: %r" % pattern)
 
 
 class ExclusionPlugin(Plugin):
     """A nose plugin to effect our exclusions of files and directories.
     """
     name = 'exclusions'
     score = 3000  # Should come before any other plugins
     
     def __init__(self, exclude_patterns=None):
         """
         Parameters
         ----------
 
         exclude_patterns : sequence of strings, optional
           Filenames containing these patterns (as raw strings, not as regular
           expressions) are excluded from the tests.
         """
         self.exclude_patterns = exclude_patterns or []
         super(ExclusionPlugin, self).__init__()
 
     def options(self, parser, env=os.environ):
         Plugin.options(self, parser, env)
     
     def configure(self, options, config):
         Plugin.configure(self, options, config)
         # Override nose trying to disable plugin.
         self.enabled = True
         
     def wantFile(self, filename):
         """Return whether the given filename should be scanned for tests.
         """
         if any(pat in filename for pat in self.exclude_patterns):
             return False
         return None
 
     def wantDirectory(self, directory):
         """Return whether the given directory should be scanned for tests.
         """
         if any(pat in directory for pat in self.exclude_patterns):
             return False
         return None
 
 
 class StreamCapturer(Thread):
     daemon = True  # Don't hang if main thread crashes
     started = False
     def __init__(self, echo=False):
         super(StreamCapturer, self).__init__()
         self.echo = echo
         self.streams = []
         self.buffer = BytesIO()
         self.readfd, self.writefd = os.pipe()
         self.buffer_lock = Lock()
         self.stop = Event()
 
     def run(self):
         self.started = True
 
         while not self.stop.is_set():
             chunk = os.read(self.readfd, 1024)
 
             with self.buffer_lock:
                 self.buffer.write(chunk)
             if self.echo:
                 sys.stdout.write(bytes_to_str(chunk))
     
         os.close(self.readfd)
         os.close(self.writefd)
 
     def reset_buffer(self):
         with self.buffer_lock:
             self.buffer.truncate(0)
             self.buffer.seek(0)
 
     def get_buffer(self):
         with self.buffer_lock:
             return self.buffer.getvalue()
 
     def ensure_started(self):
         if not self.started:
             self.start()
 
     def halt(self):
         """Safely stop the thread."""
         if not self.started:
             return
 
         self.stop.set()
         os.write(self.writefd, b'\0')  # Ensure we're not locked in a read()
         self.join()
 
 class SubprocessStreamCapturePlugin(Plugin):
     name='subprocstreams'
     def __init__(self):
         Plugin.__init__(self)
         self.stream_capturer = StreamCapturer()
         self.destination = os.environ.get('IPTEST_SUBPROC_STREAMS', 'capture')
         # This is ugly, but distant parts of the test machinery need to be able
         # to redirect streams, so we make the object globally accessible.
         nose.iptest_stdstreams_fileno = self.get_write_fileno
 
     def get_write_fileno(self):
         if self.destination == 'capture':
             self.stream_capturer.ensure_started()
             return self.stream_capturer.writefd
         elif self.destination == 'discard':
             return os.open(os.devnull, os.O_WRONLY)
         else:
             return sys.__stdout__.fileno()
     
     def configure(self, options, config):
         Plugin.configure(self, options, config)
         # Override nose trying to disable plugin.
         if self.destination == 'capture':
             self.enabled = True
     
     def startTest(self, test):
         # Reset log capture
         self.stream_capturer.reset_buffer()
     
     def formatFailure(self, test, err):
         # Show output
         ec, ev, tb = err
         captured = self.stream_capturer.get_buffer().decode('utf-8', 'replace')
         if captured.strip():
             ev = safe_str(ev)
             out = [ev, '>> begin captured subprocess output <<',
                     captured,
                     '>> end captured subprocess output <<']
             return ec, '\n'.join(out), tb
 
         return err
     
     formatError = formatFailure
     
     def finalize(self, result):
         self.stream_capturer.halt()
 
 
 def run_iptest():
     """Run the IPython test suite using nose.
 
     This function is called when this script is **not** called with the form
     `iptest all`.  It simply calls nose with appropriate command line flags
     and accepts all of the standard nose arguments.
     """
     # Apply our monkeypatch to Xunit
     if '--with-xunit' in sys.argv and not hasattr(Xunit, 'orig_addError'):
         monkeypatch_xunit()
 
     arg1 = sys.argv[1]
     if arg1 in test_sections:
         section = test_sections[arg1]
         sys.argv[1:2] = section.includes
     elif arg1.startswith('IPython.') and arg1[8:] in test_sections:
         section = test_sections[arg1[8:]]
         sys.argv[1:2] = section.includes
     else:
         section = TestSection(arg1, includes=[arg1])
         
 
     argv = sys.argv + [ '--detailed-errors',  # extra info in tracebacks
                         # We add --exe because of setuptools' imbecility (it
                         # blindly does chmod +x on ALL files).  Nose does the
                         # right thing and it tries to avoid executables,
                         # setuptools unfortunately forces our hand here.  This
                         # has been discussed on the distutils list and the
                         # setuptools devs refuse to fix this problem!
                         '--exe',
                         ]
     if '-a' not in argv and '-A' not in argv:
         argv = argv + ['-a', '!crash']
 
     if nose.__version__ >= '0.11':
         # I don't fully understand why we need this one, but depending on what
         # directory the test suite is run from, if we don't give it, 0 tests
         # get run.  Specifically, if the test suite is run from the source dir
         # with an argument (like 'iptest.py IPython.core', 0 tests are run,
         # even if the same call done in this directory works fine).  It appears
         # that if the requested package is in the current dir, nose bails early
         # by default.  Since it's otherwise harmless, leave it in by default
         # for nose >= 0.11, though unfortunately nose 0.10 doesn't support it.
         argv.append('--traverse-namespace')
 
     plugins = [ ExclusionPlugin(section.excludes), KnownFailure(),
                SubprocessStreamCapturePlugin() ]
     
     # we still have some vestigial doctests in core
     if (section.name.startswith(('core', 'IPython.core'))):
         plugins.append(IPythonDoctest())
         argv.extend([
             '--with-ipdoctest',
             '--ipdoctest-tests',
             '--ipdoctest-extension=txt',
         ])
 
     
     # Use working directory set by parent process (see iptestcontroller)
     if 'IPTEST_WORKING_DIR' in os.environ:
         os.chdir(os.environ['IPTEST_WORKING_DIR'])
     
     # We need a global ipython running in this process, but the special
     # in-process group spawns its own IPython kernels, so for *that* group we
     # must avoid also opening the global one (otherwise there's a conflict of
     # singletons).  Ultimately the solution to this problem is to refactor our
     # assumptions about what needs to be a singleton and what doesn't (app
     # objects should, individual shells shouldn't).  But for now, this
     # workaround allows the test suite for the inprocess module to complete.
     if 'kernel.inprocess' not in section.name:
         from IPython.testing import globalipapp
         globalipapp.start_ipython()
 
     # Now nose can run
     TestProgram(argv=argv, addplugins=plugins)
 
 if __name__ == '__main__':
     run_iptest()

IPython/testing/skipdoctest.py

@@ -1,43 +1,30 b''
-"""Decorators marks that a doctest should be skipped, for both python 2 and 3.
+"""Decorators marks that a doctest should be skipped.
 
 The IPython.testing.decorators module triggers various extra imports, including
 numpy and sympy if they're present. Since this decorator is used in core parts
 of IPython, it's in a separate module so that running IPython doesn't trigger
 those imports."""
 
-#-----------------------------------------------------------------------------
-#  Copyright (C) 2009-2011  The IPython Development Team
-#
-#  Distributed under the terms of the BSD License.  The full license is in
-#  the file COPYING, distributed as part of this software.
-#-----------------------------------------------------------------------------
+# Copyright (C) IPython Development Team
+# Distributed under the terms of the Modified BSD License.
 
-#-----------------------------------------------------------------------------
-# Imports
-#-----------------------------------------------------------------------------
-
-import sys
-
-#-----------------------------------------------------------------------------
-# Decorators
-#-----------------------------------------------------------------------------
 
 def skip_doctest(f):
     """Decorator - mark a function or method for skipping its doctest.
 
     This decorator allows you to mark a function whose docstring you wish to
     omit from testing, while preserving the docstring for introspection, help,
     etc."""
     f.skip_doctest = True
     return f
 
 
 def skip_doctest_py3(f):
     """Decorator - skip the doctest under Python 3."""
     f.skip_doctest = (sys.version_info[0] >= 3)
     return f
 
 def skip_doctest_py2(f):
     """Decorator - skip the doctest under Python 3."""
     f.skip_doctest = (sys.version_info[0] < 3)
     return f

IPython/utils/tokenize2.py

@@ -1,9 +1,595 b''
-"""Load our patched versions of tokenize.
+"""Patched version of standard library tokenize, to deal with various bugs.
+
+Based on Python 3.2 code.
+
+Patches:
+
+- Gareth Rees' patch for Python issue #12691 (untokenizing)
+  - Except we don't encode the output of untokenize
+  - Python 2 compatible syntax, so that it can be byte-compiled at installation
+- Newlines in comments and blank lines should be either NL or NEWLINE, depending
+  on whether they are in a multi-line statement. Filed as Python issue #17061.
+- Export generate_tokens & TokenError
+- u and rb literals are allowed under Python 3.3 and above.
+
+------------------------------------------------------------------------------
+Tokenization help for Python programs.
+
+tokenize(readline) is a generator that breaks a stream of bytes into
+Python tokens.  It decodes the bytes according to PEP-0263 for
+determining source file encoding.
+
+It accepts a readline-like method which is called repeatedly to get the
+next line of input (or b"" for EOF).  It generates 5-tuples with these
+members:
+
+    the token type (see token.py)
+    the token (a string)
+    the starting (row, column) indices of the token (a 2-tuple of ints)
+    the ending (row, column) indices of the token (a 2-tuple of ints)
+    the original line (string)
+
+It is designed to match the working of the Python tokenizer exactly, except
+that it produces COMMENT tokens for comments and gives type OP for all
+operators.  Additionally, all token lists start with an ENCODING token
+which tells you which encoding was used to decode the bytes stream.
 """
+from __future__ import absolute_import
 
+__author__ = 'Ka-Ping Yee <[e-mail]>'
+__credits__ = ('GvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, '
+               'Skip Montanaro, Raymond Hettinger, Trent Nelson, '
+               'Michael Foord')
+import builtins
+import re
 import sys
+from token import *
+from codecs import lookup, BOM_UTF8
+import collections
+from io import TextIOWrapper
+cookie_re = re.compile("coding[:=]\s*([-\w.]+)")
+
+import token
+__all__ = token.__all__ + ["COMMENT", "tokenize", "detect_encoding",
+                           "NL", "untokenize", "ENCODING", "TokenInfo"]
+del token
+
+__all__ += ["generate_tokens", "TokenError"]
 
-if sys.version_info[0] >= 3:
-    from ._tokenize_py3 import *
+COMMENT = N_TOKENS
+tok_name[COMMENT] = 'COMMENT'
+NL = N_TOKENS + 1
+tok_name[NL] = 'NL'
+ENCODING = N_TOKENS + 2
+tok_name[ENCODING] = 'ENCODING'
+N_TOKENS += 3
+
+class TokenInfo(collections.namedtuple('TokenInfo', 'type string start end line')):
+    def __repr__(self):
+        annotated_type = '%d (%s)' % (self.type, tok_name[self.type])
+        return ('TokenInfo(type=%s, string=%r, start=%r, end=%r, line=%r)' %
+                self._replace(type=annotated_type))
+
+def group(*choices): return '(' + '|'.join(choices) + ')'
+def any(*choices): return group(*choices) + '*'
+def maybe(*choices): return group(*choices) + '?'
+
+# Note: we use unicode matching for names ("\w") but ascii matching for
+# number literals.
+Whitespace = r'[ \f\t]*'
+Comment = r'#[^\r\n]*'
+Ignore = Whitespace + any(r'\\\r?\n' + Whitespace) + maybe(Comment)
+Name = r'\w+'
+
+Hexnumber = r'0[xX][0-9a-fA-F]+'
+Binnumber = r'0[bB][01]+'
+Octnumber = r'0[oO][0-7]+'
+Decnumber = r'(?:0+|[1-9][0-9]*)'
+Intnumber = group(Hexnumber, Binnumber, Octnumber, Decnumber)
+Exponent = r'[eE][-+]?[0-9]+'
+Pointfloat = group(r'[0-9]+\.[0-9]*', r'\.[0-9]+') + maybe(Exponent)
+Expfloat = r'[0-9]+' + Exponent
+Floatnumber = group(Pointfloat, Expfloat)
+Imagnumber = group(r'[0-9]+[jJ]', Floatnumber + r'[jJ]')
+Number = group(Imagnumber, Floatnumber, Intnumber)
+
+if sys.version_info.minor >= 3:
+    StringPrefix = r'(?:[bB][rR]?|[rR][bB]?|[uU])?'
 else:
-    from ._tokenize_py2 import *
+    StringPrefix = r'(?:[bB]?[rR]?)?'
+
+# Tail end of ' string.
+Single = r"[^'\\]*(?:\\.[^'\\]*)*'"
+# Tail end of " string.
+Double = r'[^"\\]*(?:\\.[^"\\]*)*"'
+# Tail end of ''' string.
+Single3 = r"[^'\\]*(?:(?:\\.|'(?!''))[^'\\]*)*'''"
+# Tail end of """ string.
+Double3 = r'[^"\\]*(?:(?:\\.|"(?!""))[^"\\]*)*"""'
+Triple = group(StringPrefix + "'''", StringPrefix + '"""')
+# Single-line ' or " string.
+String = group(StringPrefix + r"'[^\n'\\]*(?:\\.[^\n'\\]*)*'",
+               StringPrefix + r'"[^\n"\\]*(?:\\.[^\n"\\]*)*"')
+
+# Because of leftmost-then-longest match semantics, be sure to put the
+# longest operators first (e.g., if = came before ==, == would get
+# recognized as two instances of =).
+Operator = group(r"\*\*=?", r">>=?", r"<<=?", r"!=",
+                 r"//=?", r"->",
+                 r"[+\-*/%&|^=<>]=?",
+                 r"~")
+
+Bracket = '[][(){}]'
+Special = group(r'\r?\n', r'\.\.\.', r'[:;.,@]')
+Funny = group(Operator, Bracket, Special)
+
+PlainToken = group(Number, Funny, String, Name)
+Token = Ignore + PlainToken
+
+# First (or only) line of ' or " string.
+ContStr = group(StringPrefix + r"'[^\n'\\]*(?:\\.[^\n'\\]*)*" +
+                group("'", r'\\\r?\n'),
+                StringPrefix + r'"[^\n"\\]*(?:\\.[^\n"\\]*)*' +
+                group('"', r'\\\r?\n'))
+PseudoExtras = group(r'\\\r?\n', Comment, Triple)
+PseudoToken = Whitespace + group(PseudoExtras, Number, Funny, ContStr, Name)
+
+def _compile(expr):
+    return re.compile(expr, re.UNICODE)
+
+tokenprog, pseudoprog, single3prog, double3prog = map(
+    _compile, (Token, PseudoToken, Single3, Double3))
+endprogs = {"'": _compile(Single), '"': _compile(Double),
+            "'''": single3prog, '"""': double3prog,
+            "r'''": single3prog, 'r"""': double3prog,
+            "b'''": single3prog, 'b"""': double3prog,
+            "R'''": single3prog, 'R"""': double3prog,
+            "B'''": single3prog, 'B"""': double3prog,
+            "br'''": single3prog, 'br"""': double3prog,
+            "bR'''": single3prog, 'bR"""': double3prog,
+            "Br'''": single3prog, 'Br"""': double3prog,
+            "BR'''": single3prog, 'BR"""': double3prog,
+            'r': None, 'R': None, 'b': None, 'B': None}
+
+triple_quoted = {}
+for t in ("'''", '"""',
+          "r'''", 'r"""', "R'''", 'R"""',
+          "b'''", 'b"""', "B'''", 'B"""',
+          "br'''", 'br"""', "Br'''", 'Br"""',
+          "bR'''", 'bR"""', "BR'''", 'BR"""'):
+    triple_quoted[t] = t
+single_quoted = {}
+for t in ("'", '"',
+          "r'", 'r"', "R'", 'R"',
+          "b'", 'b"', "B'", 'B"',
+          "br'", 'br"', "Br'", 'Br"',
+          "bR'", 'bR"', "BR'", 'BR"' ):
+    single_quoted[t] = t
+
+if sys.version_info.minor >= 3:
+    # Python 3.3
+    for _prefix in ['rb', 'rB', 'Rb', 'RB', 'u', 'U']:
+        _t2 = _prefix+'"""'
+        endprogs[_t2] = double3prog
+        triple_quoted[_t2] = _t2
+        _t1 = _prefix + "'''"
+        endprogs[_t1] = single3prog
+        triple_quoted[_t1] = _t1
+        single_quoted[_prefix+'"'] = _prefix+'"'
+        single_quoted[_prefix+"'"] = _prefix+"'"
+    del _prefix, _t2, _t1
+    endprogs['u'] = None
+    endprogs['U'] = None
+
+del _compile
+
+tabsize = 8
+
+class TokenError(Exception): pass
+
+class StopTokenizing(Exception): pass
+
+
+class Untokenizer:
+
+    def __init__(self):
+        self.tokens = []
+        self.prev_row = 1
+        self.prev_col = 0
+        self.encoding = 'utf-8'
+
+    def add_whitespace(self, tok_type, start):
+        row, col = start
+        assert row >= self.prev_row
+        col_offset = col - self.prev_col
+        if col_offset > 0:
+            self.tokens.append(" " * col_offset)
+        elif row > self.prev_row and tok_type not in (NEWLINE, NL, ENDMARKER):
+            # Line was backslash-continued.
+            self.tokens.append(" ")
+
+    def untokenize(self, tokens):
+        iterable = iter(tokens)
+        for t in iterable:
+            if len(t) == 2:
+                self.compat(t, iterable)
+                break
+            tok_type, token, start, end = t[:4]
+            if tok_type == ENCODING:
+                self.encoding = token
+                continue
+            self.add_whitespace(tok_type, start)
+            self.tokens.append(token)
+            self.prev_row, self.prev_col = end
+            if tok_type in (NEWLINE, NL):
+                self.prev_row += 1
+                self.prev_col = 0
+        return "".join(self.tokens)
+
+    def compat(self, token, iterable):
+        # This import is here to avoid problems when the itertools
+        # module is not built yet and tokenize is imported.
+        from itertools import chain
+        startline = False
+        prevstring = False
+        indents = []
+        toks_append = self.tokens.append
+
+        for tok in chain([token], iterable):
+            toknum, tokval = tok[:2]
+            if toknum == ENCODING:
+                self.encoding = tokval
+                continue
+
+            if toknum in (NAME, NUMBER):
+                tokval += ' '
+
+            # Insert a space between two consecutive strings
+            if toknum == STRING:
+                if prevstring:
+                    tokval = ' ' + tokval
+                prevstring = True
+            else:
+                prevstring = False
+
+            if toknum == INDENT:
+                indents.append(tokval)
+                continue
+            elif toknum == DEDENT:
+                indents.pop()
+                continue
+            elif toknum in (NEWLINE, NL):
+                startline = True
+            elif startline and indents:
+                toks_append(indents[-1])
+                startline = False
+            toks_append(tokval)
+
+
+def untokenize(tokens):
+    """
+    Convert ``tokens`` (an iterable) back into Python source code. Return
+    a bytes object, encoded using the encoding specified by the last
+    ENCODING token in ``tokens``, or UTF-8 if no ENCODING token is found.
+
+    The result is guaranteed to tokenize back to match the input so that
+    the conversion is lossless and round-trips are assured.  The
+    guarantee applies only to the token type and token string as the
+    spacing between tokens (column positions) may change.
+
+    :func:`untokenize` has two modes. If the input tokens are sequences
+    of length 2 (``type``, ``string``) then spaces are added as necessary to
+    preserve the round-trip property.
+
+    If the input tokens are sequences of length 4 or more (``type``,
+    ``string``, ``start``, ``end``), as returned by :func:`tokenize`, then
+    spaces are added so that each token appears in the result at the
+    position indicated by ``start`` and ``end``, if possible.
+    """
+    return Untokenizer().untokenize(tokens)
+
+
+def _get_normal_name(orig_enc):
+    """Imitates get_normal_name in tokenizer.c."""
+    # Only care about the first 12 characters.
+    enc = orig_enc[:12].lower().replace("_", "-")
+    if enc == "utf-8" or enc.startswith("utf-8-"):
+        return "utf-8"
+    if enc in ("latin-1", "iso-8859-1", "iso-latin-1") or \
+       enc.startswith(("latin-1-", "iso-8859-1-", "iso-latin-1-")):
+        return "iso-8859-1"
+    return orig_enc
+
+def detect_encoding(readline):
+    """
+    The detect_encoding() function is used to detect the encoding that should
+    be used to decode a Python source file.  It requires one argment, readline,
+    in the same way as the tokenize() generator.
+
+    It will call readline a maximum of twice, and return the encoding used
+    (as a string) and a list of any lines (left as bytes) it has read in.
+
+    It detects the encoding from the presence of a utf-8 bom or an encoding
+    cookie as specified in pep-0263.  If both a bom and a cookie are present,
+    but disagree, a SyntaxError will be raised.  If the encoding cookie is an
+    invalid charset, raise a SyntaxError.  Note that if a utf-8 bom is found,
+    'utf-8-sig' is returned.
+
+    If no encoding is specified, then the default of 'utf-8' will be returned.
+    """
+    bom_found = False
+    encoding = None
+    default = 'utf-8'
+    def read_or_stop():
+        try:
+            return readline()
+        except StopIteration:
+            return b''
+
+    def find_cookie(line):
+        try:
+            # Decode as UTF-8. Either the line is an encoding declaration,
+            # in which case it should be pure ASCII, or it must be UTF-8
+            # per default encoding.
+            line_string = line.decode('utf-8')
+        except UnicodeDecodeError:
+            raise SyntaxError("invalid or missing encoding declaration")
+
+        matches = cookie_re.findall(line_string)
+        if not matches:
+            return None
+        encoding = _get_normal_name(matches[0])
+        try:
+            codec = lookup(encoding)
+        except LookupError:
+            # This behaviour mimics the Python interpreter
+            raise SyntaxError("unknown encoding: " + encoding)
+
+        if bom_found:
+            if encoding != 'utf-8':
+                # This behaviour mimics the Python interpreter
+                raise SyntaxError('encoding problem: utf-8')
+            encoding += '-sig'
+        return encoding
+
+    first = read_or_stop()
+    if first.startswith(BOM_UTF8):
+        bom_found = True
+        first = first[3:]
+        default = 'utf-8-sig'
+    if not first:
+        return default, []
+
+    encoding = find_cookie(first)
+    if encoding:
+        return encoding, [first]
+
+    second = read_or_stop()
+    if not second:
+        return default, [first]
+
+    encoding = find_cookie(second)
+    if encoding:
+        return encoding, [first, second]
+
+    return default, [first, second]
+
+
+def open(filename):
+    """Open a file in read only mode using the encoding detected by
+    detect_encoding().
+    """
+    buffer = builtins.open(filename, 'rb')
+    encoding, lines = detect_encoding(buffer.readline)
+    buffer.seek(0)
+    text = TextIOWrapper(buffer, encoding, line_buffering=True)
+    text.mode = 'r'
+    return text
+
+
+def tokenize(readline):
+    """
+    The tokenize() generator requires one argment, readline, which
+    must be a callable object which provides the same interface as the
+    readline() method of built-in file objects.  Each call to the function
+    should return one line of input as bytes.  Alternately, readline
+    can be a callable function terminating with StopIteration:
+        readline = open(myfile, 'rb').__next__  # Example of alternate readline
+
+    The generator produces 5-tuples with these members: the token type; the
+    token string; a 2-tuple (srow, scol) of ints specifying the row and
+    column where the token begins in the source; a 2-tuple (erow, ecol) of
+    ints specifying the row and column where the token ends in the source;
+    and the line on which the token was found.  The line passed is the
+    logical line; continuation lines are included.
+
+    The first token sequence will always be an ENCODING token
+    which tells you which encoding was used to decode the bytes stream.
+    """
+    # This import is here to avoid problems when the itertools module is not
+    # built yet and tokenize is imported.
+    from itertools import chain, repeat
+    encoding, consumed = detect_encoding(readline)
+    rl_gen = iter(readline, b"")
+    empty = repeat(b"")
+    return _tokenize(chain(consumed, rl_gen, empty).__next__, encoding)
+
+
+def _tokenize(readline, encoding):
+    lnum = parenlev = continued = 0
+    numchars = '0123456789'
+    contstr, needcont = '', 0
+    contline = None
+    indents = [0]
+
+    if encoding is not None:
+        if encoding == "utf-8-sig":
+            # BOM will already have been stripped.
+            encoding = "utf-8"
+        yield TokenInfo(ENCODING, encoding, (0, 0), (0, 0), '')
+    while True:             # loop over lines in stream
+        try:
+            line = readline()
+        except StopIteration:
+            line = b''
+
+        if encoding is not None:
+            line = line.decode(encoding)
+        lnum += 1
+        pos, max = 0, len(line)
+
+        if contstr:                            # continued string
+            if not line:
+                raise TokenError("EOF in multi-line string", strstart)
+            endmatch = endprog.match(line)
+            if endmatch:
+                pos = end = endmatch.end(0)
+                yield TokenInfo(STRING, contstr + line[:end],
+                       strstart, (lnum, end), contline + line)
+                contstr, needcont = '', 0
+                contline = None
+            elif needcont and line[-2:] != '\\\n' and line[-3:] != '\\\r\n':
+                yield TokenInfo(ERRORTOKEN, contstr + line,
+                           strstart, (lnum, len(line)), contline)
+                contstr = ''
+                contline = None
+                continue
+            else:
+                contstr = contstr + line
+                contline = contline + line
+                continue
+
+        elif parenlev == 0 and not continued:  # new statement
+            if not line: break
+            column = 0
+            while pos < max:                   # measure leading whitespace
+                if line[pos] == ' ':
+                    column += 1
+                elif line[pos] == '\t':
+                    column = (column//tabsize + 1)*tabsize
+                elif line[pos] == '\f':
+                    column = 0
+                else:
+                    break
+                pos += 1
+            if pos == max:
+                break
+
+            if line[pos] in '#\r\n':           # skip comments or blank lines
+                if line[pos] == '#':
+                    comment_token = line[pos:].rstrip('\r\n')
+                    nl_pos = pos + len(comment_token)
+                    yield TokenInfo(COMMENT, comment_token,
+                           (lnum, pos), (lnum, pos + len(comment_token)), line)
+                    yield TokenInfo(NEWLINE, line[nl_pos:],
+                           (lnum, nl_pos), (lnum, len(line)), line)
+                else:
+                    yield TokenInfo(NEWLINE, line[pos:],
+                           (lnum, pos), (lnum, len(line)), line)
+                continue
+
+            if column > indents[-1]:           # count indents or dedents
+                indents.append(column)
+                yield TokenInfo(INDENT, line[:pos], (lnum, 0), (lnum, pos), line)
+            while column < indents[-1]:
+                if column not in indents:
+                    raise IndentationError(
+                        "unindent does not match any outer indentation level",
+                        ("<tokenize>", lnum, pos, line))
+                indents = indents[:-1]
+                yield TokenInfo(DEDENT, '', (lnum, pos), (lnum, pos), line)
+
+        else:                                  # continued statement
+            if not line:
+                raise TokenError("EOF in multi-line statement", (lnum, 0))
+            continued = 0
+
+        while pos < max:
+            pseudomatch = pseudoprog.match(line, pos)
+            if pseudomatch:                                # scan for tokens
+                start, end = pseudomatch.span(1)
+                spos, epos, pos = (lnum, start), (lnum, end), end
+                token, initial = line[start:end], line[start]
+
+                if (initial in numchars or                  # ordinary number
+                    (initial == '.' and token != '.' and token != '...')):
+                    yield TokenInfo(NUMBER, token, spos, epos, line)
+                elif initial in '\r\n':
+                    yield TokenInfo(NL if parenlev > 0 else NEWLINE,
+                           token, spos, epos, line)
+                elif initial == '#':
+                    assert not token.endswith("\n")
+                    yield TokenInfo(COMMENT, token, spos, epos, line)
+                elif token in triple_quoted:
+                    endprog = endprogs[token]
+                    endmatch = endprog.match(line, pos)
+                    if endmatch:                           # all on one line
+                        pos = endmatch.end(0)
+                        token = line[start:pos]
+                        yield TokenInfo(STRING, token, spos, (lnum, pos), line)
+                    else:
+                        strstart = (lnum, start)           # multiple lines
+                        contstr = line[start:]
+                        contline = line
+                        break
+                elif initial in single_quoted or \
+                    token[:2] in single_quoted or \
+                    token[:3] in single_quoted:
+                    if token[-1] == '\n':                  # continued string
+                        strstart = (lnum, start)
+                        endprog = (endprogs[initial] or endprogs[token[1]] or
+                                   endprogs[token[2]])
+                        contstr, needcont = line[start:], 1
+                        contline = line
+                        break
+                    else:                                  # ordinary string
+                        yield TokenInfo(STRING, token, spos, epos, line)
+                elif initial.isidentifier():               # ordinary name
+                    yield TokenInfo(NAME, token, spos, epos, line)
+                elif initial == '\\':                      # continued stmt
+                    continued = 1
+                else:
+                    if initial in '([{':
+                        parenlev += 1
+                    elif initial in ')]}':
+                        parenlev -= 1
+                    yield TokenInfo(OP, token, spos, epos, line)
+            else:
+                yield TokenInfo(ERRORTOKEN, line[pos],
+                           (lnum, pos), (lnum, pos+1), line)
+                pos += 1
+
+    for indent in indents[1:]:                 # pop remaining indent levels
+        yield TokenInfo(DEDENT, '', (lnum, 0), (lnum, 0), '')
+    yield TokenInfo(ENDMARKER, '', (lnum, 0), (lnum, 0), '')
+
+
+# An undocumented, backwards compatible, API for all the places in the standard
+# library that expect to be able to use tokenize with strings
+def generate_tokens(readline):
+    return _tokenize(readline, None)
+
+if __name__ == "__main__":
+    # Quick sanity check
+    s = b'''def parseline(self, line):
+            """Parse the line into a command name and a string containing
+            the arguments.  Returns a tuple containing (command, args, line).
+            'command' and 'args' may be None if the line couldn't be parsed.
+            """
+            line = line.strip()
+            if not line:
+                return None, None, line
+            elif line[0] == '?':
+                line = 'help ' + line[1:]
+            elif line[0] == '!':
+                if hasattr(self, 'do_shell'):
+                    line = 'shell ' + line[1:]
+                else:
+                    return None, None, line
+            i, n = 0, len(line)
+            while i < n and line[i] in self.identchars: i = i+1
+            cmd, arg = line[:i], line[i:].strip()
+            return cmd, arg, line
+    '''
+    for tok in tokenize(iter(s.splitlines()).__next__):
+        print(tok)

setup.py

@@ -1,299 +1,297 b''
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 """Setup script for IPython.
 
 Under Posix environments it works like a typical setup.py script.
 Under Windows, the command sdist is not supported, since IPython
 requires utilities which are not available under Windows."""
 
 #-----------------------------------------------------------------------------
 #  Copyright (c) 2008-2011, IPython Development Team.
 #  Copyright (c) 2001-2007, Fernando Perez <[e-mail]>
 #  Copyright (c) 2001, Janko Hauser <[e-mail]>
 #  Copyright (c) 2001, Nathaniel Gray <[e-mail]>
 #
 #  Distributed under the terms of the Modified BSD License.
 #
 #  The full license is in the file COPYING.rst, distributed with this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Minimal Python version sanity check
 #-----------------------------------------------------------------------------
 from __future__ import print_function
 
 import sys
 
 # This check is also made in IPython/__init__, don't forget to update both when
 # changing Python version requirements.
 if sys.version_info < (3,3):
     error = """
 IPython 6.0+ does not support Python 2.6, 2.7, 3.0, 3.1, or 3.2.
 When using Python 2.7, please install IPython 5.x LTS Long Term Support version.
 Beginning with IPython 6.0, Python 3.3 and above is required.
 
 See IPython `README.rst` file for more information:
 
     https://github.com/ipython/ipython/blob/master/README.rst
 
 """
 
     print(error, file=sys.stderr)
     sys.exit(1)
 
-PY3 = (sys.version_info[0] >= 3)
-
 # At least we're on the python version we need, move on.
 
 #-------------------------------------------------------------------------------
 # Imports
 #-------------------------------------------------------------------------------
 
 # Stdlib imports
 import os
 
 from glob import glob
 
 # BEFORE importing distutils, remove MANIFEST. distutils doesn't properly
 # update it when the contents of directories change.
 if os.path.exists('MANIFEST'): os.remove('MANIFEST')
 
 from distutils.core import setup
 
 # Our own imports
 from setupbase import target_update
 
 from setupbase import (
     setup_args,
     find_packages,
     find_package_data,
     check_package_data_first,
     find_entry_points,
     build_scripts_entrypt,
     find_data_files,
     git_prebuild,
     install_symlinked,
     install_lib_symlink,
     install_scripts_for_symlink,
     unsymlink,
 )
 
 isfile = os.path.isfile
 pjoin = os.path.join
 
 #-------------------------------------------------------------------------------
 # Handle OS specific things
 #-------------------------------------------------------------------------------
 
 if os.name in ('nt','dos'):
     os_name = 'windows'
 else:
     os_name = os.name
 
 # Under Windows, 'sdist' has not been supported.  Now that the docs build with
 # Sphinx it might work, but let's not turn it on until someone confirms that it
 # actually works.
 if os_name == 'windows' and 'sdist' in sys.argv:
     print('The sdist command is not available under Windows.  Exiting.')
     sys.exit(1)
 
 
 #-------------------------------------------------------------------------------
 # Things related to the IPython documentation
 #-------------------------------------------------------------------------------
 
 # update the manuals when building a source dist
 if len(sys.argv) >= 2 and sys.argv[1] in ('sdist','bdist_rpm'):
 
     # List of things to be updated. Each entry is a triplet of args for
     # target_update()
     to_update = [
                  ('docs/man/ipython.1.gz',
                   ['docs/man/ipython.1'],
                   'cd docs/man && gzip -9c ipython.1 > ipython.1.gz'),
                  ]
 
 
     [ target_update(*t) for t in to_update ]
 
 #---------------------------------------------------------------------------
 # Find all the packages, package data, and data_files
 #---------------------------------------------------------------------------
 
 packages = find_packages()
 package_data = find_package_data()
 
 data_files = find_data_files()
 
 setup_args['packages'] = packages
 setup_args['package_data'] = package_data
 setup_args['data_files'] = data_files
 
 #---------------------------------------------------------------------------
 # custom distutils commands
 #---------------------------------------------------------------------------
 # imports here, so they are after setuptools import if there was one
 from distutils.command.sdist import sdist
 from distutils.command.upload import upload
 
 class UploadWindowsInstallers(upload):
 
     description = "Upload Windows installers to PyPI (only used from tools/release_windows.py)"
     user_options = upload.user_options + [
         ('files=', 'f', 'exe file (or glob) to upload')
     ]
     def initialize_options(self):
         upload.initialize_options(self)
         meta = self.distribution.metadata
         base = '{name}-{version}'.format(
             name=meta.get_name(),
             version=meta.get_version()
         )
         self.files = os.path.join('dist', '%s.*.exe' % base)
 
     def run(self):
         for dist_file in glob(self.files):
             self.upload_file('bdist_wininst', 'any', dist_file)
 
 setup_args['cmdclass'] = {
     'build_py': \
             check_package_data_first(git_prebuild('IPython')),
     'sdist' : git_prebuild('IPython', sdist),
     'upload_wininst' : UploadWindowsInstallers,
     'symlink': install_symlinked,
     'install_lib_symlink': install_lib_symlink,
     'install_scripts_sym': install_scripts_for_symlink,
     'unsymlink': unsymlink,
 }
 
 
 #---------------------------------------------------------------------------
 # Handle scripts, dependencies, and setuptools specific things
 #---------------------------------------------------------------------------
 
 # For some commands, use setuptools.  Note that we do NOT list install here!
 # If you want a setuptools-enhanced install, just run 'setupegg.py install'
 needs_setuptools = set(('develop', 'release', 'bdist_egg', 'bdist_rpm',
            'bdist', 'bdist_dumb', 'bdist_wininst', 'bdist_wheel',
            'egg_info', 'easy_install', 'upload', 'install_egg_info',
             ))
 
 if len(needs_setuptools.intersection(sys.argv)) > 0:
     import setuptools
 
 # This dict is used for passing extra arguments that are setuptools
 # specific to setup
 setuptools_extra_args = {}
 
 # setuptools requirements
 
 extras_require = dict(
     parallel = ['ipyparallel'],
     qtconsole = ['qtconsole'],
     doc = ['Sphinx>=1.3'],
     test = ['nose>=0.10.1', 'requests', 'testpath', 'pygments', 'nbformat', 'ipykernel', 'numpy'],
     terminal = [],
     kernel = ['ipykernel'],
     nbformat = ['nbformat'],
     notebook = ['notebook', 'ipywidgets'],
     nbconvert = ['nbconvert'],
 )
 
 install_requires = [
     'setuptools>=18.5',
     'decorator',
     'pickleshare',
     'simplegeneric>0.8',
     'traitlets>=4.2',
     'prompt_toolkit>=1.0.3,<2.0.0',
     'pygments',
 ]
 
 # Platform-specific dependencies:
 # This is the correct way to specify these,
 # but requires pip >= 6. pip < 6 ignores these.
 
 extras_require.update({
     ':python_version == "2.7"': ['backports.shutil_get_terminal_size'],
     ':python_version == "2.7" or python_version == "3.3"': ['pathlib2'],
     ':sys_platform != "win32"': ['pexpect'],
     ':sys_platform == "darwin"': ['appnope'],
     ':sys_platform == "win32"': ['colorama'],
     ':sys_platform == "win32" and python_version < "3.6"': ['win_unicode_console>=0.5'],
     'test:python_version == "2.7"': ['mock'],
 })
 # FIXME: re-specify above platform dependencies for pip < 6
 # These would result in non-portable bdists.
 if not any(arg.startswith('bdist') for arg in sys.argv):
     if sys.version_info < (3, 3):
         extras_require['test'].append('mock')
 
     if sys.platform == 'darwin':
         install_requires.extend(['appnope'])
 
     if not sys.platform.startswith('win'):
         install_requires.append('pexpect')
 
     # workaround pypa/setuptools#147, where setuptools misspells
     # platform_python_implementation as python_implementation
     if 'setuptools' in sys.modules:
         for key in list(extras_require):
             if 'platform_python_implementation' in key:
                 new_key = key.replace('platform_python_implementation', 'python_implementation')
                 extras_require[new_key] = extras_require.pop(key)
 
 everything = set()
 for key, deps in extras_require.items():
     if ':' not in key:
         everything.update(deps)
 extras_require['all'] = everything
 
 if 'setuptools' in sys.modules:
     setuptools_extra_args['python_requires'] = '>=3.3'
     setuptools_extra_args['zip_safe'] = False
     setuptools_extra_args['entry_points'] = {
         'console_scripts': find_entry_points(),
         'pygments.lexers': [
             'ipythonconsole = IPython.lib.lexers:IPythonConsoleLexer',
             'ipython = IPython.lib.lexers:IPythonLexer',
             'ipython3 = IPython.lib.lexers:IPython3Lexer',
         ],
     }
     setup_args['extras_require'] = extras_require
     requires = setup_args['install_requires'] = install_requires
 
     # Script to be run by the windows binary installer after the default setup
     # routine, to add shortcuts and similar windows-only things.  Windows
     # post-install scripts MUST reside in the scripts/ dir, otherwise distutils
     # doesn't find them.
     if 'bdist_wininst' in sys.argv:
         if len(sys.argv) > 2 and \
                ('sdist' in sys.argv or 'bdist_rpm' in sys.argv):
             print("ERROR: bdist_wininst must be run alone. Exiting.", file=sys.stderr)
             sys.exit(1)
         setup_args['data_files'].append(
             ['Scripts', ('scripts/ipython.ico', 'scripts/ipython_nb.ico')])
         setup_args['scripts'] = [pjoin('scripts','ipython_win_post_install.py')]
         setup_args['options'] = {"bdist_wininst":
                                  {"install_script":
                                   "ipython_win_post_install.py"}}
 
 else:
     # scripts has to be a non-empty list, or install_scripts isn't called
     setup_args['scripts'] = [e.split('=')[0].strip() for e in find_entry_points()]
 
     setup_args['cmdclass']['build_scripts'] = build_scripts_entrypt
 
 #---------------------------------------------------------------------------
 # Do the actual setup now
 #---------------------------------------------------------------------------
 
 setup_args.update(setuptools_extra_args)
 
 
 
 def main():
     setup(**setup_args)
 
 if __name__ == '__main__':
     main()

tools/github_stats.py

@@ -1,235 +1,232 b''
 #!/usr/bin/env python
 """Simple tools to query github.com and gather stats about issues.
 
 To generate a report for IPython 2.0, run:
 
     python github_stats.py --milestone 2.0 --since-tag rel-1.0.0
 """
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 from __future__ import print_function
 
 import codecs
 import sys
 
 from argparse import ArgumentParser
 from datetime import datetime, timedelta
 from subprocess import check_output
 
 from gh_api import (
     get_paged_request, make_auth_header, get_pull_request, is_pull_request,
     get_milestone_id, get_issues_list, get_authors,
 )
 #-----------------------------------------------------------------------------
 # Globals
 #-----------------------------------------------------------------------------
 
 ISO8601 = "%Y-%m-%dT%H:%M:%SZ"
 PER_PAGE = 100
 
 #-----------------------------------------------------------------------------
 # Functions
 #-----------------------------------------------------------------------------
 
 def round_hour(dt):
     return dt.replace(minute=0,second=0,microsecond=0)
 
 def _parse_datetime(s):
     """Parse dates in the format returned by the Github API."""
     if s:
         return datetime.strptime(s, ISO8601)
     else:
         return datetime.fromtimestamp(0)
 
 def issues2dict(issues):
     """Convert a list of issues to a dict, keyed by issue number."""
     idict = {}
     for i in issues:
         idict[i['number']] = i
     return idict
 
 def split_pulls(all_issues, project="ipython/ipython"):
     """split a list of closed issues into non-PR Issues and Pull Requests"""
     pulls = []
     issues = []
     for i in all_issues:
         if is_pull_request(i):
             pull = get_pull_request(project, i['number'], auth=True)
             pulls.append(pull)
         else:
             issues.append(i)
     return issues, pulls
 
 
 def issues_closed_since(period=timedelta(days=365), project="ipython/ipython", pulls=False):
     """Get all issues closed since a particular point in time. period
     can either be a datetime object, or a timedelta object. In the
     latter case, it is used as a time before the present.
     """
 
     which = 'pulls' if pulls else 'issues'
 
     if isinstance(period, timedelta):
         since = round_hour(datetime.utcnow() - period)
     else:
         since = period
     url = "https://api.github.com/repos/%s/%s?state=closed&sort=updated&since=%s&per_page=%i" % (project, which, since.strftime(ISO8601), PER_PAGE)
     allclosed = get_paged_request(url, headers=make_auth_header())
     
     filtered = [ i for i in allclosed if _parse_datetime(i['closed_at']) > since ]
     if pulls:
         filtered = [ i for i in filtered if _parse_datetime(i['merged_at']) > since ]
         # filter out PRs not against master (backports)
         filtered = [ i for i in filtered if i['base']['ref'] == 'master' ]
     else:
         filtered = [ i for i in filtered if not is_pull_request(i) ]
     
     return filtered
 
 
 def sorted_by_field(issues, field='closed_at', reverse=False):
     """Return a list of issues sorted by closing date date."""
     return sorted(issues, key = lambda i:i[field], reverse=reverse)
 
 
 def report(issues, show_urls=False):
     """Summary report about a list of issues, printing number and title."""
     if show_urls:
         for i in issues:
             role = 'ghpull' if 'merged_at' in i else 'ghissue'
             print(u'* :%s:`%d`: %s' % (role, i['number'],
                                         i['title'].replace(u'`', u'``')))
     else:
         for i in issues:
             print(u'* %d: %s' % (i['number'], i['title'].replace(u'`', u'``')))
 
 #-----------------------------------------------------------------------------
 # Main script
 #-----------------------------------------------------------------------------
 
 if __name__ == "__main__":
 
     print("DEPRECATE: backport_pr.py is deprecated and is is now recommended"
           "to install `ghpro` from PyPI.", file=sys.stderr)
 
-    # deal with unicode
-    if sys.version_info < (3,):
-        sys.stdout = codecs.getwriter('utf8')(sys.stdout)
     
     # Whether to add reST urls for all issues in printout.
     show_urls = True
     
     parser = ArgumentParser()
     parser.add_argument('--since-tag', type=str,
         help="The git tag to use for the starting point (typically the last major release)."
     )
     parser.add_argument('--milestone', type=str,
         help="The GitHub milestone to use for filtering issues [optional]."
     )
     parser.add_argument('--days', type=int,
         help="The number of days of data to summarize (use this or --since-tag)."
     )
     parser.add_argument('--project', type=str, default="ipython/ipython",
         help="The project to summarize."
     )
     parser.add_argument('--links', action='store_true', default=False,
         help="Include links to all closed Issues and PRs in the output."
     )
     
     opts = parser.parse_args()
     tag = opts.since_tag
     
     # set `since` from days or git tag
     if opts.days:
         since = datetime.utcnow() - timedelta(days=opts.days)
     else:
         if not tag:
             tag = check_output(['git', 'describe', '--abbrev=0']).strip().decode('utf8')
         cmd = ['git', 'log', '-1', '--format=%ai', tag]
         tagday, tz = check_output(cmd).strip().decode('utf8').rsplit(' ', 1)
         since = datetime.strptime(tagday, "%Y-%m-%d %H:%M:%S")
         h = int(tz[1:3])
         m = int(tz[3:])
         td = timedelta(hours=h, minutes=m)
         if tz[0] == '-':
             since += td
         else:
             since -= td
     
     since = round_hour(since)
     
     milestone = opts.milestone
     project = opts.project
 
     print("fetching GitHub stats since %s (tag: %s, milestone: %s)" % (since, tag, milestone), file=sys.stderr)
     if milestone:
         milestone_id = get_milestone_id(project=project, milestone=milestone,
                 auth=True)
         issues_and_pulls = get_issues_list(project=project,
                 milestone=milestone_id,
                 state='closed',
                 auth=True,
         )
         issues, pulls = split_pulls(issues_and_pulls, project=project)
     else:
         issues = issues_closed_since(since, project=project, pulls=False)
         pulls = issues_closed_since(since, project=project, pulls=True)
     
     # For regular reports, it's nice to show them in reverse chronological order
     issues = sorted_by_field(issues, reverse=True)
     pulls = sorted_by_field(pulls, reverse=True)
     
     n_issues, n_pulls = map(len, (issues, pulls))
     n_total = n_issues + n_pulls
     
     # Print summary report we can directly include into release notes.
     
     print()
     since_day = since.strftime("%Y/%m/%d")
     today = datetime.today().strftime("%Y/%m/%d")
     print("GitHub stats for %s - %s (tag: %s)" % (since_day, today, tag))
     print()
     print("These lists are automatically generated, and may be incomplete or contain duplicates.")
     print()
     
     ncommits = 0
     all_authors = []
     if tag:
         # print git info, in addition to GitHub info:
         since_tag = tag+'..'
         cmd = ['git', 'log', '--oneline', since_tag]
         ncommits += len(check_output(cmd).splitlines())
         
         author_cmd = ['git', 'log', '--use-mailmap', "--format=* %aN", since_tag]
         all_authors.extend(check_output(author_cmd).decode('utf-8', 'replace').splitlines())
     
     pr_authors = []
     for pr in pulls:
         pr_authors.extend(get_authors(pr))
     ncommits = len(pr_authors) + ncommits - len(pulls)
     author_cmd = ['git', 'check-mailmap'] + pr_authors
     with_email = check_output(author_cmd).decode('utf-8', 'replace').splitlines()
     all_authors.extend([ u'* ' + a.split(' <')[0] for a in with_email ])
     unique_authors = sorted(set(all_authors), key=lambda s: s.lower())
 
     print("We closed %d issues and merged %d pull requests." % (n_issues, n_pulls))
     if milestone:
         print("The full list can be seen `on GitHub <https://github.com/{project}/issues?q=milestone%3A{milestone}+>`__".format(project=project,milestone=milestone)
         )
     
     print()
     print("The following %i authors contributed %i commits." % (len(unique_authors), ncommits))
     print()
     print('\n'.join(unique_authors))
     
     if opts.links:
         print()
         print("GitHub issues and pull requests:")
         print()
         print('Pull Requests (%d):\n' % n_pulls)
         report(pulls, show_urls)
         print()
         print('Issues (%d):\n' % n_issues)
         report(issues, show_urls)