Fix a couple of warnings/errors in doc builds.

IPython/terminal/embed.py

@@ -1,393 +1,392 b''
 # encoding: utf-8
 """
 An embedded IPython shell.
 """
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
 
 import sys
 import warnings
 
 from IPython.core import ultratb, compilerop
 from IPython.core import magic_arguments
 from IPython.core.magic import Magics, magics_class, line_magic
 from IPython.core.interactiveshell import DummyMod, InteractiveShell
 from IPython.terminal.interactiveshell import TerminalInteractiveShell
 from IPython.terminal.ipapp import load_default_config
 
 from traitlets import Bool, CBool, Unicode
 from IPython.utils.io import ask_yes_no
 
 class KillEmbeded(Exception):pass
 
 # This is an additional magic that is exposed in embedded shells.
 @magics_class
 class EmbeddedMagics(Magics):
 
     @line_magic
     @magic_arguments.magic_arguments()
     @magic_arguments.argument('-i', '--instance', action='store_true',
                               help='Kill instance instead of call location')
     @magic_arguments.argument('-x', '--exit', action='store_true',
                               help='Also exit the current session')
     @magic_arguments.argument('-y', '--yes', action='store_true',
                               help='Do not ask confirmation')
     def kill_embedded(self, parameter_s=''):
         """%kill_embedded : deactivate for good the current embedded IPython
 
         This function (after asking for confirmation) sets an internal flag so
         that an embedded IPython will never activate again for the given call
         location. This is useful to permanently disable a shell that is being
         called inside a loop: once you've figured out what you needed from it,
         you may then kill it and the program will then continue to run without
         the interactive shell interfering again.
 
 
-        Kill Instance Option
-        --------------------
-
-        If for some reasons you need to kill the location where the instance is
-        created and not called, for example if you create a single instance in
-        one place and debug in many locations, you can use the ``--instance``
-        option to kill this specific instance. Like for the ``call location``
-        killing an "instance" should work even if it is recreated within a
-        loop.
+        Kill Instance Option:
+
+            If for some reasons you need to kill the location where the instance
+            is created and not called, for example if you create a single
+            instance in one place and debug in many locations, you can use the
+            ``--instance`` option to kill this specific instance. Like for the
+            ``call location`` killing an "instance" should work even if it is
+            recreated within a loop.
 
         .. note::
 
             This was the default behavior before IPython 5.2
 
         """
 
         args = magic_arguments.parse_argstring(self.kill_embedded, parameter_s)
         print(args)
         if args.instance:
             # let no ask
             if not args.yes:
                 kill = ask_yes_no(
                     "Are you sure you want to kill this embedded instance? [y/N] ", 'n')
             else:
                 kill = True
             if kill:
                 self.shell._disable_init_location()
                 print("This embedded IPython instance will not reactivate anymore "
                       "once you exit.")
         else:
             if not args.yes:
                 kill = ask_yes_no(
                     "Are you sure you want to kill this embedded call_location? [y/N] ", 'n')
             else:
                 kill = True
             if kill:
                 self.shell.embedded_active = False
                 print("This embedded IPython  call location will not reactivate anymore "
                       "once you exit.")
 
         if args.exit:
             # Ask-exit does not really ask, it just set internals flags to exit
             # on next loop.
             self.shell.ask_exit()
 
 
     @line_magic
     def exit_raise(self, parameter_s=''):
         """%exit_raise Make the current embedded kernel exit and raise and exception.
 
         This function sets an internal flag so that an embedded IPython will
         raise a `IPython.terminal.embed.KillEmbeded` Exception on exit, and then exit the current I. This is
         useful to permanently exit a loop that create IPython embed instance.
         """
 
         self.shell.should_raise = True
         self.shell.ask_exit()
 
 
 
 class InteractiveShellEmbed(TerminalInteractiveShell):
 
     dummy_mode = Bool(False)
     exit_msg = Unicode('')
     embedded = CBool(True)
     should_raise = CBool(False)
     # Like the base class display_banner is not configurable, but here it
     # is True by default.
     display_banner = CBool(True)
     exit_msg = Unicode()
 
     # When embedding, by default we don't change the terminal title
     term_title = Bool(False,
         help="Automatically set the terminal title"
     ).tag(config=True)
 
     _inactive_locations = set()
 
     @property
     def embedded_active(self):
         return (self._call_location_id not in InteractiveShellEmbed._inactive_locations)\
             and (self._init_location_id not in InteractiveShellEmbed._inactive_locations)
 
     def _disable_init_location(self):
         """Disable the current Instance creation location"""
         InteractiveShellEmbed._inactive_locations.add(self._init_location_id)
 
     @embedded_active.setter
     def embedded_active(self, value):
         if value:
             InteractiveShellEmbed._inactive_locations.discard(
                 self._call_location_id)
             InteractiveShellEmbed._inactive_locations.discard(
                 self._init_location_id)
         else:
             InteractiveShellEmbed._inactive_locations.add(
                 self._call_location_id)
 
     def __init__(self, **kw):
         if kw.get('user_global_ns', None) is not None:
             raise DeprecationWarning(
                 "Key word argument `user_global_ns` has been replaced by `user_module` since IPython 4.0.")
 
         clid = kw.pop('_init_location_id', None)
         if not clid:
             frame = sys._getframe(1)
             clid = '%s:%s' % (frame.f_code.co_filename, frame.f_lineno)
         self._init_location_id = clid
 
         super(InteractiveShellEmbed,self).__init__(**kw)
 
         # don't use the ipython crash handler so that user exceptions aren't
         # trapped
         sys.excepthook = ultratb.FormattedTB(color_scheme=self.colors,
                                              mode=self.xmode,
                                              call_pdb=self.pdb)
 
     def init_sys_modules(self):
         """
         Explicitly overwrite :mod:`IPython.core.interactiveshell` to do nothing.
         """
         pass
 
     def init_magics(self):
         super(InteractiveShellEmbed, self).init_magics()
         self.register_magics(EmbeddedMagics)
 
     def __call__(self, header='', local_ns=None, module=None, dummy=None,
                  stack_depth=1, global_ns=None, compile_flags=None, **kw):
         """Activate the interactive interpreter.
 
         __call__(self,header='',local_ns=None,module=None,dummy=None) -> Start
         the interpreter shell with the given local and global namespaces, and
         optionally print a header string at startup.
 
         The shell can be globally activated/deactivated using the
         dummy_mode attribute. This allows you to turn off a shell used
         for debugging globally.
 
         However, *each* time you call the shell you can override the current
         state of dummy_mode with the optional keyword parameter 'dummy'. For
         example, if you set dummy mode on with IPShell.dummy_mode = True, you
         can still have a specific call work by making it as IPShell(dummy=False).
         """
 
         # we are called, set the underlying interactiveshell not to exit.
         self.keep_running = True
 
         # If the user has turned it off, go away
         clid = kw.pop('_call_location_id', None)
         if not clid:
             frame = sys._getframe(1)
             clid = '%s:%s' % (frame.f_code.co_filename, frame.f_lineno)
         self._call_location_id = clid
 
         if not self.embedded_active:
             return
 
         # Normal exits from interactive mode set this flag, so the shell can't
         # re-enter (it checks this variable at the start of interactive mode).
         self.exit_now = False
 
         # Allow the dummy parameter to override the global __dummy_mode
         if dummy or (dummy != 0 and self.dummy_mode):
             return
 
         # self.banner is auto computed
         if header:
             self.old_banner2 = self.banner2
             self.banner2 = self.banner2 + '\n' + header + '\n'
         else:
             self.old_banner2 = ''
 
         if self.display_banner:
             self.show_banner()
 
         # Call the embedding code with a stack depth of 1 so it can skip over
         # our call and get the original caller's namespaces.
         self.mainloop(local_ns, module, stack_depth=stack_depth,
                       global_ns=global_ns, compile_flags=compile_flags)
 
         self.banner2 = self.old_banner2
 
         if self.exit_msg is not None:
             print(self.exit_msg)
 
         if self.should_raise:
             raise KillEmbeded('Embedded IPython raising error, as user requested.')
 
 
     def mainloop(self, local_ns=None, module=None, stack_depth=0,
                  display_banner=None, global_ns=None, compile_flags=None):
         """Embeds IPython into a running python program.
 
         Parameters
         ----------
 
         local_ns, module
           Working local namespace (a dict) and module (a module or similar
           object). If given as None, they are automatically taken from the scope
           where the shell was called, so that program variables become visible.
 
         stack_depth : int
           How many levels in the stack to go to looking for namespaces (when
           local_ns or module is None). This allows an intermediate caller to
           make sure that this function gets the namespace from the intended
           level in the stack. By default (0) it will get its locals and globals
           from the immediate caller.
 
         compile_flags
           A bit field identifying the __future__ features
           that are enabled, as passed to the builtin :func:`compile` function.
           If given as None, they are automatically taken from the scope where
           the shell was called.
 
         """
         
         if (global_ns is not None) and (module is None):
             raise DeprecationWarning("'global_ns' keyword argument is deprecated, and has been removed in IPython 5.0 use `module` keyword argument instead.")
 
         if (display_banner is not None):
             warnings.warn("The display_banner parameter is deprecated since IPython 4.0", DeprecationWarning)
 
         # Get locals and globals from caller
         if ((local_ns is None or module is None or compile_flags is None)
             and self.default_user_namespaces):
             call_frame = sys._getframe(stack_depth).f_back
 
             if local_ns is None:
                 local_ns = call_frame.f_locals
             if module is None:
                 global_ns = call_frame.f_globals
                 try:
                     module = sys.modules[global_ns['__name__']]
                 except KeyError:
                     warnings.warn("Failed to get module %s" % \
                         global_ns.get('__name__', 'unknown module')
                     )
                     module = DummyMod()
                     module.__dict__ = global_ns
             if compile_flags is None:
                 compile_flags = (call_frame.f_code.co_flags &
                                  compilerop.PyCF_MASK)
         
         # Save original namespace and module so we can restore them after 
         # embedding; otherwise the shell doesn't shut down correctly.
         orig_user_module = self.user_module
         orig_user_ns = self.user_ns
         orig_compile_flags = self.compile.flags
         
         # Update namespaces and fire up interpreter
         
         # The global one is easy, we can just throw it in
         if module is not None:
             self.user_module = module
 
         # But the user/local one is tricky: ipython needs it to store internal
         # data, but we also need the locals. We'll throw our hidden variables
         # like _ih and get_ipython() into the local namespace, but delete them
         # later.
         if local_ns is not None:
             reentrant_local_ns = {k: v for (k, v) in local_ns.items() if k not in self.user_ns_hidden.keys()}
             self.user_ns = reentrant_local_ns
             self.init_user_ns()
 
         # Compiler flags
         if compile_flags is not None:
             self.compile.flags = compile_flags
 
         # make sure the tab-completer has the correct frame information, so it
         # actually completes using the frame's locals/globals
         self.set_completer_frame()
 
         with self.builtin_trap, self.display_trap:
             self.interact()
         
         # now, purge out the local namespace of IPython's hidden variables.
         if local_ns is not None:
             local_ns.update({k: v for (k, v) in self.user_ns.items() if k not in self.user_ns_hidden.keys()})
 
         
         # Restore original namespace so shell can shut down when we exit.
         self.user_module = orig_user_module
         self.user_ns = orig_user_ns
         self.compile.flags = orig_compile_flags
 
 
 def embed(**kwargs):
     """Call this to embed IPython at the current point in your program.
 
     The first invocation of this will create an :class:`InteractiveShellEmbed`
     instance and then call it.  Consecutive calls just call the already
     created instance.
 
     If you don't want the kernel to initialize the namespace
     from the scope of the surrounding function,
     and/or you want to load full IPython configuration,
     you probably want `IPython.start_ipython()` instead.
 
     Here is a simple example::
 
         from IPython import embed
         a = 10
         b = 20
         embed(header='First time')
         c = 30
         d = 40
         embed()
 
     Full customization can be done by passing a :class:`Config` in as the
     config argument.
     """
     config = kwargs.get('config')
     header = kwargs.pop('header', u'')
     compile_flags = kwargs.pop('compile_flags', None)
     if config is None:
         config = load_default_config()
         config.InteractiveShellEmbed = config.TerminalInteractiveShell
         kwargs['config'] = config
     #save ps1/ps2 if defined
     ps1 = None
     ps2 = None
     try:
         ps1 = sys.ps1
         ps2 = sys.ps2
     except AttributeError:
         pass
     #save previous instance
     saved_shell_instance = InteractiveShell._instance
     if saved_shell_instance is not None:
         cls = type(saved_shell_instance)
         cls.clear_instance()
     frame = sys._getframe(1)
     shell = InteractiveShellEmbed.instance(_init_location_id='%s:%s' % (
         frame.f_code.co_filename, frame.f_lineno), **kwargs)
     shell(header=header, stack_depth=2, compile_flags=compile_flags,
           _call_location_id='%s:%s' % (frame.f_code.co_filename, frame.f_lineno))
     InteractiveShellEmbed.clear_instance()
     #restore previous instance
     if saved_shell_instance is not None:
         cls = type(saved_shell_instance)
         cls.clear_instance()
         for subclass in cls._walk_mro():
             subclass._instance = saved_shell_instance
     if ps1 is not None:
         sys.ps1 = ps1
         sys.ps2 = ps2

IPython/terminal/interactiveshell.py

@@ -1,518 +1,516 b''
 """IPython terminal interface using prompt_toolkit"""
 
 import os
 import sys
 import warnings
 from warnings import warn
 
 from IPython.core.interactiveshell import InteractiveShell, InteractiveShellABC
 from IPython.utils import io
 from IPython.utils.py3compat import cast_unicode_py2, input
 from IPython.utils.terminal import toggle_set_term_title, set_term_title
 from IPython.utils.process import abbrev_cwd
 from traitlets import Bool, Unicode, Dict, Integer, observe, Instance, Type, default, Enum, Union
 
 from prompt_toolkit.document import Document
 from prompt_toolkit.enums import DEFAULT_BUFFER, EditingMode
 from prompt_toolkit.filters import (HasFocus, Condition, IsDone)
 from prompt_toolkit.history import InMemoryHistory
 from prompt_toolkit.shortcuts import create_prompt_application, create_eventloop, create_prompt_layout, create_output
 from prompt_toolkit.interface import CommandLineInterface
 from prompt_toolkit.key_binding.manager import KeyBindingManager
 from prompt_toolkit.layout.processors import ConditionalProcessor, HighlightMatchingBracketProcessor
 from prompt_toolkit.styles import PygmentsStyle, DynamicStyle
 
 from pygments.styles import get_style_by_name, get_all_styles
 from pygments.style import Style
 from pygments.token import Token
 
 from .debugger import TerminalPdb, Pdb
 from .magics import TerminalMagics
 from .pt_inputhooks import get_inputhook_name_and_func
 from .prompts import Prompts, ClassicPrompts, RichPromptDisplayHook
 from .ptutils import IPythonPTCompleter, IPythonPTLexer
 from .shortcuts import register_ipython_shortcuts
 
 DISPLAY_BANNER_DEPRECATED = object()
 
 
 class _NoStyle(Style): pass
 
 
 
 _style_overrides_light_bg = {
             Token.Prompt: '#0000ff',
             Token.PromptNum: '#0000ee bold',
             Token.OutPrompt: '#cc0000',
             Token.OutPromptNum: '#bb0000 bold',
 }
 
 _style_overrides_linux = {
             Token.Prompt: '#00cc00',
             Token.PromptNum: '#00bb00 bold',
             Token.OutPrompt: '#cc0000',
             Token.OutPromptNum: '#bb0000 bold',
 }
 
 
 
 def get_default_editor():
     try:
         return os.environ['EDITOR']
     except KeyError:
         pass
     except UnicodeError:
         warn("$EDITOR environment variable is not pure ASCII. Using platform "
              "default editor.")
 
     if os.name == 'posix':
         return 'vi'  # the only one guaranteed to be there!
     else:
         return 'notepad' # same in Windows!
 
 # conservatively check for tty
 # overridden streams can result in things like:
 # - sys.stdin = None
 # - no isatty method
 for _name in ('stdin', 'stdout', 'stderr'):
     _stream = getattr(sys, _name)
     if not _stream or not hasattr(_stream, 'isatty') or not _stream.isatty():
         _is_tty = False
         break
 else:
     _is_tty = True
 
 
 _use_simple_prompt = ('IPY_TEST_SIMPLE_PROMPT' in os.environ) or (not _is_tty)
 
 class TerminalInteractiveShell(InteractiveShell):
     space_for_menu = Integer(6, help='Number of line at the bottom of the screen '
                                                   'to reserve for the completion menu'
                             ).tag(config=True)
 
     def _space_for_menu_changed(self, old, new):
         self._update_layout()
 
     pt_cli = None
     debugger_history = None
     _pt_app = None
 
     simple_prompt = Bool(_use_simple_prompt,
         help="""Use `raw_input` for the REPL, without completion, multiline input, and prompt colors.
 
             Useful when controlling IPython as a subprocess, and piping STDIN/OUT/ERR. Known usage are:
             IPython own testing machinery, and emacs inferior-shell integration through elpy.
 
             This mode default to `True` if the `IPY_TEST_SIMPLE_PROMPT`
-            environment variable is set, or the current terminal is not a tty.
-
-            """
+            environment variable is set, or the current terminal is not a tty."""
             ).tag(config=True)
 
     @property
     def debugger_cls(self):
         return Pdb if self.simple_prompt else TerminalPdb
 
     confirm_exit = Bool(True,
         help="""
         Set to confirm when you try to exit IPython with an EOF (Control-D
         in Unix, Control-Z/Enter in Windows). By typing 'exit' or 'quit',
         you can force a direct exit without any confirmation.""",
     ).tag(config=True)
 
     editing_mode = Unicode('emacs',
         help="Shortcut style to use at the prompt. 'vi' or 'emacs'.",
     ).tag(config=True)
 
     mouse_support = Bool(False,
         help="Enable mouse support in the prompt"
     ).tag(config=True)
 
     highlighting_style = Union([Unicode('legacy'), Type(klass=Style)],
         help="""The name or class of a Pygments style to use for syntax
         highlighting: \n %s""" % ', '.join(get_all_styles())
     ).tag(config=True)
 
 
     @observe('highlighting_style')
     @observe('colors')
     def _highlighting_style_changed(self, change):
         self.refresh_style()
 
     def refresh_style(self):
         self._style = self._make_style_from_name_or_cls(self.highlighting_style)
 
 
     highlighting_style_overrides = Dict(
         help="Override highlighting format for specific tokens"
     ).tag(config=True)
 
     true_color = Bool(False,
         help=("Use 24bit colors instead of 256 colors in prompt highlighting. "
               "If your terminal supports true color, the following command "
               "should print 'TRUECOLOR' in orange: "
               "printf \"\\x1b[38;2;255;100;0mTRUECOLOR\\x1b[0m\\n\"")
     ).tag(config=True)
 
     editor = Unicode(get_default_editor(),
         help="Set the editor used by IPython (default to $EDITOR/vi/notepad)."
     ).tag(config=True)
 
     prompts_class = Type(Prompts, help='Class used to generate Prompt token for prompt_toolkit').tag(config=True)
 
     prompts = Instance(Prompts)
 
     @default('prompts')
     def _prompts_default(self):
         return self.prompts_class(self)
 
     @observe('prompts')
     def _(self, change):
         self._update_layout()
 
     @default('displayhook_class')
     def _displayhook_class_default(self):
         return RichPromptDisplayHook
 
     term_title = Bool(True,
         help="Automatically set the terminal title"
     ).tag(config=True)
 
     display_completions = Enum(('column', 'multicolumn','readlinelike'),
         help= ( "Options for displaying tab completions, 'column', 'multicolumn', and "
                 "'readlinelike'. These options are for `prompt_toolkit`, see "
                 "`prompt_toolkit` documentation for more information."
                 ),
         default_value='multicolumn').tag(config=True)
 
     highlight_matching_brackets = Bool(True,
         help="Highlight matching brackets.",
     ).tag(config=True)
 
     extra_open_editor_shortcuts = Bool(False,
         help="Enable vi (v) or Emacs (C-X C-E) shortcuts to open an external editor. "
              "This is in addition to the F2 binding, which is always enabled."
     ).tag(config=True)
 
     @observe('term_title')
     def init_term_title(self, change=None):
         # Enable or disable the terminal title.
         if self.term_title:
             toggle_set_term_title(True)
             set_term_title('IPython: ' + abbrev_cwd())
         else:
             toggle_set_term_title(False)
 
     def init_display_formatter(self):
         super(TerminalInteractiveShell, self).init_display_formatter()
         # terminal only supports plain text
         self.display_formatter.active_types = ['text/plain']
         # disable `_ipython_display_`
         self.display_formatter.ipython_display_formatter.enabled = False
 
     def init_prompt_toolkit_cli(self):
         if self.simple_prompt:
             # Fall back to plain non-interactive output for tests.
             # This is very limited, and only accepts a single line.
             def prompt():
                 return cast_unicode_py2(input('In [%d]: ' % self.execution_count))
             self.prompt_for_code = prompt
             return
 
         # Set up keyboard shortcuts
         kbmanager = KeyBindingManager.for_prompt(
             enable_open_in_editor=self.extra_open_editor_shortcuts,
         )
         register_ipython_shortcuts(kbmanager.registry, self)
 
         # Pre-populate history from IPython's history database
         history = InMemoryHistory()
         last_cell = u""
         for __, ___, cell in self.history_manager.get_tail(self.history_load_length,
                                                         include_latest=True):
             # Ignore blank lines and consecutive duplicates
             cell = cell.rstrip()
             if cell and (cell != last_cell):
                 history.append(cell)
                 last_cell = cell
 
         self._style = self._make_style_from_name_or_cls(self.highlighting_style)
         style = DynamicStyle(lambda: self._style)
 
         editing_mode = getattr(EditingMode, self.editing_mode.upper())
 
         def patch_stdout(**kwargs):
             return self.pt_cli.patch_stdout_context(**kwargs)
 
         self._pt_app = create_prompt_application(
                             editing_mode=editing_mode,
                             key_bindings_registry=kbmanager.registry,
                             history=history,
                             completer=IPythonPTCompleter(shell=self,
                                                     patch_stdout=patch_stdout),
                             enable_history_search=True,
                             style=style,
                             mouse_support=self.mouse_support,
                             **self._layout_options()
         )
         self._eventloop = create_eventloop(self.inputhook)
         self.pt_cli = CommandLineInterface(
             self._pt_app, eventloop=self._eventloop,
             output=create_output(true_color=self.true_color))
 
     def _make_style_from_name_or_cls(self, name_or_cls):
         """
         Small wrapper that make an IPython compatible style from a style name
 
         We need that to add style for prompt ... etc.
         """
         style_overrides = {}
         if name_or_cls == 'legacy':
             legacy = self.colors.lower()
             if legacy == 'linux':
                 style_cls = get_style_by_name('monokai')
                 style_overrides = _style_overrides_linux
             elif legacy == 'lightbg':
                 style_overrides = _style_overrides_light_bg
                 style_cls = get_style_by_name('pastie')
             elif legacy == 'neutral':
                 # The default theme needs to be visible on both a dark background
                 # and a light background, because we can't tell what the terminal
                 # looks like. These tweaks to the default theme help with that.
                 style_cls = get_style_by_name('default')
                 style_overrides.update({
                     Token.Number: '#007700',
                     Token.Operator: 'noinherit',
                     Token.String: '#BB6622',
                     Token.Name.Function: '#2080D0',
                     Token.Name.Class: 'bold #2080D0',
                     Token.Name.Namespace: 'bold #2080D0',
                     Token.Prompt: '#009900',
                     Token.PromptNum: '#00ff00 bold',
                     Token.OutPrompt: '#990000',
                     Token.OutPromptNum: '#ff0000 bold',
                 })
 
                 # Hack: Due to limited color support on the Windows console
                 # the prompt colors will be wrong without this
                 if os.name == 'nt':
                     style_overrides.update({
                         Token.Prompt: '#ansidarkgreen',
                         Token.PromptNum: '#ansigreen bold',
                         Token.OutPrompt: '#ansidarkred',
                         Token.OutPromptNum: '#ansired bold',
                     })
             elif legacy =='nocolor':
                 style_cls=_NoStyle
                 style_overrides = {}
             else :
                 raise ValueError('Got unknown colors: ', legacy)
         else :
             if isinstance(name_or_cls, str):
                 style_cls = get_style_by_name(name_or_cls)
             else:
                 style_cls = name_or_cls
             style_overrides = {
                 Token.Prompt: '#009900',
                 Token.PromptNum: '#00ff00 bold',
                 Token.OutPrompt: '#990000',
                 Token.OutPromptNum: '#ff0000 bold',
             }
         style_overrides.update(self.highlighting_style_overrides)
         style = PygmentsStyle.from_defaults(pygments_style_cls=style_cls,
                                             style_dict=style_overrides)
 
         return style
 
     def _layout_options(self):
         """
         Return the current layout option for the current Terminal InteractiveShell
         """
         return {
                 'lexer':IPythonPTLexer(),
                 'reserve_space_for_menu':self.space_for_menu,
                 'get_prompt_tokens':self.prompts.in_prompt_tokens,
                 'get_continuation_tokens':self.prompts.continuation_prompt_tokens,
                 'multiline':True,
                 'display_completions_in_columns': (self.display_completions == 'multicolumn'),
 
                 # Highlight matching brackets, but only when this setting is
                 # enabled, and only when the DEFAULT_BUFFER has the focus.
                 'extra_input_processors': [ConditionalProcessor(
                         processor=HighlightMatchingBracketProcessor(chars='[](){}'),
                         filter=HasFocus(DEFAULT_BUFFER) & ~IsDone() &
                             Condition(lambda cli: self.highlight_matching_brackets))],
                 }
 
     def _update_layout(self):
         """
         Ask for a re computation of the application layout, if for example ,
         some configuration options have changed.
         """
         if self._pt_app:
             self._pt_app.layout = create_prompt_layout(**self._layout_options())
 
     def prompt_for_code(self):
         document = self.pt_cli.run(
             pre_run=self.pre_prompt, reset_current_buffer=True)
         return document.text
 
     def enable_win_unicode_console(self):
         if sys.version_info >= (3, 6):
             # Since PEP 528, Python uses the unicode APIs for the Windows
             # console by default, so WUC shouldn't be needed.
             return
 
         import win_unicode_console
         win_unicode_console.enable()
 
     def init_io(self):
         if sys.platform not in {'win32', 'cli'}:
             return
 
         self.enable_win_unicode_console()
 
         import colorama
         colorama.init()
 
         # For some reason we make these wrappers around stdout/stderr.
         # For now, we need to reset them so all output gets coloured.
         # https://github.com/ipython/ipython/issues/8669
         # io.std* are deprecated, but don't show our own deprecation warnings
         # during initialization of the deprecated API.
         with warnings.catch_warnings():
             warnings.simplefilter('ignore', DeprecationWarning)
             io.stdout = io.IOStream(sys.stdout)
             io.stderr = io.IOStream(sys.stderr)
 
     def init_magics(self):
         super(TerminalInteractiveShell, self).init_magics()
         self.register_magics(TerminalMagics)
 
     def init_alias(self):
         # The parent class defines aliases that can be safely used with any
         # frontend.
         super(TerminalInteractiveShell, self).init_alias()
 
         # Now define aliases that only make sense on the terminal, because they
         # need direct access to the console in a way that we can't emulate in
         # GUI or web frontend
         if os.name == 'posix':
             for cmd in ['clear', 'more', 'less', 'man']:
                 self.alias_manager.soft_define_alias(cmd, cmd)
 
 
     def __init__(self, *args, **kwargs):
         super(TerminalInteractiveShell, self).__init__(*args, **kwargs)
         self.init_prompt_toolkit_cli()
         self.init_term_title()
         self.keep_running = True
 
         self.debugger_history = InMemoryHistory()
 
     def ask_exit(self):
         self.keep_running = False
 
     rl_next_input = None
 
     def pre_prompt(self):
         if self.rl_next_input:
             # We can't set the buffer here, because it will be reset just after
             # this. Adding a callable to pre_run_callables does what we need
             # after the buffer is reset.
             s = cast_unicode_py2(self.rl_next_input)
             def set_doc():
                 self.pt_cli.application.buffer.document = Document(s)
             if hasattr(self.pt_cli, 'pre_run_callables'):
                 self.pt_cli.pre_run_callables.append(set_doc)
             else:
                 # Older version of prompt_toolkit; it's OK to set the document
                 # directly here.
                 set_doc()
             self.rl_next_input = None
 
     def interact(self, display_banner=DISPLAY_BANNER_DEPRECATED):
 
         if display_banner is not DISPLAY_BANNER_DEPRECATED:
             warn('interact `display_banner` argument is deprecated since IPython 5.0. Call `show_banner()` if needed.', DeprecationWarning, stacklevel=2)
 
         self.keep_running = True
         while self.keep_running:
             print(self.separate_in, end='')
 
             try:
                 code = self.prompt_for_code()
             except EOFError:
                 if (not self.confirm_exit) \
                         or self.ask_yes_no('Do you really want to exit ([y]/n)?','y','n'):
                     self.ask_exit()
 
             else:
                 if code:
                     self.run_cell(code, store_history=True)
 
     def mainloop(self, display_banner=DISPLAY_BANNER_DEPRECATED):
         # An extra layer of protection in case someone mashing Ctrl-C breaks
         # out of our internal code.
         if display_banner is not DISPLAY_BANNER_DEPRECATED:
             warn('mainloop `display_banner` argument is deprecated since IPython 5.0. Call `show_banner()` if needed.', DeprecationWarning, stacklevel=2)
         while True:
             try:
                 self.interact()
                 break
             except KeyboardInterrupt as e:
                 print("\n%s escaped interact()\n" % type(e).__name__)
             finally:
                 # An interrupt during the eventloop will mess up the
                 # internal state of the prompt_toolkit library.
                 # Stopping the eventloop fixes this, see
                 # https://github.com/ipython/ipython/pull/9867
                 if hasattr(self, '_eventloop'):
                     self._eventloop.stop()
 
     _inputhook = None
     def inputhook(self, context):
         if self._inputhook is not None:
             self._inputhook(context)
 
     active_eventloop = None
     def enable_gui(self, gui=None):
         if gui:
             self.active_eventloop, self._inputhook =\
                 get_inputhook_name_and_func(gui)
         else:
             self.active_eventloop = self._inputhook = None
 
     # Run !system commands directly, not through pipes, so terminal programs
     # work correctly.
     system = InteractiveShell.system_raw
 
     def auto_rewrite_input(self, cmd):
         """Overridden from the parent class to use fancy rewriting prompt"""
         if not self.show_rewritten_input:
             return
 
         tokens = self.prompts.rewrite_prompt_tokens()
         if self.pt_cli:
             self.pt_cli.print_tokens(tokens)
             print(cmd)
         else:
             prompt = ''.join(s for t, s in tokens)
             print(prompt, cmd, sep='')
 
     _prompts_before = None
     def switch_doctest_mode(self, mode):
         """Switch prompts to classic for %doctest_mode"""
         if mode:
             self._prompts_before = self.prompts
             self.prompts = ClassicPrompts(self)
         elif self._prompts_before:
             self.prompts = self._prompts_before
             self._prompts_before = None
         self._update_layout()
 
 
 InteractiveShellABC.register(TerminalInteractiveShell)
 
 if __name__ == '__main__':
     TerminalInteractiveShell.instance().interact()

IPython/utils/tokenize2.py

@@ -1,594 +1,595 b''
 """Patched version of standard library tokenize, to deal with various bugs.
 
 Based on Python 3.2 code.
 
 Patches:
 
 - Gareth Rees' patch for Python issue #12691 (untokenizing)
   - Except we don't encode the output of untokenize
   - Python 2 compatible syntax, so that it can be byte-compiled at installation
 - Newlines in comments and blank lines should be either NL or NEWLINE, depending
   on whether they are in a multi-line statement. Filed as Python issue #17061.
 - Export generate_tokens & TokenError
 - u and rb literals are allowed under Python 3.3 and above.
 
 ------------------------------------------------------------------------------
 Tokenization help for Python programs.
 
 tokenize(readline) is a generator that breaks a stream of bytes into
 Python tokens.  It decodes the bytes according to PEP-0263 for
 determining source file encoding.
 
 It accepts a readline-like method which is called repeatedly to get the
 next line of input (or b"" for EOF).  It generates 5-tuples with these
 members:
 
     the token type (see token.py)
     the token (a string)
     the starting (row, column) indices of the token (a 2-tuple of ints)
     the ending (row, column) indices of the token (a 2-tuple of ints)
     the original line (string)
 
 It is designed to match the working of the Python tokenizer exactly, except
 that it produces COMMENT tokens for comments and gives type OP for all
 operators.  Additionally, all token lists start with an ENCODING token
 which tells you which encoding was used to decode the bytes stream.
 """
 
 __author__ = 'Ka-Ping Yee <[e-mail]>'
 __credits__ = ('GvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, '
                'Skip Montanaro, Raymond Hettinger, Trent Nelson, '
                'Michael Foord')
 import builtins
 import re
 import sys
 from token import *
 from codecs import lookup, BOM_UTF8
 import collections
 from io import TextIOWrapper
 cookie_re = re.compile("coding[:=]\s*([-\w.]+)")
 
 import token
 __all__ = token.__all__ + ["COMMENT", "tokenize", "detect_encoding",
                            "NL", "untokenize", "ENCODING", "TokenInfo"]
 del token
 
 __all__ += ["generate_tokens", "TokenError"]
 
 COMMENT = N_TOKENS
 tok_name[COMMENT] = 'COMMENT'
 NL = N_TOKENS + 1
 tok_name[NL] = 'NL'
 ENCODING = N_TOKENS + 2
 tok_name[ENCODING] = 'ENCODING'
 N_TOKENS += 3
 
 class TokenInfo(collections.namedtuple('TokenInfo', 'type string start end line')):
     def __repr__(self):
         annotated_type = '%d (%s)' % (self.type, tok_name[self.type])
         return ('TokenInfo(type=%s, string=%r, start=%r, end=%r, line=%r)' %
                 self._replace(type=annotated_type))
 
 def group(*choices): return '(' + '|'.join(choices) + ')'
 def any(*choices): return group(*choices) + '*'
 def maybe(*choices): return group(*choices) + '?'
 
 # Note: we use unicode matching for names ("\w") but ascii matching for
 # number literals.
 Whitespace = r'[ \f\t]*'
 Comment = r'#[^\r\n]*'
 Ignore = Whitespace + any(r'\\\r?\n' + Whitespace) + maybe(Comment)
 Name = r'\w+'
 
 Hexnumber = r'0[xX][0-9a-fA-F]+'
 Binnumber = r'0[bB][01]+'
 Octnumber = r'0[oO][0-7]+'
 Decnumber = r'(?:0+|[1-9][0-9]*)'
 Intnumber = group(Hexnumber, Binnumber, Octnumber, Decnumber)
 Exponent = r'[eE][-+]?[0-9]+'
 Pointfloat = group(r'[0-9]+\.[0-9]*', r'\.[0-9]+') + maybe(Exponent)
 Expfloat = r'[0-9]+' + Exponent
 Floatnumber = group(Pointfloat, Expfloat)
 Imagnumber = group(r'[0-9]+[jJ]', Floatnumber + r'[jJ]')
 Number = group(Imagnumber, Floatnumber, Intnumber)
 
 if sys.version_info.minor >= 3:
     StringPrefix = r'(?:[bB][rR]?|[rR][bB]?|[uU])?'
 else:
     StringPrefix = r'(?:[bB]?[rR]?)?'
 
 # Tail end of ' string.
 Single = r"[^'\\]*(?:\\.[^'\\]*)*'"
 # Tail end of " string.
 Double = r'[^"\\]*(?:\\.[^"\\]*)*"'
 # Tail end of ''' string.
 Single3 = r"[^'\\]*(?:(?:\\.|'(?!''))[^'\\]*)*'''"
 # Tail end of """ string.
 Double3 = r'[^"\\]*(?:(?:\\.|"(?!""))[^"\\]*)*"""'
 Triple = group(StringPrefix + "'''", StringPrefix + '"""')
 # Single-line ' or " string.
 String = group(StringPrefix + r"'[^\n'\\]*(?:\\.[^\n'\\]*)*'",
                StringPrefix + r'"[^\n"\\]*(?:\\.[^\n"\\]*)*"')
 
 # Because of leftmost-then-longest match semantics, be sure to put the
 # longest operators first (e.g., if = came before ==, == would get
 # recognized as two instances of =).
 Operator = group(r"\*\*=?", r">>=?", r"<<=?", r"!=",
                  r"//=?", r"->",
                  r"[+\-*/%&|^=<>]=?",
                  r"~")
 
 Bracket = '[][(){}]'
 Special = group(r'\r?\n', r'\.\.\.', r'[:;.,@]')
 Funny = group(Operator, Bracket, Special)
 
 PlainToken = group(Number, Funny, String, Name)
 Token = Ignore + PlainToken
 
 # First (or only) line of ' or " string.
 ContStr = group(StringPrefix + r"'[^\n'\\]*(?:\\.[^\n'\\]*)*" +
                 group("'", r'\\\r?\n'),
                 StringPrefix + r'"[^\n"\\]*(?:\\.[^\n"\\]*)*' +
                 group('"', r'\\\r?\n'))
 PseudoExtras = group(r'\\\r?\n', Comment, Triple)
 PseudoToken = Whitespace + group(PseudoExtras, Number, Funny, ContStr, Name)
 
 def _compile(expr):
     return re.compile(expr, re.UNICODE)
 
 tokenprog, pseudoprog, single3prog, double3prog = map(
     _compile, (Token, PseudoToken, Single3, Double3))
 endprogs = {"'": _compile(Single), '"': _compile(Double),
             "'''": single3prog, '"""': double3prog,
             "r'''": single3prog, 'r"""': double3prog,
             "b'''": single3prog, 'b"""': double3prog,
             "R'''": single3prog, 'R"""': double3prog,
             "B'''": single3prog, 'B"""': double3prog,
             "br'''": single3prog, 'br"""': double3prog,
             "bR'''": single3prog, 'bR"""': double3prog,
             "Br'''": single3prog, 'Br"""': double3prog,
             "BR'''": single3prog, 'BR"""': double3prog,
             'r': None, 'R': None, 'b': None, 'B': None}
 
 triple_quoted = {}
 for t in ("'''", '"""',
           "r'''", 'r"""', "R'''", 'R"""',
           "b'''", 'b"""', "B'''", 'B"""',
           "br'''", 'br"""', "Br'''", 'Br"""',
           "bR'''", 'bR"""', "BR'''", 'BR"""'):
     triple_quoted[t] = t
 single_quoted = {}
 for t in ("'", '"',
           "r'", 'r"', "R'", 'R"',
           "b'", 'b"', "B'", 'B"',
           "br'", 'br"', "Br'", 'Br"',
           "bR'", 'bR"', "BR'", 'BR"' ):
     single_quoted[t] = t
 
 if sys.version_info.minor >= 3:
     # Python 3.3
     for _prefix in ['rb', 'rB', 'Rb', 'RB', 'u', 'U']:
         _t2 = _prefix+'"""'
         endprogs[_t2] = double3prog
         triple_quoted[_t2] = _t2
         _t1 = _prefix + "'''"
         endprogs[_t1] = single3prog
         triple_quoted[_t1] = _t1
         single_quoted[_prefix+'"'] = _prefix+'"'
         single_quoted[_prefix+"'"] = _prefix+"'"
     del _prefix, _t2, _t1
     endprogs['u'] = None
     endprogs['U'] = None
 
 del _compile
 
 tabsize = 8
 
 class TokenError(Exception): pass
 
 class StopTokenizing(Exception): pass
 
 
 class Untokenizer:
 
     def __init__(self):
         self.tokens = []
         self.prev_row = 1
         self.prev_col = 0
         self.encoding = 'utf-8'
 
     def add_whitespace(self, tok_type, start):
         row, col = start
         assert row >= self.prev_row
         col_offset = col - self.prev_col
         if col_offset > 0:
             self.tokens.append(" " * col_offset)
         elif row > self.prev_row and tok_type not in (NEWLINE, NL, ENDMARKER):
             # Line was backslash-continued.
             self.tokens.append(" ")
 
     def untokenize(self, tokens):
         iterable = iter(tokens)
         for t in iterable:
             if len(t) == 2:
                 self.compat(t, iterable)
                 break
             tok_type, token, start, end = t[:4]
             if tok_type == ENCODING:
                 self.encoding = token
                 continue
             self.add_whitespace(tok_type, start)
             self.tokens.append(token)
             self.prev_row, self.prev_col = end
             if tok_type in (NEWLINE, NL):
                 self.prev_row += 1
                 self.prev_col = 0
         return "".join(self.tokens)
 
     def compat(self, token, iterable):
         # This import is here to avoid problems when the itertools
         # module is not built yet and tokenize is imported.
         from itertools import chain
         startline = False
         prevstring = False
         indents = []
         toks_append = self.tokens.append
 
         for tok in chain([token], iterable):
             toknum, tokval = tok[:2]
             if toknum == ENCODING:
                 self.encoding = tokval
                 continue
 
             if toknum in (NAME, NUMBER):
                 tokval += ' '
 
             # Insert a space between two consecutive strings
             if toknum == STRING:
                 if prevstring:
                     tokval = ' ' + tokval
                 prevstring = True
             else:
                 prevstring = False
 
             if toknum == INDENT:
                 indents.append(tokval)
                 continue
             elif toknum == DEDENT:
                 indents.pop()
                 continue
             elif toknum in (NEWLINE, NL):
                 startline = True
             elif startline and indents:
                 toks_append(indents[-1])
                 startline = False
             toks_append(tokval)
 
 
 def untokenize(tokens):
     """
     Convert ``tokens`` (an iterable) back into Python source code. Return
     a bytes object, encoded using the encoding specified by the last
     ENCODING token in ``tokens``, or UTF-8 if no ENCODING token is found.
 
     The result is guaranteed to tokenize back to match the input so that
     the conversion is lossless and round-trips are assured.  The
     guarantee applies only to the token type and token string as the
     spacing between tokens (column positions) may change.
 
     :func:`untokenize` has two modes. If the input tokens are sequences
     of length 2 (``type``, ``string``) then spaces are added as necessary to
     preserve the round-trip property.
 
     If the input tokens are sequences of length 4 or more (``type``,
     ``string``, ``start``, ``end``), as returned by :func:`tokenize`, then
     spaces are added so that each token appears in the result at the
     position indicated by ``start`` and ``end``, if possible.
     """
     return Untokenizer().untokenize(tokens)
 
 
 def _get_normal_name(orig_enc):
     """Imitates get_normal_name in tokenizer.c."""
     # Only care about the first 12 characters.
     enc = orig_enc[:12].lower().replace("_", "-")
     if enc == "utf-8" or enc.startswith("utf-8-"):
         return "utf-8"
     if enc in ("latin-1", "iso-8859-1", "iso-latin-1") or \
        enc.startswith(("latin-1-", "iso-8859-1-", "iso-latin-1-")):
         return "iso-8859-1"
     return orig_enc
 
 def detect_encoding(readline):
     """
     The detect_encoding() function is used to detect the encoding that should
     be used to decode a Python source file.  It requires one argment, readline,
     in the same way as the tokenize() generator.
 
     It will call readline a maximum of twice, and return the encoding used
     (as a string) and a list of any lines (left as bytes) it has read in.
 
     It detects the encoding from the presence of a utf-8 bom or an encoding
     cookie as specified in pep-0263.  If both a bom and a cookie are present,
     but disagree, a SyntaxError will be raised.  If the encoding cookie is an
     invalid charset, raise a SyntaxError.  Note that if a utf-8 bom is found,
     'utf-8-sig' is returned.
 
     If no encoding is specified, then the default of 'utf-8' will be returned.
     """
     bom_found = False
     encoding = None
     default = 'utf-8'
     def read_or_stop():
         try:
             return readline()
         except StopIteration:
             return b''
 
     def find_cookie(line):
         try:
             # Decode as UTF-8. Either the line is an encoding declaration,
             # in which case it should be pure ASCII, or it must be UTF-8
             # per default encoding.
             line_string = line.decode('utf-8')
         except UnicodeDecodeError:
             raise SyntaxError("invalid or missing encoding declaration")
 
         matches = cookie_re.findall(line_string)
         if not matches:
             return None
         encoding = _get_normal_name(matches[0])
         try:
             codec = lookup(encoding)
         except LookupError:
             # This behaviour mimics the Python interpreter
             raise SyntaxError("unknown encoding: " + encoding)
 
         if bom_found:
             if encoding != 'utf-8':
                 # This behaviour mimics the Python interpreter
                 raise SyntaxError('encoding problem: utf-8')
             encoding += '-sig'
         return encoding
 
     first = read_or_stop()
     if first.startswith(BOM_UTF8):
         bom_found = True
         first = first[3:]
         default = 'utf-8-sig'
     if not first:
         return default, []
 
     encoding = find_cookie(first)
     if encoding:
         return encoding, [first]
 
     second = read_or_stop()
     if not second:
         return default, [first]
 
     encoding = find_cookie(second)
     if encoding:
         return encoding, [first, second]
 
     return default, [first, second]
 
 
 def open(filename):
     """Open a file in read only mode using the encoding detected by
     detect_encoding().
     """
     buffer = builtins.open(filename, 'rb')
     encoding, lines = detect_encoding(buffer.readline)
     buffer.seek(0)
     text = TextIOWrapper(buffer, encoding, line_buffering=True)
     text.mode = 'r'
     return text
 
 
 def tokenize(readline):
     """
     The tokenize() generator requires one argment, readline, which
     must be a callable object which provides the same interface as the
     readline() method of built-in file objects.  Each call to the function
     should return one line of input as bytes.  Alternately, readline
-    can be a callable function terminating with StopIteration:
+    can be a callable function terminating with :class:`StopIteration`::
+
         readline = open(myfile, 'rb').__next__  # Example of alternate readline
 
     The generator produces 5-tuples with these members: the token type; the
     token string; a 2-tuple (srow, scol) of ints specifying the row and
     column where the token begins in the source; a 2-tuple (erow, ecol) of
     ints specifying the row and column where the token ends in the source;
     and the line on which the token was found.  The line passed is the
     logical line; continuation lines are included.
 
     The first token sequence will always be an ENCODING token
     which tells you which encoding was used to decode the bytes stream.
     """
     # This import is here to avoid problems when the itertools module is not
     # built yet and tokenize is imported.
     from itertools import chain, repeat
     encoding, consumed = detect_encoding(readline)
     rl_gen = iter(readline, b"")
     empty = repeat(b"")
     return _tokenize(chain(consumed, rl_gen, empty).__next__, encoding)
 
 
 def _tokenize(readline, encoding):
     lnum = parenlev = continued = 0
     numchars = '0123456789'
     contstr, needcont = '', 0
     contline = None
     indents = [0]
 
     if encoding is not None:
         if encoding == "utf-8-sig":
             # BOM will already have been stripped.
             encoding = "utf-8"
         yield TokenInfo(ENCODING, encoding, (0, 0), (0, 0), '')
     while True:             # loop over lines in stream
         try:
             line = readline()
         except StopIteration:
             line = b''
 
         if encoding is not None:
             line = line.decode(encoding)
         lnum += 1
         pos, max = 0, len(line)
 
         if contstr:                            # continued string
             if not line:
                 raise TokenError("EOF in multi-line string", strstart)
             endmatch = endprog.match(line)
             if endmatch:
                 pos = end = endmatch.end(0)
                 yield TokenInfo(STRING, contstr + line[:end],
                        strstart, (lnum, end), contline + line)
                 contstr, needcont = '', 0
                 contline = None
             elif needcont and line[-2:] != '\\\n' and line[-3:] != '\\\r\n':
                 yield TokenInfo(ERRORTOKEN, contstr + line,
                            strstart, (lnum, len(line)), contline)
                 contstr = ''
                 contline = None
                 continue
             else:
                 contstr = contstr + line
                 contline = contline + line
                 continue
 
         elif parenlev == 0 and not continued:  # new statement
             if not line: break
             column = 0
             while pos < max:                   # measure leading whitespace
                 if line[pos] == ' ':
                     column += 1
                 elif line[pos] == '\t':
                     column = (column//tabsize + 1)*tabsize
                 elif line[pos] == '\f':
                     column = 0
                 else:
                     break
                 pos += 1
             if pos == max:
                 break
 
             if line[pos] in '#\r\n':           # skip comments or blank lines
                 if line[pos] == '#':
                     comment_token = line[pos:].rstrip('\r\n')
                     nl_pos = pos + len(comment_token)
                     yield TokenInfo(COMMENT, comment_token,
                            (lnum, pos), (lnum, pos + len(comment_token)), line)
                     yield TokenInfo(NEWLINE, line[nl_pos:],
                            (lnum, nl_pos), (lnum, len(line)), line)
                 else:
                     yield TokenInfo(NEWLINE, line[pos:],
                            (lnum, pos), (lnum, len(line)), line)
                 continue
 
             if column > indents[-1]:           # count indents or dedents
                 indents.append(column)
                 yield TokenInfo(INDENT, line[:pos], (lnum, 0), (lnum, pos), line)
             while column < indents[-1]:
                 if column not in indents:
                     raise IndentationError(
                         "unindent does not match any outer indentation level",
                         ("<tokenize>", lnum, pos, line))
                 indents = indents[:-1]
                 yield TokenInfo(DEDENT, '', (lnum, pos), (lnum, pos), line)
 
         else:                                  # continued statement
             if not line:
                 raise TokenError("EOF in multi-line statement", (lnum, 0))
             continued = 0
 
         while pos < max:
             pseudomatch = pseudoprog.match(line, pos)
             if pseudomatch:                                # scan for tokens
                 start, end = pseudomatch.span(1)
                 spos, epos, pos = (lnum, start), (lnum, end), end
                 token, initial = line[start:end], line[start]
 
                 if (initial in numchars or                  # ordinary number
                     (initial == '.' and token != '.' and token != '...')):
                     yield TokenInfo(NUMBER, token, spos, epos, line)
                 elif initial in '\r\n':
                     yield TokenInfo(NL if parenlev > 0 else NEWLINE,
                            token, spos, epos, line)
                 elif initial == '#':
                     assert not token.endswith("\n")
                     yield TokenInfo(COMMENT, token, spos, epos, line)
                 elif token in triple_quoted:
                     endprog = endprogs[token]
                     endmatch = endprog.match(line, pos)
                     if endmatch:                           # all on one line
                         pos = endmatch.end(0)
                         token = line[start:pos]
                         yield TokenInfo(STRING, token, spos, (lnum, pos), line)
                     else:
                         strstart = (lnum, start)           # multiple lines
                         contstr = line[start:]
                         contline = line
                         break
                 elif initial in single_quoted or \
                     token[:2] in single_quoted or \
                     token[:3] in single_quoted:
                     if token[-1] == '\n':                  # continued string
                         strstart = (lnum, start)
                         endprog = (endprogs[initial] or endprogs[token[1]] or
                                    endprogs[token[2]])
                         contstr, needcont = line[start:], 1
                         contline = line
                         break
                     else:                                  # ordinary string
                         yield TokenInfo(STRING, token, spos, epos, line)
                 elif initial.isidentifier():               # ordinary name
                     yield TokenInfo(NAME, token, spos, epos, line)
                 elif initial == '\\':                      # continued stmt
                     continued = 1
                 else:
                     if initial in '([{':
                         parenlev += 1
                     elif initial in ')]}':
                         parenlev -= 1
                     yield TokenInfo(OP, token, spos, epos, line)
             else:
                 yield TokenInfo(ERRORTOKEN, line[pos],
                            (lnum, pos), (lnum, pos+1), line)
                 pos += 1
 
     for indent in indents[1:]:                 # pop remaining indent levels
         yield TokenInfo(DEDENT, '', (lnum, 0), (lnum, 0), '')
     yield TokenInfo(ENDMARKER, '', (lnum, 0), (lnum, 0), '')
 
 
 # An undocumented, backwards compatible, API for all the places in the standard
 # library that expect to be able to use tokenize with strings
 def generate_tokens(readline):
     return _tokenize(readline, None)
 
 if __name__ == "__main__":
     # Quick sanity check
     s = b'''def parseline(self, line):
             """Parse the line into a command name and a string containing
             the arguments.  Returns a tuple containing (command, args, line).
             'command' and 'args' may be None if the line couldn't be parsed.
             """
             line = line.strip()
             if not line:
                 return None, None, line
             elif line[0] == '?':
                 line = 'help ' + line[1:]
             elif line[0] == '!':
                 if hasattr(self, 'do_shell'):
                     line = 'shell ' + line[1:]
                 else:
                     return None, None, line
             i, n = 0, len(line)
             while i < n and line[i] in self.identchars: i = i+1
             cmd, arg = line[:i], line[i:].strip()
             return cmd, arg, line
     '''
     for tok in tokenize(iter(s.splitlines()).__next__):
         print(tok)

IPython/utils/tokenutil.py

@@ -1,127 +1,127 b''
 """Token-related utilities"""
 
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
 
 from collections import namedtuple
 from io import StringIO
 from keyword import iskeyword
 
 from . import tokenize2
 from .py3compat import cast_unicode_py2
 
 Token = namedtuple('Token', ['token', 'text', 'start', 'end', 'line'])
 
 def generate_tokens(readline):
     """wrap generate_tokens to catch EOF errors"""
     try:
         for token in tokenize2.generate_tokens(readline):
             yield token
     except tokenize2.TokenError:
         # catch EOF error
         return
 
 def line_at_cursor(cell, cursor_pos=0):
     """Return the line in a cell at a given cursor position
     
     Used for calling line-based APIs that don't support multi-line input, yet.
     
     Parameters
     ----------
     
     cell: str
         multiline block of text
     cursor_pos: integer
         the cursor position
     
     Returns
     -------
     
-    (line, offset): (text, integer)
+    (line, offset): (string, integer)
         The line with the current cursor, and the character offset of the start of the line.
     """
     offset = 0
     lines = cell.splitlines(True)
     for line in lines:
         next_offset = offset + len(line)
         if next_offset >= cursor_pos:
             break
         offset = next_offset
     else:
         line = ""
     return (line, offset)
 
 def token_at_cursor(cell, cursor_pos=0):
     """Get the token at a given cursor
     
     Used for introspection.
     
     Function calls are prioritized, so the token for the callable will be returned
     if the cursor is anywhere inside the call.
     
     Parameters
     ----------
     
     cell : unicode
         A block of Python code
     cursor_pos : int
         The location of the cursor in the block where the token should be found
     """
     cell = cast_unicode_py2(cell)
     names = []
     tokens = []
     call_names = []
     
     offsets = {1: 0} # lines start at 1
     for tup in generate_tokens(StringIO(cell).readline):
         
         tok = Token(*tup)
         
         # token, text, start, end, line = tup
         start_line, start_col = tok.start
         end_line, end_col = tok.end
         if end_line + 1 not in offsets:
             # keep track of offsets for each line
             lines = tok.line.splitlines(True)
             for lineno, line in enumerate(lines, start_line + 1):
                 if lineno not in offsets:
                     offsets[lineno] = offsets[lineno-1] + len(line)
         
         offset = offsets[start_line]
         # allow '|foo' to find 'foo' at the beginning of a line
         boundary = cursor_pos + 1 if start_col == 0 else cursor_pos
         if offset + start_col >= boundary:
             # current token starts after the cursor,
             # don't consume it
             break
         
         if tok.token == tokenize2.NAME and not iskeyword(tok.text):
             if names and tokens and tokens[-1].token == tokenize2.OP and tokens[-1].text == '.':
                 names[-1] = "%s.%s" % (names[-1], tok.text)
             else:
                 names.append(tok.text)
         elif tok.token == tokenize2.OP:
             if tok.text == '=' and names:
                 # don't inspect the lhs of an assignment
                 names.pop(-1)
             if tok.text == '(' and names:
                 # if we are inside a function call, inspect the function
                 call_names.append(names[-1])
             elif tok.text == ')' and call_names:
                 call_names.pop(-1)
         
         tokens.append(tok)
         
         if offsets[end_line] + end_col > cursor_pos:
             # we found the cursor, stop reading
             break
         
     if call_names:
         return call_names[-1]
     elif names:
         return names[-1]
     else:
         return ''
     
 

docs/source/config/details.rst

@@ -1,230 +1,230 b''
 =======================
 Specific config details
 =======================
 
 .. _custom_prompts:
 
 Custom Prompts
 ==============
 
 .. versionchanged:: 5.0
 
 From IPython 5, prompts are produced as a list of Pygments tokens, which are
 tuples of (token_type, text). You can customise prompts by writing a method
 which generates a list of tokens.
 
 There are four kinds of prompt:
 
 * The **in** prompt is shown before the first line of input
   (default like ``In [1]:``).
 * The **continuation** prompt is shown before further lines of input
   (default like ``...:``).
 * The **rewrite** prompt is shown to highlight how special syntax has been
   interpreted (default like ``----->``).
 * The **out** prompt is shown before the result from evaluating the input
   (default like ``Out[1]:``).
 
 Custom prompts are supplied together as a class. If you want to customise only
 some of the prompts, inherit from :class:`IPython.terminal.prompts.Prompts`,
 which defines the defaults. The required interface is like this:
 
 .. class:: MyPrompts(shell)
 
    Prompt style definition. *shell* is a reference to the
    :class:`~.TerminalInteractiveShell` instance.
 
    .. method:: in_prompt_tokens(cli=None)
                continuation_prompt_tokens(self, cli=None, width=None)
                rewrite_prompt_tokens()
                out_prompt_tokens()
 
       Return the respective prompts as lists of ``(token_type, text)`` tuples.
 
       For continuation prompts, *width* is an integer representing the width of
       the prompt area in terminal columns.
 
       *cli*, where used, is the prompt_toolkit ``CommandLineInterface`` instance.
       This is mainly for compatibility with the API prompt_toolkit expects.
 
 Here is an example Prompt class that will show the current working directory
 in the input prompt:
 
 .. code-block:: python
 
     from IPython.terminal.prompts import Prompts, Token
     import os
 
     class MyPrompt(Prompts):
          def in_prompt_tokens(self, cli=None):
              return [(Token, os.getcwd()),
                      (Token.Prompt, ' >>>')]
 
 To set the new prompt, assign it to the ``prompts`` attribute of the IPython
 shell:
 
 .. code-block:: python
 
     In [2]: ip = get_ipython()
        ...: ip.prompts = MyPrompt(ip)
 
     /home/bob >>> # it works
 
 See ``IPython/example/utils/cwd_prompt.py`` for an example of how to write an
 extensions to customise prompts.
 
 Inside IPython or in a startup script, you can use a custom prompts class
 by setting ``get_ipython().prompts`` to an *instance* of the class.
 In configuration, ``TerminalInteractiveShell.prompts_class`` may be set to
 either the class object, or a string of its full importable name.
 
 To include invisible terminal control sequences in a prompt, use
 ``Token.ZeroWidthEscape`` as the token type. Tokens with this type are ignored
 when calculating the width.
 
 Colours in the prompt are determined by the token types and the highlighting
 style; see below for more details. The tokens used in the default prompts are
 ``Prompt``, ``PromptNum``, ``OutPrompt`` and ``OutPromptNum``.
 
 .. _termcolour:
 
 Terminal Colors
 ===============
 
 .. versionchanged:: 5.0
 
 There are two main configuration options controlling colours.
 
 ``InteractiveShell.colors`` sets the colour of tracebacks and object info (the
 output from e.g. ``zip?``). It may also affect other things if the option below
 is set to ``'legacy'``. It has four case-insensitive values:
 ``'nocolor', 'neutral', 'linux', 'lightbg'``. The default is *neutral*, which
 should be legible on either dark or light terminal backgrounds. *linux* is
 optimised for dark backgrounds and *lightbg* for light ones.
 
 ``TerminalInteractiveShell.highlighting_style`` determines prompt colours and
 syntax highlighting. It takes the name (as a string) or class (as a subclass of
 ``pygments.style.Style``) of a Pygments style, or the special value ``'legacy'``
 to pick a style in accordance with ``InteractiveShell.colors``.
 
 You can see the Pygments styles available on your system by running::
 
     import pygments
     list(pygments.styles.get_all_styles())
 
 Additionally, ``TerminalInteractiveShell.highlighting_style_overrides`` can override
 specific styles in the highlighting. It should be a dictionary mapping Pygments
 token types to strings defining the style. See `Pygments' documentation
 <http://pygments.org/docs/styles/#creating-own-styles>`__ for the language used
 to define styles.
 
 Colors in the pager
 -------------------
 
 On some systems, the default pager has problems with ANSI colour codes.
 To configure your default pager to allow these:
 
 1. Set the environment PAGER variable to ``less``.
 2. Set the environment LESS variable to ``-r`` (plus any other options
    you always want to pass to less by default). This tells less to
    properly interpret control sequences, which is how color
    information is given to your terminal.
 
 .. _editors:
 
 Editor configuration
 ====================
 
 IPython can integrate with text editors in a number of different ways:
 
 * Editors (such as `(X)Emacs`_, vim_ and TextMate_) can
   send code to IPython for execution.
 
 * IPython's ``%edit`` magic command can open an editor of choice to edit
   a code block.
 
 The %edit command (and its alias %ed) will invoke the editor set in your
 environment as :envvar:`EDITOR`. If this variable is not set, it will default
 to vi under Linux/Unix and to notepad under Windows. You may want to set this
 variable properly and to a lightweight editor which doesn't take too long to
 start (that is, something other than a new instance of Emacs). This way you
 can edit multi-line code quickly and with the power of a real editor right
 inside IPython.
 
 You can also control the editor by setting :attr:`TerminalInteractiveShell.editor`
 in :file:`ipython_config.py`.
 
 Vim
 ---
 
 Paul Ivanov's `vim-ipython <https://github.com/ivanov/vim-ipython>`_ provides
 powerful IPython integration for vim.
 
 .. _emacs:
 
 (X)Emacs
 --------
 
 If you are a dedicated Emacs user, and want to use Emacs when IPython's
 ``%edit`` magic command is called you should set up the Emacs server so that
 new requests are handled by the original process. This means that almost no
 time is spent in handling the request (assuming an Emacs process is already
 running). For this to work, you need to set your EDITOR environment variable
 to 'emacsclient'. The code below, supplied by Francois Pinard, can then be
 used in your :file:`.emacs` file to enable the server:
 
 .. code-block:: common-lisp
 
     (defvar server-buffer-clients)
     (when (and (fboundp 'server-start) (string-equal (getenv "TERM") 'xterm))
       (server-start)
       (defun fp-kill-server-with-buffer-routine ()
         (and server-buffer-clients (server-done)))
       (add-hook 'kill-buffer-hook 'fp-kill-server-with-buffer-routine))
 
 Thanks to the work of Alexander Schmolck and Prabhu Ramachandran,
 currently (X)Emacs and IPython get along very well in other ways.
 
 With (X)EMacs >= 24, You can enable IPython in python-mode with:
 
 .. code-block:: common-lisp
 
     (require 'python)
     (setq python-shell-interpreter "ipython")
 
 .. _`(X)Emacs`: http://www.gnu.org/software/emacs/
 .. _TextMate: http://macromates.com/
 .. _vim: http://www.vim.org/
 
-.. _custom_keyboard_shortcuts
+.. _custom_keyboard_shortcuts:
 
 Keyboard Shortcuts
 ==================
 
 .. versionchanged:: 5.0
 
 You can customise keyboard shortcuts for terminal IPython. Put code like this in
 a :ref:`startup file <startup_files>`::
 
     from IPython import get_ipython
     from prompt_toolkit.enums import DEFAULT_BUFFER
     from prompt_toolkit.keys import Keys
     from prompt_toolkit.filters import HasFocus, HasSelection, ViInsertMode, EmacsInsertMode
 
     ip = get_ipython()
     insert_mode = ViInsertMode() | EmacsInsertMode()
 
     def insert_unexpected(event):
         buf = event.current_buffer
         buf.insert_text('The Spanish Inquisition')
 
     # Register the shortcut if IPython is using prompt_toolkit
     if getattr(ip, 'pt_cli'):
         registry = ip.pt_cli.application.key_bindings_registry
         registry.add_binding(Keys.ControlN,
                          filter=(HasFocus(DEFAULT_BUFFER)
                                  & ~HasSelection()
                                  & insert_mode))(insert_unexpected)
 
 For more information on filters and what you can do with the ``event`` object,
 `see the prompt_toolkit docs
 <http://python-prompt-toolkit.readthedocs.io/en/latest/pages/building_prompts.html#adding-custom-key-bindings>`__.

docs/source/whatsnew/development.rst

@@ -1,175 +1,176 b''
 =====================
  Development version
 =====================
 
 This document describes in-flight development work.
 
 .. warning::
 
     Please do not edit this file by hand (doing so will likely cause merge
     conflicts for other Pull Requests). Instead, create a new file in the
     `docs/source/whatsnew/pr` folder
 
 IPython 6.0
 ===========
 
 Released .... ...., 2017
 
 IPython 6 feature a major improvement in the completion machinery which is now
 capable of completing non-executed code. It is also the first version of IPython
 to stop compatibility with Python 2, which is still supported on the bugfix only
 5.x branch. Read below to have a non-exhaustive list of new features.
 
 Make sure you have pip > 9.0 before upgrading. 
 You should be able to update by using:
 
 .. code::
 
     pip install ipython --upgrade
 
 New completion API and Interface
 --------------------------------
 
 The completer Completion API has seen an overhaul, and the new completer have
 plenty of improvement both from the end users of terminal IPython or for
 consumers of the API.
 
 This new API is capable of pulling completions from :any:`jedi`, thus allowing
 type inference on non-executed code. If :any:`jedi` is installed completion like
 the following are now becoming possible without code evaluation:
 
     >>> data = ['Number of users', 123_456]
     ... data[0].<tab>
 
 That is to say, IPython is now capable of inferring that `data[0]` is a string,
 and will suggest completions like `.capitalize`. The completion power of IPython
 will increase with new Jedi releases, and a number of bugs and more completions
 are already available on development version of :any:`jedi` if you are curious.
 
 With the help of prompt toolkit, types of completions can be shown in the
 completer interface:
 
 .. image:: ../_images/jedi_type_inference_60.png
     :alt: Jedi showing ability to do type inference
     :align: center
+    :width: 400px
     :target: ../_images/jedi_type_inference_60.png
 
 The appearance of the completer is controlled by the
 ``c.TerminalInteractiveShell.display_completions`` option that will show the
 type differently depending on the value among ``'column'``, ``'multicolumn'``
 and ``'readlinelike'``
 
 The use of Jedi also full fill a number of request and fix a number of bugs
 like case insensitive completion, completion after division operator: See
 :ghpull:`10182`.
 
-Extra patches and updates will be needed to the :any:`ipykernel` package for
+Extra patches and updates will be needed to the :mod:`ipykernel` package for
 this feature to be available to other clients like jupyter Notebook, Lab,
 Nteract, Hydrogen...
 
 The use of Jedi can is barely noticeable on recent enough machines, but can be
 feel on older ones,  in cases were Jedi behavior need to be adjusted, the amount
 of time given to Jedi to compute type inference can be adjusted with
 ``c.IPCompleter.jedi_compute_type_timeout``, with object whose type were not
 inferred will be shown as ``<unknown>``. Jedi can also be completely deactivated
 by using the ``c.Completer.use_jedi=False`` option.
 
 
 The old ``Completer.complete()`` API is waiting deprecation and should be
 replaced replaced by ``Completer.completions()`` in a near future. Feedback on
 the current state of the API and suggestions welcome.
 
 Python 3 only codebase
 ----------------------
 
 One of the large challenges in IPython 6.0 has been the adoption of a pure
 Python 3 code base, which lead us to great length to upstream patches in pip,
 pypi and warehouse to make sure Python 2 system still upgrade to the latest
 compatible Python version compatible.
 
 We remind our Python 2 users that IPython 5 is still compatible with Python 2.7,
 still maintained and get regular releases. Using pip 9+, upgrading IPython will
 automatically upgrade to the latest version compatible with your system.
 
 .. warning::
 
   If you are on a system using an older verison of pip on Python 2, pip may
   still install IPython 6.0 on your system, and IPython will refuse to start. 
   You can fix this by ugrading pip, and reinstalling ipython, or forcing pip to
   install an earlier version: ``pip install 'ipython<6'``
 
 The ability to use only Python 3 on the code base of IPython has bring a number
 of advantage. Most of the newly written code make use of `optional function type
 anotation <https://www.python.org/dev/peps/pep-0484/>`_ leading to clearer code
 and better documentation.
 
 The total size of the repository has also for a first time between releases
 (excluding the big split for 4.0) decreased by about 1500 lines, potentially
 quite a bit more codewide as some documents like this one are append only and
 are about 300 lines long.
 
 The removal as of Python2/Python3 shim layer has made the code quite clearer and
 more idiomatic in a number of location, and much friendlier to work with and
 understand. We hope to further embrace Python 3 capability in the next release
 cycle and introduce more of the Python 3 only idioms (yield from, kwarg only,
 general unpacking) in the code base of IPython, and see if we can take advantage
 of these as well to improve user experience with better error messages and
 hints.
 
 
 Miscs improvements
 ------------------
 
 
 - The :cellmagic:`capture` magic can now capture the result of a cell (from an
   expression on the last line), as well as printed and displayed output.
   :ghpull:`9851`.
 
 - Pressing Ctrl-Z in the terminal debugger now suspends IPython, as it already
   does in the main terminal prompt.
 
 - autoreload can now reload ``Enum``. See :ghissue:`10232` and :ghpull:`10316`
 
 - IPython.display has gained a :any:`GeoJSON <IPython.display.GeoJSON>` object.
   :ghpull:`10288` and :ghpull:`10253`
 
 .. DO NOT EDIT THIS LINE BEFORE RELEASE. FEATURE INSERTION POINT.
 
 
 Functions Deprecated in 6.x Development cycle
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 - Loading extensions from ``ipython_extension_dir`` print a warning that this
   location is pending deprecation. This should only affect users still having
   extensions installed with ``%install_ext`` which has been deprecated since
   IPython 4.0, and removed in 5.0. Extensions still present in
   ``ipython_extension_dir`` may shadow more recently installed versions using
   pip. It is thus recommended to clean ``ipython_extension_dir`` of any
   extension now available as a package.
 
 
 - ``IPython.utils.warn`` was deprecated in IPython 4.0, and has now been removed.
   instead of ``IPython.utils.warn`` inbuilt :any:`warnings` module is used.
 
 
 - The function `IPython.core.oinspect.py:call_tip` is unused, was marked as
   Deprecated (raising a Deprecation Warning) and marked for later removal
   :ghpull:`10104`
 
 Backwards incompatible changes
 ------------------------------
 
 Functions Removed in 6.x Development cycle
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 The following functions have been removed in the
 development cycle marked for Milestone 6.0.
 
 - ``IPython/utils/process.py`` - ``is_cmd_found``
 - ``IPython/utils/process.py`` - ``pycmd2argv``
 
 - The `--deep-reload` flag and the corresponding options to inject `dreload` or
   `reload` into the interactive namespace have been removed. You have to
   explicitly import `reload` from `IPython.lib.deepreload` to use it.
 
 .. DO NOT EDIT THIS LINE BEFORE RELEASE. INCOMPAT INSERTION POINT.