Merge pull request #9118 from takluyver/ptshell

Terminal interface based on prompt_toolkit

IPython/terminal/pt_inputhooks/__init__.py

@@ -0,0 +1,16 b''
+import importlib
+import os
+
+aliases = {
+    'qt4': 'qt',
+}
+
+def get_inputhook_func(gui):
+    if gui in aliases:
+        return get_inputhook_func(aliases[gui])
+
+    if gui == 'qt5':
+        os.environ['QT_API'] = 'pyqt5'
+
+    mod = importlib.import_module('IPython.terminal.pt_inputhooks.'+gui)
+    return mod.inputhook

IPython/terminal/pt_inputhooks/glut.py

@@ -0,0 +1,141 b''
+"""GLUT Input hook for interactive use with prompt_toolkit
+"""
+from __future__ import print_function
+
+
+# GLUT is quite an old library and it is difficult to ensure proper
+# integration within IPython since original GLUT does not allow to handle
+# events one by one. Instead, it requires for the mainloop to be entered
+# and never returned (there is not even a function to exit he
+# mainloop). Fortunately, there are alternatives such as freeglut
+# (available for linux and windows) and the OSX implementation gives
+# access to a glutCheckLoop() function that blocks itself until a new
+# event is received. This means we have to setup the idle callback to
+# ensure we got at least one event that will unblock the function.
+#
+# Furthermore, it is not possible to install these handlers without a window
+# being first created. We choose to make this window invisible. This means that
+# display mode options are set at this level and user won't be able to change
+# them later without modifying the code. This should probably be made available
+# via IPython options system.
+
+import sys
+import time
+import signal
+import OpenGL.GLUT as glut
+import OpenGL.platform as platform
+from timeit import default_timer as clock
+
+# Frame per second : 60
+# Should probably be an IPython option
+glut_fps = 60
+
+# Display mode : double buffeed + rgba + depth
+# Should probably be an IPython option
+glut_display_mode = (glut.GLUT_DOUBLE |
+                     glut.GLUT_RGBA   |
+                     glut.GLUT_DEPTH)
+
+glutMainLoopEvent = None
+if sys.platform == 'darwin':
+    try:
+        glutCheckLoop = platform.createBaseFunction(
+            'glutCheckLoop', dll=platform.GLUT, resultType=None,
+            argTypes=[],
+            doc='glutCheckLoop(  ) -> None',
+            argNames=(),
+            )
+    except AttributeError:
+        raise RuntimeError(
+            '''Your glut implementation does not allow interactive sessions'''
+            '''Consider installing freeglut.''')
+    glutMainLoopEvent = glutCheckLoop
+elif glut.HAVE_FREEGLUT:
+    glutMainLoopEvent = glut.glutMainLoopEvent
+else:
+    raise RuntimeError(
+        '''Your glut implementation does not allow interactive sessions. '''
+        '''Consider installing freeglut.''')
+
+
+def glut_display():
+    # Dummy display function
+    pass
+
+def glut_idle():
+    # Dummy idle function
+    pass
+
+def glut_close():
+    # Close function only hides the current window
+    glut.glutHideWindow()
+    glutMainLoopEvent()
+
+def glut_int_handler(signum, frame):
+    # Catch sigint and print the defaultipyt   message
+    signal.signal(signal.SIGINT, signal.default_int_handler)
+    print('\nKeyboardInterrupt')
+    # Need to reprint the prompt at this stage
+
+# Initialisation code
+glut.glutInit( sys.argv )
+glut.glutInitDisplayMode( glut_display_mode )
+# This is specific to freeglut
+if bool(glut.glutSetOption):
+    glut.glutSetOption( glut.GLUT_ACTION_ON_WINDOW_CLOSE,
+                        glut.GLUT_ACTION_GLUTMAINLOOP_RETURNS )
+glut.glutCreateWindow( b'ipython' )
+glut.glutReshapeWindow( 1, 1 )
+glut.glutHideWindow( )
+glut.glutWMCloseFunc( glut_close )
+glut.glutDisplayFunc( glut_display )
+glut.glutIdleFunc( glut_idle )
+
+
+def inputhook(context):
+    """Run the pyglet event loop by processing pending events only.
+
+    This keeps processing pending events until stdin is ready.  After
+    processing all pending events, a call to time.sleep is inserted.  This is
+    needed, otherwise, CPU usage is at 100%.  This sleep time should be tuned
+    though for best performance.
+    """
+    # We need to protect against a user pressing Control-C when IPython is
+    # idle and this is running. We trap KeyboardInterrupt and pass.
+
+    signal.signal(signal.SIGINT, glut_int_handler)
+
+    try:
+        t = clock()
+
+        # Make sure the default window is set after a window has been closed
+        if glut.glutGetWindow() == 0:
+            glut.glutSetWindow( 1 )
+            glutMainLoopEvent()
+            return 0
+
+        while not context.input_is_ready():
+            glutMainLoopEvent()
+            # We need to sleep at this point to keep the idle CPU load
+            # low.  However, if sleep to long, GUI response is poor.  As
+            # a compromise, we watch how often GUI events are being processed
+            # and switch between a short and long sleep time.  Here are some
+            # stats useful in helping to tune this.
+            # time    CPU load
+            # 0.001   13%
+            # 0.005   3%
+            # 0.01    1.5%
+            # 0.05    0.5%
+            used_time = clock() - t
+            if used_time > 10.0:
+                # print 'Sleep for 1 s'  # dbg
+                time.sleep(1.0)
+            elif used_time > 0.1:
+                # Few GUI events coming in, so we can sleep longer
+                # print 'Sleep for 0.05 s'  # dbg
+                time.sleep(0.05)
+            else:
+                # Many GUI events coming in, so sleep only very little
+                time.sleep(0.001)
+    except KeyboardInterrupt:
+        pass

IPython/terminal/pt_inputhooks/gtk.py

@@ -0,0 +1,59 b''
+# Code borrowed from python-prompt-toolkit examples
+# https://github.com/jonathanslenders/python-prompt-toolkit/blob/77cdcfbc7f4b4c34a9d2f9a34d422d7152f16209/examples/inputhook.py
+
+# Copyright (c) 2014, Jonathan Slenders
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification,
+# are permitted provided that the following conditions are met:
+#
+# * Redistributions of source code must retain the above copyright notice, this
+#   list of conditions and the following disclaimer.
+#
+# * Redistributions in binary form must reproduce the above copyright notice, this
+#   list of conditions and the following disclaimer in the documentation and/or
+#   other materials provided with the distribution.
+#
+# * Neither the name of the {organization} nor the names of its
+#   contributors may be used to endorse or promote products derived from
+#   this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""
+PyGTK input hook for prompt_toolkit.
+
+Listens on the pipe prompt_toolkit sets up for a notification that it should
+return control to the terminal event loop.
+"""
+from __future__ import absolute_import
+
+import gtk, gobject
+
+# Enable threading in GTK. (Otherwise, GTK will keep the GIL.)
+gtk.gdk.threads_init()
+
+def inputhook(context):
+    """
+    When the eventloop of prompt-toolkit is idle, call this inputhook.
+
+    This will run the GTK main loop until the file descriptor
+    `context.fileno()` becomes ready.
+
+    :param context: An `InputHookContext` instance.
+    """
+    def _main_quit(*a, **kw):
+        gtk.main_quit()
+        return False
+
+    gobject.io_add_watch(context.fileno(), gobject.IO_IN, _main_quit)
+    gtk.main()

IPython/terminal/pt_inputhooks/gtk3.py

@@ -0,0 +1,12 b''
+"""prompt_toolkit input hook for GTK 3
+"""
+
+from gi.repository import Gtk, GLib
+
+def _main_quit(*args, **kwargs):
+    Gtk.main_quit()
+    return False
+
+def inputhook(context):
+    GLib.io_add_watch(context.fileno(), GLib.IO_IN, _main_quit)
+    Gtk.main()

IPython/terminal/pt_inputhooks/pyglet.py

@@ -0,0 +1,68 b''
+"""Enable pyglet to be used interacively with prompt_toolkit
+"""
+from __future__ import absolute_import
+
+import os
+import sys
+import time
+from timeit import default_timer as clock
+import pyglet
+
+# On linux only, window.flip() has a bug that causes an AttributeError on
+# window close.  For details, see:
+# http://groups.google.com/group/pyglet-users/browse_thread/thread/47c1aab9aa4a3d23/c22f9e819826799e?#c22f9e819826799e
+
+if sys.platform.startswith('linux'):
+    def flip(window):
+        try:
+            window.flip()
+        except AttributeError:
+            pass
+else:
+    def flip(window):
+        window.flip()
+
+
+def inputhook(context):
+    """Run the pyglet event loop by processing pending events only.
+
+    This keeps processing pending events until stdin is ready.  After
+    processing all pending events, a call to time.sleep is inserted.  This is
+    needed, otherwise, CPU usage is at 100%.  This sleep time should be tuned
+    though for best performance.
+    """
+    # We need to protect against a user pressing Control-C when IPython is
+    # idle and this is running. We trap KeyboardInterrupt and pass.
+    try:
+        t = clock()
+        while not context.input_is_ready():
+            pyglet.clock.tick()
+            for window in pyglet.app.windows:
+                window.switch_to()
+                window.dispatch_events()
+                window.dispatch_event('on_draw')
+                flip(window)
+
+            # We need to sleep at this point to keep the idle CPU load
+            # low.  However, if sleep to long, GUI response is poor.  As
+            # a compromise, we watch how often GUI events are being processed
+            # and switch between a short and long sleep time.  Here are some
+            # stats useful in helping to tune this.
+            # time    CPU load
+            # 0.001   13%
+            # 0.005   3%
+            # 0.01    1.5%
+            # 0.05    0.5%
+            used_time = clock() - t
+            if used_time > 10.0:
+                # print 'Sleep for 1 s'  # dbg
+                time.sleep(1.0)
+            elif used_time > 0.1:
+                # Few GUI events coming in, so we can sleep longer
+                # print 'Sleep for 0.05 s'  # dbg
+                time.sleep(0.05)
+            else:
+                # Many GUI events coming in, so sleep only very little
+                time.sleep(0.001)
+    except KeyboardInterrupt:
+        pass

IPython/terminal/pt_inputhooks/qt.py

@@ -0,0 +1,11 b''
+from IPython.external.qt_for_kernel import QtCore, QtGui
+
+def inputhook(context):
+    app = QtCore.QCoreApplication.instance()
+    if not app:
+        return
+    event_loop = QtCore.QEventLoop(app)
+    notifier = QtCore.QSocketNotifier(context.fileno(), QtCore.QSocketNotifier.Read)
+    notifier.setEnabled(True)
+    notifier.activated.connect(event_loop.exit)
+    event_loop.exec_()

IPython/terminal/pt_inputhooks/tk.py

@@ -0,0 +1,93 b''
+# Code borrowed from ptpython
+# https://github.com/jonathanslenders/ptpython/blob/86b71a89626114b18898a0af463978bdb32eeb70/ptpython/eventloop.py
+
+# Copyright (c) 2015, Jonathan Slenders
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification,
+# are permitted provided that the following conditions are met:
+#
+# * Redistributions of source code must retain the above copyright notice, this
+#   list of conditions and the following disclaimer.
+#
+# * Redistributions in binary form must reproduce the above copyright notice, this
+#   list of conditions and the following disclaimer in the documentation and/or
+#   other materials provided with the distribution.
+#
+# * Neither the name of the {organization} nor the names of its
+#   contributors may be used to endorse or promote products derived from
+#   this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+"""
+Wrapper around the eventloop that gives some time to the Tkinter GUI to process
+events when it's loaded and while we are waiting for input at the REPL. This
+way we don't block the UI of for instance ``turtle`` and other Tk libraries.
+
+(Normally Tkinter registeres it's callbacks in ``PyOS_InputHook`` to integrate
+in readline. ``prompt-toolkit`` doesn't understand that input hook, but this
+will fix it for Tk.)
+"""
+import time
+
+import _tkinter
+try:
+    import tkinter
+except ImportError:
+    import Tkinter as tkinter  # Python 2
+
+def inputhook(inputhook_context):
+    """
+    Inputhook for Tk.
+    Run the Tk eventloop until prompt-toolkit needs to process the next input.
+    """
+    # Get the current TK application.
+    root = tkinter._default_root
+
+    def wait_using_filehandler():
+        """
+        Run the TK eventloop until the file handler that we got from the
+        inputhook becomes readable.
+        """
+        # Add a handler that sets the stop flag when `prompt-toolkit` has input
+        # to process.
+        stop = [False]
+        def done(*a):
+            stop[0] = True
+
+        root.createfilehandler(inputhook_context.fileno(), _tkinter.READABLE, done)
+
+        # Run the TK event loop as long as we don't receive input.
+        while root.dooneevent(_tkinter.ALL_EVENTS):
+            if stop[0]:
+                break
+
+        root.deletefilehandler(inputhook_context.fileno())
+
+    def wait_using_polling():
+        """
+        Windows TK doesn't support 'createfilehandler'.
+        So, run the TK eventloop and poll until input is ready.
+        """
+        while not inputhook_context.input_is_ready():
+            while root.dooneevent(_tkinter.ALL_EVENTS | _tkinter.DONT_WAIT):
+                 pass
+            # Sleep to make the CPU idle, but not too long, so that the UI
+            # stays responsive.
+            time.sleep(.01)
+
+    if root is not None:
+        if hasattr(root, 'createfilehandler'):
+            wait_using_filehandler()
+        else:
+            wait_using_polling()

IPython/terminal/pt_inputhooks/wx.py

@@ -0,0 +1,148 b''
+"""Enable wxPython to be used interacively in prompt_toolkit
+"""
+from __future__ import absolute_import
+
+import sys
+import signal
+import time
+from timeit import default_timer as clock
+import wx
+
+
+def inputhook_wx1(context):
+    """Run the wx event loop by processing pending events only.
+
+    This approach seems to work, but its performance is not great as it
+    relies on having PyOS_InputHook called regularly.
+    """
+    try:
+        app = wx.GetApp()
+        if app is not None:
+            assert wx.Thread_IsMain()
+
+            # Make a temporary event loop and process system events until
+            # there are no more waiting, then allow idle events (which
+            # will also deal with pending or posted wx events.)
+            evtloop = wx.EventLoop()
+            ea = wx.EventLoopActivator(evtloop)
+            while evtloop.Pending():
+                evtloop.Dispatch()
+            app.ProcessIdle()
+            del ea
+    except KeyboardInterrupt:
+        pass
+    return 0
+
+class EventLoopTimer(wx.Timer):
+
+    def __init__(self, func):
+        self.func = func
+        wx.Timer.__init__(self)
+
+    def Notify(self):
+        self.func()
+
+class EventLoopRunner(object):
+
+    def Run(self, time, input_is_ready):
+        self.input_is_ready = input_is_ready
+        self.evtloop = wx.EventLoop()
+        self.timer = EventLoopTimer(self.check_stdin)
+        self.timer.Start(time)
+        self.evtloop.Run()
+
+    def check_stdin(self):
+        if self.input_is_ready():
+            self.timer.Stop()
+            self.evtloop.Exit()
+
+def inputhook_wx2(context):
+    """Run the wx event loop, polling for stdin.
+
+    This version runs the wx eventloop for an undetermined amount of time,
+    during which it periodically checks to see if anything is ready on
+    stdin.  If anything is ready on stdin, the event loop exits.
+
+    The argument to elr.Run controls how often the event loop looks at stdin.
+    This determines the responsiveness at the keyboard.  A setting of 1000
+    enables a user to type at most 1 char per second.  I have found that a
+    setting of 10 gives good keyboard response.  We can shorten it further,
+    but eventually performance would suffer from calling select/kbhit too
+    often.
+    """
+    try:
+        app = wx.GetApp()
+        if app is not None:
+            assert wx.Thread_IsMain()
+            elr = EventLoopRunner()
+            # As this time is made shorter, keyboard response improves, but idle
+            # CPU load goes up.  10 ms seems like a good compromise.
+            elr.Run(time=10,  # CHANGE time here to control polling interval
+                    input_is_ready=context.input_is_ready)
+    except KeyboardInterrupt:
+        pass
+    return 0
+
+def inputhook_wx3(context):
+    """Run the wx event loop by processing pending events only.
+
+    This is like inputhook_wx1, but it keeps processing pending events
+    until stdin is ready.  After processing all pending events, a call to
+    time.sleep is inserted.  This is needed, otherwise, CPU usage is at 100%.
+    This sleep time should be tuned though for best performance.
+    """
+    # We need to protect against a user pressing Control-C when IPython is
+    # idle and this is running. We trap KeyboardInterrupt and pass.
+    try:
+        app = wx.GetApp()
+        if app is not None:
+            assert wx.Thread_IsMain()
+
+            # The import of wx on Linux sets the handler for signal.SIGINT
+            # to 0.  This is a bug in wx or gtk.  We fix by just setting it
+            # back to the Python default.
+            if not callable(signal.getsignal(signal.SIGINT)):
+                signal.signal(signal.SIGINT, signal.default_int_handler)
+
+            evtloop = wx.EventLoop()
+            ea = wx.EventLoopActivator(evtloop)
+            t = clock()
+            while not context.input_is_ready():
+                while evtloop.Pending():
+                    t = clock()
+                    evtloop.Dispatch()
+                app.ProcessIdle()
+                # We need to sleep at this point to keep the idle CPU load
+                # low.  However, if sleep to long, GUI response is poor.  As
+                # a compromise, we watch how often GUI events are being processed
+                # and switch between a short and long sleep time.  Here are some
+                # stats useful in helping to tune this.
+                # time    CPU load
+                # 0.001   13%
+                # 0.005   3%
+                # 0.01    1.5%
+                # 0.05    0.5%
+                used_time = clock() - t
+                if used_time > 10.0:
+                    # print 'Sleep for 1 s'  # dbg
+                    time.sleep(1.0)
+                elif used_time > 0.1:
+                    # Few GUI events coming in, so we can sleep longer
+                    # print 'Sleep for 0.05 s'  # dbg
+                    time.sleep(0.05)
+                else:
+                    # Many GUI events coming in, so sleep only very little
+                    time.sleep(0.001)
+            del ea
+    except KeyboardInterrupt:
+        pass
+    return 0
+
+if sys.platform == 'darwin':
+    # On OSX, evtloop.Pending() always returns True, regardless of there being
+    # any events pending. As such we can't use implementations 1 or 3 of the
+    # inputhook as those depend on a pending/dispatch loop.
+    inputhook = inputhook_wx2
+else:
+    # This is our default implementation
+    inputhook = inputhook_wx3

IPython/terminal/ptshell.py

@@ -0,0 +1,240 b''
+"""IPython terminal interface using prompt_toolkit in place of readline"""
+from __future__ import print_function
+
+import sys
+
+from IPython.core.interactiveshell import InteractiveShell
+from IPython.utils.py3compat import PY3, cast_unicode_py2, input
+from traitlets import Bool, Unicode, Dict
+
+from prompt_toolkit.completion import Completer, Completion
+from prompt_toolkit.enums import DEFAULT_BUFFER
+from prompt_toolkit.filters import HasFocus, HasSelection
+from prompt_toolkit.history import InMemoryHistory
+from prompt_toolkit.shortcuts import create_prompt_application, create_eventloop
+from prompt_toolkit.interface import CommandLineInterface
+from prompt_toolkit.key_binding.manager import KeyBindingManager
+from prompt_toolkit.key_binding.vi_state import InputMode
+from prompt_toolkit.key_binding.bindings.vi import ViStateFilter
+from prompt_toolkit.keys import Keys
+from prompt_toolkit.layout.lexers import PygmentsLexer
+from prompt_toolkit.styles import PygmentsStyle
+
+from pygments.styles import get_style_by_name
+from pygments.lexers import Python3Lexer, PythonLexer
+from pygments.token import Token
+
+from .pt_inputhooks import get_inputhook_func
+from .interactiveshell import get_default_editor
+
+
+class IPythonPTCompleter(Completer):
+    """Adaptor to provide IPython completions to prompt_toolkit"""
+    def __init__(self, ipy_completer):
+        self.ipy_completer = ipy_completer
+
+    def get_completions(self, document, complete_event):
+        if not document.current_line.strip():
+            return
+
+        used, matches = self.ipy_completer.complete(
+                            line_buffer=document.current_line,
+                            cursor_pos=document.cursor_position_col
+        )
+        start_pos = -len(used)
+        for m in matches:
+            yield Completion(m, start_position=start_pos)
+
+class PTInteractiveShell(InteractiveShell):
+    colors_force = True
+
+    pt_cli = None
+
+    vi_mode = Bool(False, config=True,
+        help="Use vi style keybindings at the prompt",
+    )
+
+    mouse_support = Bool(False, config=True,
+        help="Enable mouse support in the prompt"
+    )
+
+    highlighting_style = Unicode('', config=True,
+        help="The name of a Pygments style to use for syntax highlighting"
+    )
+
+    highlighting_style_overrides = Dict(config=True,
+        help="Override highlighting format for specific tokens"
+    )
+
+    editor = Unicode(get_default_editor(), config=True,
+        help="Set the editor used by IPython (default to $EDITOR/vi/notepad)."
+    )
+
+    def get_prompt_tokens(self, cli):
+        return [
+            (Token.Prompt, 'In ['),
+            (Token.PromptNum, str(self.execution_count)),
+            (Token.Prompt, ']: '),
+        ]
+
+    def get_continuation_tokens(self, cli, width):
+        return [
+            (Token.Prompt, (' ' * (width - 2)) + ': '),
+        ]
+
+    def init_prompt_toolkit_cli(self):
+        if not sys.stdin.isatty():
+            # Piped input - e.g. for tests. Fall back to plain non-interactive
+            # output. This is very limited, and only accepts a single line.
+            def prompt():
+                return cast_unicode_py2(input('In [%d]: ' % self.execution_count))
+            self.prompt_for_code = prompt
+            return
+
+        kbmanager = KeyBindingManager.for_prompt(enable_vi_mode=self.vi_mode)
+        insert_mode = ViStateFilter(kbmanager.get_vi_state, InputMode.INSERT)
+        # Ctrl+J == Enter, seemingly
+        @kbmanager.registry.add_binding(Keys.ControlJ,
+                            filter=(HasFocus(DEFAULT_BUFFER)
+                                    & ~HasSelection()
+                                    & insert_mode
+                                   ))
+        def _(event):
+            b = event.current_buffer
+            d = b.document
+            if not (d.on_last_line or d.cursor_position_row >= d.line_count
+                                           - d.empty_line_count_at_the_end()):
+                b.newline()
+                return
+
+            status, indent = self.input_splitter.check_complete(d.text)
+
+            if (status != 'incomplete') and b.accept_action.is_returnable:
+                b.accept_action.validate_and_handle(event.cli, b)
+            else:
+                b.insert_text('\n' + (' ' * (indent or 0)))
+
+        @kbmanager.registry.add_binding(Keys.ControlC)
+        def _(event):
+            event.current_buffer.reset()
+
+        # Pre-populate history from IPython's history database
+        history = InMemoryHistory()
+        last_cell = u""
+        for _, _, cell in self.history_manager.get_tail(self.history_load_length,
+                                                        include_latest=True):
+            # Ignore blank lines and consecutive duplicates
+            cell = cell.rstrip()
+            if cell and (cell != last_cell):
+                history.append(cell)
+
+        style_overrides = {
+            Token.Prompt: '#009900',
+            Token.PromptNum: '#00ff00 bold',
+        }
+        if self.highlighting_style:
+            style_cls = get_style_by_name(self.highlighting_style)
+        else:
+            style_cls = get_style_by_name('default')
+            # The default theme needs to be visible on both a dark background
+            # and a light background, because we can't tell what the terminal
+            # looks like. These tweaks to the default theme help with that.
+            style_overrides.update({
+                Token.Number: '#007700',
+                Token.Operator: 'noinherit',
+                Token.String: '#BB6622',
+                Token.Name.Function: '#2080D0',
+                Token.Name.Class: 'bold #2080D0',
+                Token.Name.Namespace: 'bold #2080D0',
+            })
+        style_overrides.update(self.highlighting_style_overrides)
+        style = PygmentsStyle.from_defaults(pygments_style_cls=style_cls,
+                                            style_dict=style_overrides)
+
+        app = create_prompt_application(multiline=True,
+                            lexer=PygmentsLexer(Python3Lexer if PY3 else PythonLexer),
+                            get_prompt_tokens=self.get_prompt_tokens,
+                            # The line below is waiting for a new release of
+                            # prompt_toolkit (> 0.57)
+                            #get_continuation_tokens=self.get_continuation_tokens,
+                            key_bindings_registry=kbmanager.registry,
+                            history=history,
+                            completer=IPythonPTCompleter(self.Completer),
+                            enable_history_search=True,
+                            style=style,
+                            mouse_support=self.mouse_support,
+        )
+
+        self.pt_cli = CommandLineInterface(app,
+                           eventloop=create_eventloop(self.inputhook))
+
+    def prompt_for_code(self):
+        document = self.pt_cli.run(pre_run=self.pre_prompt)
+        return document.text
+
+    def init_io(self):
+        if sys.platform not in {'win32', 'cli'}:
+            return
+
+        import colorama
+        colorama.init()
+
+        # For some reason we make these wrappers around stdout/stderr.
+        # For now, we need to reset them so all output gets coloured.
+        # https://github.com/ipython/ipython/issues/8669
+        from IPython.utils import io
+        io.stdout = io.IOStream(sys.stdout)
+        io.stderr = io.IOStream(sys.stderr)
+
+    def __init__(self, *args, **kwargs):
+        super(PTInteractiveShell, self).__init__(*args, **kwargs)
+        self.init_prompt_toolkit_cli()
+        self.keep_running = True
+
+    def ask_exit(self):
+        self.keep_running = False
+
+    rl_next_input = None
+
+    def pre_prompt(self):
+        if self.rl_next_input:
+            self.pt_cli.application.buffer.text = cast_unicode_py2(self.rl_next_input)
+            self.rl_next_input = None
+
+    def interact(self):
+        while self.keep_running:
+            print(self.separate_in, end='')
+
+            try:
+                code = self.prompt_for_code()
+            except EOFError:
+                if self.ask_yes_no('Do you really want to exit ([y]/n)?','y','n'):
+                    self.ask_exit()
+
+            else:
+                if code:
+                    self.run_cell(code, store_history=True)
+
+    def mainloop(self):
+        # An extra layer of protection in case someone mashing Ctrl-C breaks
+        # out of our internal code.
+        while True:
+            try:
+                self.interact()
+                break
+            except KeyboardInterrupt:
+                print("\nKeyboardInterrupt escaped interact()\n")
+
+    _inputhook = None
+    def inputhook(self, context):
+        if self._inputhook is not None:
+            self._inputhook(context)
+
+    def enable_gui(self, gui=None):
+        if gui:
+            self._inputhook = get_inputhook_func(gui)
+        else:
+            self._inputhook = None
+
+if __name__ == '__main__':
+    PTInteractiveShell.instance().interact()

IPython/core/completer.py

@@ -170,41 +170,38 b' def compress_user(path, tilde_expand, tilde_val):'
 
 
 
-def penalize_magics_key(word):
-    """key for sorting that penalizes magic commands in the ordering
+def completions_sorting_key(word):
+    """key for sorting completions
 
-    Normal words are left alone.
-
-    Magic commands have the initial % moved to the end, e.g.
-    %matplotlib is transformed as follows:
-
-    %matplotlib -> matplotlib%
-
-    [The choice of the final % is arbitrary.]
-
-    Since "matplotlib" < "matplotlib%" as strings, 
-    "timeit" will appear before the magic "%timeit" in the ordering
-
-    For consistency, move "%%" to the end, so cell magics appear *after*
-    line magics with the same name.
-
-    A check is performed that there are no other "%" in the string; 
-    if there are, then the string is not a magic command and is left unchanged.
+    This does several things:
 
+    - Lowercase all completions, so they are sorted alphabetically with
+      upper and lower case words mingled
+    - Demote any completions starting with underscores to the end
+    - Insert any %magic and %%cellmagic completions in the alphabetical order
+      by their name
     """
+    # Case insensitive sort
+    word = word.lower()
 
-    # Move any % signs from start to end of the key 
-    # provided there are no others elsewhere in the string
+    prio1, prio2 = 0, 0
 
-    if word[:2] == "%%":
-        if not "%" in word[2:]:
-            return word[2:] + "%%" 
+    if word.startswith('__'):
+        prio1 = 2
+    elif word.startswith('_'):
+        prio1 = 1
 
-    if word[:1] == "%":
+    if word.startswith('%%'):
+        # If there's another % in there, this is something else, so leave it alone
+        if not "%" in word[2:]:
+            word = word[2:]
+            prio2 = 2
+    elif word.startswith('%'):
         if not "%" in word[1:]:
-            return word[1:] + "%"
-    
-    return word
+            word = word[1:]
+            prio2 = 1
+
+    return prio1, word, prio2
 
 
 @undoc
@@ -1206,8 +1203,7 b' class IPCompleter(Completer):'
         # simply collapse the dict into a list for readline, but we'd have
         # richer completion semantics in other evironments.
 
-        # use penalize_magics_key to put magics after variables with same name
-        self.matches = sorted(set(self.matches), key=penalize_magics_key)
+        self.matches = sorted(set(self.matches), key=completions_sorting_key)
 
         #io.rprint('COMP TEXT, MATCHES: %r, %r' % (text, self.matches)) # dbg
         return text, self.matches

IPython/core/tests/test_shellapp.py

@@ -52,8 +52,9 b' class TestFileToRun(unittest.TestCase, tt.TempFileMixin):'
         src = "True\n"
         self.mktmp(src)
 
+        out = 'In [1]: False\n\nIn [2]:'
         err = SQLITE_NOT_AVAILABLE_ERROR if sqlite_err_maybe else None
-        tt.ipexec_validate(self.fname, 'False', err, options=['-i'],
+        tt.ipexec_validate(self.fname, out, err, options=['-i'],
                            commands=['"__file__" in globals()', 'exit()'])
 
     @dec.skip_win32
@@ -63,6 +64,7 b' class TestFileToRun(unittest.TestCase, tt.TempFileMixin):'
         src = "from __future__ import division\n"
         self.mktmp(src)
 
+        out = 'In [1]: float\n\nIn [2]:'
         err = SQLITE_NOT_AVAILABLE_ERROR if sqlite_err_maybe else None
-        tt.ipexec_validate(self.fname, 'float', err, options=['-i'],
+        tt.ipexec_validate(self.fname, out, err, options=['-i'],
                            commands=['type(1/2)', 'exit()'])

IPython/terminal/ipapp.py

@@ -32,7 +32,7 b' from IPython.core.shellapp import ('
     InteractiveShellApp, shell_flags, shell_aliases
 )
 from IPython.extensions.storemagic import StoreMagics
-from IPython.terminal.interactiveshell import TerminalInteractiveShell
+from .ptshell import PTInteractiveShell as TerminalInteractiveShell
 from IPython.utils import warn
 from IPython.paths import get_ipython_dir
 from traitlets import (

examples/IPython Kernel/gui/gui-glut.py

@@ -38,7 +38,7 b' if glut.glutGetWindow() > 0:'
 else:
     interactive = False
 
-glut.glutCreateWindow('gui-glut')
+glut.glutCreateWindow(b'gui-glut')
 glut.glutDisplayFunc(display)
 glut.glutReshapeFunc(resize)
 # This is necessary on osx to be able to close the window

examples/IPython Kernel/gui/gui-gtk3.py

@@ -20,10 +20,10 b' def delete_event(widget, event, data=None):'
 def destroy(widget, data=None):
     Gtk.main_quit()
 
-window = Gtk.Window(Gtk.WindowType.TOPLEVEL)
+window = Gtk.Window(type=Gtk.WindowType.TOPLEVEL)
 window.connect("delete_event", delete_event)
 window.connect("destroy", destroy)
-button = Gtk.Button("Hello World")
+button = Gtk.Button(label="Hello World")
 button.connect("clicked", hello_world, None)
 
 window.add(button)

setup.py

@@ -195,6 +195,7 b' install_requires = ['
     'pickleshare',
     'simplegeneric>0.8',
     'traitlets',
+    'prompt_toolkit',  # We will require > 0.57 once a new release is made
 ]
 
 # Platform-specific dependencies:
@@ -204,8 +205,6 b' install_requires = ['
 extras_require.update({
     ':sys_platform != "win32"': ['pexpect'],
     ':sys_platform == "darwin"': ['appnope'],
-    ':sys_platform == "darwin" and platform_python_implementation == "CPython"': ['gnureadline'],
-    'terminal:sys_platform == "win32"': ['pyreadline>=2'],
     'test:python_version == "2.7"': ['mock'],
 })
 # FIXME: re-specify above platform dependencies for pip < 6