Merge branch 'master' into inputtransformer2

LICENSE

@@ -0,0 +1,33 b''
+BSD 3-Clause License
+
+- Copyright (c) 2008-Present, IPython Development Team
+- Copyright (c) 2001-2007, Fernando Perez <[e-mail]>
+- Copyright (c) 2001, Janko Hauser <[e-mail]>
+- Copyright (c) 2001, Nathaniel Gray <[e-mail]>
+
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice, this
+  list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright notice,
+  this list of conditions and the following disclaimer in the documentation
+  and/or other materials provided with the distribution.
+
+* Neither the name of the copyright holder nor the names of its
+  contributors may be used to endorse or promote products derived from
+  this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

docs/source/whatsnew/pr/deprecations.rst

@@ -0,0 +1,7 b''
+A couple of unused function and methods have been deprecated and will be removed
+in future versions:
+
+  - ``IPython.utils.io.raw_print_err``
+  - ``IPython.utils.io.raw_print``
+
+

.travis.yml

@@ -2,10 +2,10 b''
 language: python
 python:
     - "nightly"
+    - "3.7-dev"
     - 3.6
     - 3.5
     - 3.4
-    - 3.3
 sudo: false
 env:
   global:
@@ -16,9 +16,10 b' before_install:'
 install:
     - pip install setuptools pip --upgrade
     - pip install -e file://$PWD#egg=ipython[test] --upgrade
-    - pip install codecov --upgrade
+    - pip install codecov check-manifest --upgrade
     - sudo apt-get install graphviz
 script:
+    - check-manifest
     - cd /tmp && iptest --coverage xml && cd -
     # On the latest Python only, make sure that the docs build.
     - |

COPYING.rst

@@ -3,39 +3,8 b''
 =============================
 
 IPython is licensed under the terms of the Modified BSD License (also known as
-New or Revised or 3-Clause BSD), as follows:
+New or Revised or 3-Clause BSD). See the LICENSE file.
 
-- Copyright (c) 2008-2014, IPython Development Team
-- Copyright (c) 2001-2007, Fernando Perez <[e-mail]>
-- Copyright (c) 2001, Janko Hauser <[e-mail]>
-- Copyright (c) 2001, Nathaniel Gray <[e-mail]>
-
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
-Redistributions of source code must retain the above copyright notice, this
-list of conditions and the following disclaimer.
-
-Redistributions in binary form must reproduce the above copyright notice, this
-list of conditions and the following disclaimer in the documentation and/or
-other materials provided with the distribution.
-
-Neither the name of the IPython Development Team nor the names of its
-contributors may be used to endorse or promote products derived from this
-software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
-ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
-FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
-SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
-CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
-OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 About the IPython Development Team
 ----------------------------------
@@ -45,9 +14,7 b' Fernando Perez began IPython in 2001 based on code from Janko Hauser'
 the project lead.
 
 The IPython Development Team is the set of all contributors to the IPython
-project.  This includes all of the IPython subprojects. A full list with
-details is kept in the documentation directory, in the file
-``about/credits.txt``.
+project.  This includes all of the IPython subprojects. 
 
 The core team that coordinates development on GitHub can be found here:
 https://github.com/ipython/.

IPython/__init__.py

@@ -27,12 +27,12 b' import sys'
 #-----------------------------------------------------------------------------
 
 # Don't forget to also update setup.py when this changes!
-if sys.version_info < (3,3):
+if sys.version_info < (3,4):
     raise ImportError(
 """
-IPython 6.0+ does not support Python 2.6, 2.7, 3.0, 3.1, or 3.2.
+IPython 7.0+ supports Python 3.4 and above.
 When using Python 2.7, please install IPython 5.x LTS Long Term Support version.
-Beginning with IPython 6.0, Python 3.3 and above is required.
+Python 3.3 was supported up to IPython 6.x.
 
 See IPython `README.rst` file for more information:
 

IPython/consoleapp.py

@@ -6,7 +6,7 b' Shim to maintain backwards compatibility with old IPython.consoleapp imports.'
 
 from warnings import warn
 
-warn("The `IPython.consoleapp` package has been deprecated. "
-     "You should import from jupyter_client.consoleapp instead.")
+warn("The `IPython.consoleapp` package has been deprecated since IPython 4.0."
+     "You should import from jupyter_client.consoleapp instead.", stacklevel=2)
 
 from jupyter_client.consoleapp import *

IPython/core/compilerop.py

@@ -90,13 +90,31 b' class CachingCompiler(codeop.Compile):'
         # stdlib that call it outside our control go through our codepath
         # (otherwise we'd lose our tracebacks).
         linecache.checkcache = check_linecache_ipython
+
+
+    def _fix_module_ds(self, module):
+        """
+        Starting in python 3.7 the AST for mule have changed, and if
+        the first expressions encountered is a string it is attached to the 
+        `docstring` attribute of the `Module` ast node.
+
+        This breaks IPython, as if this string is the only expression, IPython
+        will not return it as the result of the current cell.
+        """
+        from ast import Str, Expr, Module, fix_missing_locations
+        docstring = getattr(module, 'docstring', None)
+        if not docstring:
+            return module
+        new_body=[Expr(Str(docstring, lineno=1, col_offset=0), lineno=1, col_offset=0)]
+        new_body.extend(module.body)
+        return fix_missing_locations(Module(new_body))
         
     def ast_parse(self, source, filename='<unknown>', symbol='exec'):
         """Parse code to an AST with the current compiler flags active.
         
         Arguments are exactly the same as ast.parse (in the standard library),
         and are passed to the built-in compile function."""
-        return compile(source, filename, symbol, self.flags | PyCF_ONLY_AST, 1)
+        return self._fix_module_ds(compile(source, filename, symbol, self.flags | PyCF_ONLY_AST, 1))
     
     def reset_compiler_flags(self):
         """Reset compiler flags to default state."""

IPython/core/completer.py

@@ -576,9 +576,9 b' class Completer(Configurable):'
         """
     ).tag(config=True)
 
-    use_jedi = Bool(default_value=JEDI_INSTALLED,
+    use_jedi = Bool(default_value=False,
                     help="Experimental: Use Jedi to generate autocompletions. "
-                    "Default to True if jedi is installed").tag(config=True)
+                    "Off by default.").tag(config=True)
 
     jedi_compute_type_timeout = Int(default_value=400,
         help="""Experimental: restrict time (in milliseconds) during which Jedi can compute types.
@@ -683,7 +683,7 b' class Completer(Configurable):'
         Assuming the text is of the form NAME.NAME....[NAME], and is
         evaluatable in self.namespace or self.global_namespace, it will be
         evaluated and its attributes (as revealed by dir()) are used as
-        possible completions.  (For class instances, class members are are
+        possible completions.  (For class instances, class members are
         also considered.)
 
         WARNING: this can still invoke arbitrary C code, if an object
@@ -869,7 +869,7 b' def position_to_cursor(text:str, offset:int)->Tuple[int, int]:'
 
     """
 
-    assert 0 < offset <= len(text) , "0 < %s <= %s" % (offset , len(text))
+    assert 0 <= offset <= len(text) , "0 <= %s <= %s" % (offset , len(text))
 
     before = text[:offset]
     blines = before.split('\n')  # ! splitnes trim trailing \n
@@ -949,8 +949,8 b' def _formatparamchildren(parameter) -> str:'
 
     Jedi does not expose a simple way to get `param=value` from its API.
 
-    Prameter
-    ========
+    Parameter
+    =========
 
     parameter:
         Jedi's function `Param`
@@ -1340,7 +1340,6 b' class IPCompleter(Completer):'
             namespaces.append(self.global_namespace)
 
         completion_filter = lambda x:x
-        # cursor_pos is an it, jedi wants line and column
         offset = cursor_to_position(text, cursor_line, cursor_column)
         # filter output if we are completing for object members
         if offset:
@@ -1356,7 +1355,7 b' class IPCompleter(Completer):'
                     raise ValueError("Don't understand self.omit__names == {}".format(self.omit__names))
 
         interpreter = jedi.Interpreter(
-            text, namespaces, column=cursor_column, line=cursor_line + 1)
+            text[:offset], namespaces, column=cursor_column, line=cursor_line + 1)
         try_jedi = True
 
         try:
@@ -1371,7 +1370,7 b' class IPCompleter(Completer):'
             next_to_last_tree = interpreter._get_module().tree_node.children[-2]
             completing_string = False
             if isinstance(next_to_last_tree, ErrorLeaf):
-                completing_string = next_to_last_tree.value[0] in {'"', "'"}
+                completing_string = next_to_last_tree.value.lstrip()[0] in {'"', "'"}
             # if we are in a string jedi is likely not the right candidate for
             # now. Skip it.
             try_jedi = not completing_string
@@ -1805,11 +1804,16 b' class IPCompleter(Completer):'
                       category=ProvisionalCompleterWarning, stacklevel=2)
 
         seen = set()
-        for c in self._completions(text, offset, _timeout=self.jedi_compute_type_timeout/1000):
-            if c and (c in seen):
-                continue
-            yield c
-            seen.add(c)
+        try:
+            for c in self._completions(text, offset, _timeout=self.jedi_compute_type_timeout/1000):
+                if c and (c in seen):
+                    continue
+                yield c
+                seen.add(c)
+        except KeyboardInterrupt:
+            """if completions take too long and users send keyboard interrupt,
+            do not crash and return ASAP. """
+            pass
 
     def _completions(self, full_text: str, offset: int, *, _timeout)->Iterator[Completion]:
         """

IPython/core/display.py

@@ -667,7 +667,7 b' class Pretty(TextDisplayObject):'
 class HTML(TextDisplayObject):
 
     def _repr_html_(self):
-        return self.data
+        return self._data_and_metadata()
 
     def __html__(self):
         """
@@ -681,20 +681,23 b' class HTML(TextDisplayObject):'
 class Markdown(TextDisplayObject):
 
     def _repr_markdown_(self):
-        return self.data
+        return self._data_and_metadata()
 
 
 class Math(TextDisplayObject):
 
     def _repr_latex_(self):
-        s = self.data.strip('$')
-        return "$$%s$$" % s
+        s = "$$%s$$" % self.data.strip('$')
+        if self.metadata:
+            return s, deepcopy(self.metadata)
+        else:
+            return s
 
 
 class Latex(TextDisplayObject):
 
     def _repr_latex_(self):
-        return self.data
+        return self._data_and_metadata()
 
 
 class SVG(DisplayObject):

IPython/core/displayhook.py

@@ -317,4 +317,4 b' class CapturingDisplayHook(object):'
         if result is None:
             return
         format_dict, md_dict = self.shell.display_formatter.format(result)
-        self.outputs.append((format_dict, md_dict))
+        self.outputs.append({ 'data': format_dict, 'metadata': md_dict })

IPython/core/extensions.py

@@ -7,18 +7,13 b''
 import os
 import os.path
 import sys
-from importlib import import_module
+from importlib import import_module, reload
 
 from traitlets.config.configurable import Configurable
 from IPython.utils.path import ensure_dir_exists, compress_user
 from IPython.utils.decorators import undoc
 from traitlets import Instance
 
-try:
-    from importlib import reload
-except ImportError :
-    ## deprecated since 3.4
-    from imp import reload
 
 #-----------------------------------------------------------------------------
 # Main class

IPython/core/history.py

@@ -20,7 +20,7 b' import threading'
 from traitlets.config.configurable import LoggingConfigurable
 from decorator import decorator
 from IPython.utils.decorators import undoc
-from IPython.utils.path import locate_profile
+from IPython.paths import locate_profile
 from traitlets import (
     Any, Bool, Dict, Instance, Integer, List, Unicode, TraitError,
     default, observe,

IPython/core/interactiveshell.py

@@ -170,7 +170,7 b' class SeparateUnicode(Unicode):'
 class DummyMod(object):
     """A dummy module used for IPython's interactive module when
     a namespace must be assigned to the module's __dict__."""
-    pass
+    __spec__ = None
 
 
 class ExecutionInfo(object):
@@ -748,7 +748,7 b' class InteractiveShell(SingletonConfigurable):'
 
         # executable path should end like /bin/python or \\scripts\\python.exe
         p_exe_up2 = os.path.dirname(os.path.dirname(p))
-        if p_exe_up2 and os.path.samefile(p_exe_up2, p_venv):
+        if p_exe_up2 and os.path.exists(p_venv) and os.path.samefile(p_exe_up2, p_venv):
             # Our exe is inside the virtualenv, don't need to do anything.
             return
 
@@ -1190,7 +1190,7 b' class InteractiveShell(SingletonConfigurable):'
         -----
         All data structures here are only filled in, they are NOT reset by this
         method.  If they were not empty before, data will simply be added to
-        therm.
+        them.
         """
         # This function works in two parts: first we put a few things in
         # user_ns, and we sync that contents into user_ns_hidden so that these
@@ -2884,7 +2884,6 b' class InteractiveShell(SingletonConfigurable):'
         """
         if not nodelist:
             return
-
         if interactivity == 'last_expr_or_assign':
             if isinstance(nodelist[-1], _assign_nodes):
                 asg = nodelist[-1]
@@ -2914,7 +2913,6 b' class InteractiveShell(SingletonConfigurable):'
             to_run_exec, to_run_interactive = [], nodelist
         else:
             raise ValueError("Interactivity was %r" % interactivity)
-
         try:
             for i, node in enumerate(to_run_exec):
                 mod = ast.Module([node])

IPython/core/magics/basic.py

@@ -302,21 +302,6 b' Currently the magic system has the following functions:""",'
             print('Object `%s` not found' % oname)
 
     @line_magic
-    def profile(self, parameter_s=''):
-        """DEPRECATED since IPython 2.0.
-
-        Raise `UsageError`. To profile code use the :magic:`prun` magic.
-        
-
-        See Also
-        --------
-        prun : run code using the Python profiler (:magic:`prun`)
-        """
-        raise UsageError("The `%profile` magic has been deprecated since IPython 2.0. "
-            "and removed in IPython 6.0. Please use the value of `get_ipython().profile` instead "
-            "to see current profile in use. Perhaps you meant to use `%prun` to profile code?")
-
-    @line_magic
     def pprint(self, parameter_s=''):
         """Toggle pretty printing on/off."""
         ptformatter = self.shell.display_formatter.formatters['text/plain']

IPython/core/magics/code.py

@@ -20,6 +20,8 b' import re'
 import sys
 import ast
 from itertools import chain
+from urllib.request import urlopen
+from urllib.parse import urlencode
 
 # Our own packages
 from IPython.core.error import TryNext, StdinNotImplementedError, UsageError
@@ -244,7 +246,7 b' class CodeMagics(Magics):'
 
     @line_magic
     def pastebin(self, parameter_s=''):
-        """Upload code to Github's Gist paste bin, returning the URL.
+        """Upload code to dpaste's paste bin, returning the URL.
 
         Usage:\\
           %pastebin [-d "Custom description"] 1-7
@@ -265,25 +267,14 b' class CodeMagics(Magics):'
             print(e.args[0])
             return
 
-        # Deferred import
-        try:
-            from urllib.request import urlopen # Py 3
-        except ImportError:
-            from urllib2 import urlopen
-        import json
-        post_data = json.dumps({
-          "description": opts.get('d', "Pasted from IPython"),
-          "public": True,
-          "files": {
-            "file1.py": {
-              "content": code
-            }
-          }
+        post_data = urlencode({
+          "title": opts.get('d', "Pasted from IPython"),
+          "syntax": "python3",
+          "content": code
         }).encode('utf-8')
 
-        response = urlopen("https://api.github.com/gists", post_data)
-        response_data = json.loads(response.read().decode('utf-8'))
-        return response_data['html_url']
+        response = urlopen("http://dpaste.com/api/v2/", post_data)
+        return response.headers.get('Location')
 
     @line_magic
     def loadpy(self, arg_s):

IPython/core/magics/execution.py

@@ -110,7 +110,6 b' class TimeitResult(object):'
         p.text(u'<TimeitResult : '+unic+u'>')
 
 
-
 class TimeitTemplateFiller(ast.NodeTransformer):
     """Fill in the AST template for timing execution.
 

IPython/core/magics/script.py

@@ -31,14 +31,14 b' def script_args(f):'
             '--out', type=str,
             help="""The variable in which to store stdout from the script.
             If the script is backgrounded, this will be the stdout *pipe*,
-            instead of the stderr text itself.
+            instead of the stderr text itself and will not be auto closed.
             """
         ),
         magic_arguments.argument(
             '--err', type=str,
             help="""The variable in which to store stderr from the script.
             If the script is backgrounded, this will be the stderr *pipe*,
-            instead of the stderr text itself.
+            instead of the stderr text itself and will not be autoclosed.
             """
         ),
         magic_arguments.argument(
@@ -187,11 +187,16 b' class ScriptMagics(Magics):'
         if args.bg:
             self.bg_processes.append(p)
             self._gc_bg_processes()
+            to_close = []
             if args.out:
                 self.shell.user_ns[args.out] = p.stdout
+            else:
+                to_close.append(p.stdout)
             if args.err:
                 self.shell.user_ns[args.err] = p.stderr
-            self.job_manager.new(self._run_script, p, cell, daemon=True)
+            else:
+                to_close.append(p.stderr)
+            self.job_manager.new(self._run_script, p, cell, to_close, daemon=True)
             if args.proc:
                 self.shell.user_ns[args.proc] = p
             return
@@ -231,10 +236,12 b' class ScriptMagics(Magics):'
             sys.stderr.write(err)
             sys.stderr.flush()
     
-    def _run_script(self, p, cell):
+    def _run_script(self, p, cell, to_close):
         """callback for running the script in the background"""
         p.stdin.write(cell)
         p.stdin.close()
+        for s in to_close:
+            s.close()
         p.wait()
 
     @line_magic("killbgscripts")

IPython/core/pylabtools.py

@@ -46,6 +46,9 b" backend2gui['CocoaAgg'] = 'osx'"
 # And some backends that don't need GUI integration
 del backend2gui['nbAgg']
 del backend2gui['agg']
+del backend2gui['svg']
+del backend2gui['pdf']
+del backend2gui['ps']
 del backend2gui['module://ipykernel.pylab.backend_inline']
 
 #-----------------------------------------------------------------------------

IPython/core/release.py

@@ -19,12 +19,12 b" name = 'ipython'"
 # IPython version information.  An empty _version_extra corresponds to a full
 # release.  'dev' as a _version_extra string means this is a development
 # version
-_version_major = 6
-_version_minor = 2
+_version_major = 7
+_version_minor = 0
 _version_patch = 0
 _version_extra = '.dev'
 # _version_extra = 'rc2'
-#_version_extra = ''  # Uncomment this for full releases
+# _version_extra = ''  # Uncomment this for full releases
 
 # Construct full version string from these.
 _ver = [_version_major, _version_minor, _version_patch]

IPython/core/tests/test_autocall.py

@@ -14,7 +14,6 b' from IPython.testing.globalipapp import get_ipython'
 ip = get_ipython()
 
 
[e-mail]_refactor_print
 def doctest_autocall():
     """
     In [1]: def f1(a,b,c):
@@ -39,7 +38,7 b' def doctest_autocall():'
 
     In [7]: assert _ == 'abc'
 
-    In [8]: print _
+    In [8]: print(_)
     abc
 
     In [9]: /f1 1,2,3

IPython/core/tests/test_completer.py

@@ -316,13 +316,17 b' def test_jedi():'
         start = start if start is not None else l
         end = end if end is not None else l
         with provisionalcompleter():
+            ip.Completer.use_jedi = True
             completions = set(ip.Completer.completions(s, l))
+            ip.Completer.use_jedi = False
             assert_in(Completion(start, end, comp), completions, reason)
 
     def _test_not_complete(reason, s, comp):
         l = len(s)
         with provisionalcompleter():
+            ip.Completer.use_jedi = True
             completions = set(ip.Completer.completions(s, l))
+            ip.Completer.use_jedi = False
             assert_not_in(Completion(l, l, comp), completions, reason)
 
     import jedi
@@ -341,8 +345,10 b' def test_completion_have_signature():'
     """
     ip = get_ipython()
     with provisionalcompleter():
+        ip.Completer.use_jedi = True
         completions = ip.Completer.completions('ope', 3)
         c = next(completions)  # should be `open`
+        ip.Completer.use_jedi = False
     assert 'file' in c.signature, "Signature of function was not found by completer"
     assert 'encoding' in c.signature, "Signature of function was not found by completer"
 
@@ -357,7 +363,9 b' def test_deduplicate_completions():'
         zoo = 1
     '''))
     with provisionalcompleter():
+        ip.Completer.use_jedi = True
         l = list(_deduplicate_completions('Z.z', ip.Completer.completions('Z.z', 3)))
+        ip.Completer.use_jedi = False
 
     assert len(l) == 1, 'Completions (Z.z<tab>) correctly deduplicate: %s ' % l
     assert l[0].text == 'zoo'  # and not `it.accumulate`
@@ -412,10 +420,10 b' def test_omit__names():'
         nt.assert_in('ip.__str__', matches)
         nt.assert_in('ip._hidden_attr', matches)
 
-        c.use_jedi = True
-        completions = set(c.completions('ip.', 3))
-        nt.assert_in(Completion(3, 3, '__str__'), completions)
-        nt.assert_in(Completion(3,3, "_hidden_attr"), completions)
+        # c.use_jedi = True
+        # completions = set(c.completions('ip.', 3))
+        # nt.assert_in(Completion(3, 3, '__str__'), completions)
+        # nt.assert_in(Completion(3,3, "_hidden_attr"), completions)
 
 
     cfg = Config()
@@ -427,10 +435,10 b' def test_omit__names():'
         nt.assert_not_in('ip.__str__', matches)
         # nt.assert_in('ip._hidden_attr', matches)
 
-        c.use_jedi = True
-        completions = set(c.completions('ip.', 3))
-        nt.assert_not_in(Completion(3,3,'__str__'), completions)
-        nt.assert_in(Completion(3,3, "_hidden_attr"), completions)
+        # c.use_jedi = True
+        # completions = set(c.completions('ip.', 3))
+        # nt.assert_not_in(Completion(3,3,'__str__'), completions)
+        # nt.assert_in(Completion(3,3, "_hidden_attr"), completions)
 
     cfg = Config()
     cfg.IPCompleter.omit__names = 2
@@ -441,19 +449,19 b' def test_omit__names():'
         nt.assert_not_in('ip.__str__', matches)
         nt.assert_not_in('ip._hidden_attr', matches)
 
-        c.use_jedi = True
-        completions = set(c.completions('ip.', 3))
-        nt.assert_not_in(Completion(3,3,'__str__'), completions)
-        nt.assert_not_in(Completion(3,3, "_hidden_attr"), completions)
+        # c.use_jedi = True
+        # completions = set(c.completions('ip.', 3))
+        # nt.assert_not_in(Completion(3,3,'__str__'), completions)
+        # nt.assert_not_in(Completion(3,3, "_hidden_attr"), completions)
 
     with provisionalcompleter():
         c.use_jedi = False
         s,matches = c.complete('ip._x.')
         nt.assert_in('ip._x.keys', matches)
 
-        c.use_jedi = True
-        completions = set(c.completions('ip._x.', 6))
-        nt.assert_in(Completion(6,6, "keys"), completions)
+        # c.use_jedi = True
+        # completions = set(c.completions('ip._x.', 6))
+        # nt.assert_in(Completion(6,6, "keys"), completions)
 
     del ip._hidden_attr
     del ip._x

IPython/core/tests/test_display.py

@@ -275,6 +275,10 b' def test_video_embedding():'
         html = v._repr_html_()
         nt.assert_in('src="data:video/xyz;base64,YWJj"',html)
 
+def test_html_metadata():
+    s = "<h1>Test</h1>"
+    h = display.HTML(s, metadata={"isolated": True})
+    nt.assert_equal(h._repr_html_(), (s, {"isolated": True}))
 
 def test_display_id():
     ip = get_ipython()

IPython/core/tests/test_displayhook.py

@@ -1,4 +1,7 b''
+import sys
 from IPython.testing.tools import AssertPrints, AssertNotPrints
+from IPython.core.displayhook import CapturingDisplayHook
+from IPython.utils.capture import CapturedIO
 
 ip = get_ipython()
 
@@ -101,3 +104,11 b' def test_interactivehooks_ast_modes_semi_supress():'
 
     finally:
         ip.ast_node_interactivity = saved_mode
+
+def test_capture_display_hook_format():
+    """Tests that the capture display hook conforms to the CapturedIO output format"""
+    hook = CapturingDisplayHook(ip)
+    hook({"foo": "bar"})
+    captured = CapturedIO(sys.stdout, sys.stderr, hook.outputs)
+    # Should not raise with RichOutput transformation error
+    captured.outputs

IPython/core/tests/test_iplib.py

@@ -72,7 +72,7 b' In [4]: run simpleerr.py'
 ---------------------------------------------------------------------------
 ZeroDivisionError                         Traceback (most recent call last)
 <BLANKLINE>
-... in <module>()
+... in <module>
      30         mode = 'div'
      31 
 ---> 32     bar(mode)
@@ -104,7 +104,7 b' In [6]: run simpleerr.py'
 ---------------------------------------------------------------------------
 ZeroDivisionError                         Traceback (most recent call last)
 <BLANKLINE>
-... in <module>()
+... in <module>
      30         mode = 'div'
      31 
 ---> 32     bar(mode)
@@ -161,7 +161,7 b' In [22]: %tb'
 ---------------------------------------------------------------------------
 SystemExit                                Traceback (most recent call last)
 <BLANKLINE>
-...<module>()
+...<module>
      30         mode = 'div'
      31 
 ---> 32     bar(mode)
@@ -189,7 +189,7 b' In [24]: %tb'
 ---------------------------------------------------------------------------
 SystemExit                                Traceback (most recent call last)
 <BLANKLINE>
-... in <module>()
+... in <module>
      30         mode = 'div'
      31 
 ---> 32     bar(mode)

IPython/core/tests/test_magic.py

@@ -26,7 +26,6 b' from IPython.core.magic import (Magics, magics_class, line_magic,'
 from IPython.core.magics import execution, script, code, logging
 from IPython.testing import decorators as dec
 from IPython.testing import tools as tt
-from IPython.utils import py3compat
 from IPython.utils.io import capture_output
 from IPython.utils.tempdir import TemporaryDirectory
 from IPython.utils.process import find_cmd
@@ -304,12 +303,10 b' def test_macro_run():'
     """Test that we can run a multi-line macro successfully."""
     ip = get_ipython()
     ip.history_manager.reset()
-    cmds = ["a=10", "a+=1", py3compat.doctest_refactor_print("print a"),
-                                                            "%macro test 2-3"]
+    cmds = ["a=10", "a+=1", "print(a)", "%macro test 2-3"]
     for cmd in cmds:
         ip.run_cell(cmd, store_history=True)
-    nt.assert_equal(ip.user_ns["test"].value,
-                            py3compat.doctest_refactor_print("a+=1\nprint a\n"))
+    nt.assert_equal(ip.user_ns["test"].value, "a+=1\nprint(a)\n")
     with tt.AssertPrints("12"):
         ip.run_cell("test")
     with tt.AssertPrints("13"):
@@ -532,7 +529,6 b' def test_whos():'
     _ip.user_ns['a'] = A()
     _ip.magic("whos")
 
-@py3compat.u_format
 def doctest_precision():
     """doctest for %precision
     
@@ -566,11 +562,6 b' def test_timeit_shlex():'
     _ip.magic('timeit -r1 -n1 f("a " + "b ")')
 
 
-def test_timeit_arguments():
-    "Test valid timeit arguments, should not cause SyntaxError (GH #1269)"
-    _ip.magic("timeit ('#')")
-
-
 def test_timeit_special_syntax():
     "Test %%timeit with IPython special syntax"
     @register_line_magic
@@ -837,13 +828,16 b' def test_script_out_err():'
 def test_script_bg_out():
     ip = get_ipython()
     ip.run_cell_magic("script", "--bg --out output sh", "echo 'hi'")
+
     nt.assert_equal(ip.user_ns['output'].read(), b'hi\n')
+    ip.user_ns['output'].close()
 
 @dec.skip_win32
 def test_script_bg_err():
     ip = get_ipython()
     ip.run_cell_magic("script", "--bg --err error sh", "echo 'hello' >&2")
     nt.assert_equal(ip.user_ns['error'].read(), b'hello\n')
+    ip.user_ns['error'].close()
 
 @dec.skip_win32
 def test_script_bg_out_err():
@@ -851,6 +845,8 b' def test_script_bg_out_err():'
     ip.run_cell_magic("script", "--bg --out output --err error sh", "echo 'hi'\necho 'hello' >&2")
     nt.assert_equal(ip.user_ns['output'].read(), b'hi\n')
     nt.assert_equal(ip.user_ns['error'].read(), b'hello\n')
+    ip.user_ns['output'].close()
+    ip.user_ns['error'].close()
 
 def test_script_defaults():
     ip = get_ipython()
@@ -1070,4 +1066,15 b' def test_logging_magic_not_quiet():'
                 lm.logstart(os.path.join(td, "not_quiet.log"))
         finally:
             _ip.logger.logstop()
-    
+
+## 
+# this is slow, put at the end for local testing.
+## 
+def test_timeit_arguments():
+    "Test valid timeit arguments, should not cause SyntaxError (GH #1269)"
+    if sys.version_info < (3,7):
+        _ip.magic("timeit ('#')")
+    else:
+        # 3.7 optimize no-op statement like above out, and complain there is
+        # nothing in the for loop.
+        _ip.magic("timeit a=('#')")

IPython/core/tests/test_oinspect.py

@@ -430,8 +430,5 b' def test_init_colors():'
 def test_builtin_init():
     info = inspector.info(list)
     init_def = info['init_definition']
-    # Python < 3.4 can't get init definition from builtins,
-    # but still exercise the inspection in case of error-raising bugs.
-    if sys.version_info >= (3,4):
-        nt.assert_is_not_none(init_def)
+    nt.assert_is_not_none(init_def)
 

IPython/core/tests/test_profile.py

@@ -34,7 +34,6 b' from IPython.core.profiledir import ProfileDir'
 
 from IPython.testing import decorators as dec
 from IPython.testing import tools as tt
-from IPython.utils import py3compat
 from IPython.utils.process import getoutput
 from IPython.utils.tempdir import TemporaryDirectory
 
@@ -100,15 +99,14 b' class ProfileStartupTest(TestCase):'
             f.write(startup)
         # write simple test file, to check that the startup file was run
         with open(self.fname, 'w') as f:
-            f.write(py3compat.doctest_refactor_print(test))
+            f.write(test)
 
     def validate(self, output):
         tt.ipexec_validate(self.fname, output, '', options=self.options)
 
     @dec.skipif(win32_without_pywin32(), "Test requires pywin32 on Windows")
     def test_startup_py(self):
-        self.init('00-start.py', 'zzz=123\n', 
-                  py3compat.doctest_refactor_print('print zzz\n'))
+        self.init('00-start.py', 'zzz=123\n', 'print(zzz)\n')
         self.validate('123')
 
     @dec.skipif(win32_without_pywin32(), "Test requires pywin32 on Windows")

IPython/core/tests/test_pylabtools.py

@@ -105,7 +105,7 b' def test_select_figure_formats_kwargs():'
     f = formatter.lookup_by_type(Figure)
     cell = f.__closure__[0].cell_contents
     nt.assert_equal(cell, kwargs)
-    
+
     # check that the formatter doesn't raise
     fig = plt.figure()
     ax = fig.add_subplot(1,1,1)
@@ -150,7 +150,7 b' class TestPylabSwitch(object):'
     class Shell(InteractiveShell):
         def enable_gui(self, gui):
             pass
-    
+
     def setup(self):
         import matplotlib
         def act_mpl(backend):
@@ -244,3 +244,7 b' class TestPylabSwitch(object):'
         nt.assert_equal(gui, 'qt')
         nt.assert_equal(s.pylab_gui_select, 'qt')
 
+
+def test_no_gui_backends():
+    for k in ['agg', 'svg', 'pdf', 'ps']:
+        assert k not in pt.backend2gui

IPython/core/tests/test_run.py

@@ -31,7 +31,6 b' from nose import SkipTest'
 
 from IPython.testing import decorators as dec
 from IPython.testing import tools as tt
-from IPython.utils import py3compat
 from IPython.utils.io import capture_output
 from IPython.utils.tempdir import TemporaryDirectory
 from IPython.core import debugger
@@ -145,13 +144,12 b' def doctest_run_option_parser_for_windows():'
     """
 
 
[e-mail]_refactor_print
 def doctest_reset_del():
     """Test that resetting doesn't cause errors in __del__ methods.
 
     In [2]: class A(object):
        ...:     def __del__(self):
-       ...:         print str("Hi")
+       ...:         print(str("Hi"))
        ...:
 
     In [3]: a = A()
@@ -248,9 +246,9 b' class TestMagicRunSimple(tt.TempFileMixin):'
                 raise SkipTest("Test requires pywin32")
         src = ("class A(object):\n"
                "    def __del__(self):\n"
-               "        print 'object A deleted'\n"
+               "        print('object A deleted')\n"
                "a = A()\n")
-        self.mktmp(py3compat.doctest_refactor_print(src))
+        self.mktmp(src)
         if dec.module_not_available('sqlite3'):
             err = 'WARNING: IPython History requires SQLite, your history will not be saved\n'
         else:

IPython/core/ultratb.py

@@ -116,13 +116,15 b' from IPython.core import debugger'
 from IPython.core.display_trap import DisplayTrap
 from IPython.core.excolors import exception_colors
 from IPython.utils import PyColorize
-from IPython.utils import openpy
 from IPython.utils import path as util_path
 from IPython.utils import py3compat
 from IPython.utils.data import uniq_stable
 from IPython.utils.terminal import get_terminal_size
+
 from logging import info, error, debug
 
+from importlib.util import source_from_cache
+
 import IPython.utils.colorable as colorable
 
 # Globals
@@ -374,16 +376,32 b' def _fixed_getinnerframes(etb, context=1, tb_offset=0):'
 # (SyntaxErrors have to be treated specially because they have no traceback)
 
 
-def _format_traceback_lines(lnum, index, lines, Colors, lvals=None,  _line_format=(lambda x,_:x,None)):
+def _format_traceback_lines(lnum, index, lines, Colors, lvals, _line_format):
+    """
+    Format tracebacks lines with pointing arrow, leading numbers...
+
+    Parameters
+    ==========
+
+    lnum: int
+    index: int
+    lines: list[string]
+    Colors:
+        ColorScheme used.
+    lvals: bytes
+        Values of local variables, already colored, to inject just after the error line.
+    _line_format: f (str) -> (str, bool)
+        return (colorized version of str, failure to do so)
+    """
     numbers_width = INDENT_SIZE - 1
     res = []
-    i = lnum - index
 
-    for line in lines:
+    for i,line in enumerate(lines, lnum-index):
         line = py3compat.cast_unicode(line)
 
         new_line, err = _line_format(line, 'str')
-        if not err: line = new_line
+        if not err:
+            line = new_line
 
         if i == lnum:
             # This is the line with the error
@@ -399,7 +417,6 b' def _format_traceback_lines(lnum, index, lines, Colors, lvals=None, _line_forma'
         res.append(line)
         if lvals and i == lnum:
             res.append(lvals + '\n')
-        i = i + 1
     return res
 
 def is_recursion_error(etype, value, records):
@@ -621,16 +638,6 b' class ListTB(TBTools):'
         lines = ''.join(self._format_exception_only(etype, value))
         out_list.append(lines)
 
-        # Note: this code originally read:
-
-        ## for line in lines[:-1]:
-        ##     out_list.append(" "+line)
-        ## out_list.append(lines[-1])
-
-        # This means it was indenting everything but the last line by a little
-        # bit.  I've disabled this for now, but if we see ugliness somewhere we
-        # can restore it.
-
         return out_list
 
     def _format_list(self, extracted_list):
@@ -841,13 +848,6 b' class VerboseTB(TBTools):'
                                                  Colors.vName, ColorsNormal)
         tpl_name_val = '%%s %s= %%s%s' % (Colors.valEm, ColorsNormal)
 
-        tpl_line = '%s%%s%s %%s' % (Colors.lineno, ColorsNormal)
-        tpl_line_em = '%s%%s%s %%s%s' % (Colors.linenoEm, Colors.line,
-                                         ColorsNormal)
-
-        abspath = os.path.abspath
-
-
         if not file:
             file = '?'
         elif file.startswith(str("<")) and file.endswith(str(">")):
@@ -869,17 +869,19 b' class VerboseTB(TBTools):'
 
         file = py3compat.cast_unicode(file, util_path.fs_encoding)
         link = tpl_link % util_path.compress_user(file)
-        args, varargs, varkw, locals = inspect.getargvalues(frame)
+        args, varargs, varkw, locals_ = inspect.getargvalues(frame)
 
         if func == '?':
             call = ''
+        elif func == '<module>':
+            call = tpl_call % (func, '')
         else:
             # Decide whether to include variable details or not
-            var_repr = self.include_vars and eqrepr or nullrepr
+            var_repr = eqrepr if self.include_vars else nullrepr
             try:
                 call = tpl_call % (func, inspect.formatargvalues(args,
                                                                  varargs, varkw,
-                                                                 locals, formatvalue=var_repr))
+                                                                 locals_, formatvalue=var_repr))
             except KeyError:
                 # This happens in situations like errors inside generator
                 # expressions, where local variables are listed in the
@@ -906,7 +908,7 b' class VerboseTB(TBTools):'
         elif file.endswith(('.pyc', '.pyo')):
             # Look up the corresponding source file.
             try:
-                file = openpy.source_from_cache(file)
+                file = source_from_cache(file)
             except ValueError:
                 # Failed to get the source file for some reason
                 # E.g. https://github.com/ipython/ipython/issues/9486
@@ -968,14 +970,15 b' class VerboseTB(TBTools):'
         unique_names = uniq_stable(names)
 
         # Start loop over vars
-        lvals = []
+        lvals = ''
+        lvals_list = []
         if self.include_vars:
             for name_full in unique_names:
                 name_base = name_full.split('.', 1)[0]
                 if name_base in frame.f_code.co_varnames:
-                    if name_base in locals:
+                    if name_base in locals_:
                         try:
-                            value = repr(eval(name_full, locals))
+                            value = repr(eval(name_full, locals_))
                         except:
                             value = undefined
                     else:
@@ -990,11 +993,9 b' class VerboseTB(TBTools):'
                     else:
                         value = undefined
                     name = tpl_global_var % name_full
-                lvals.append(tpl_name_val % (name, value))
-        if lvals:
-            lvals = '%s%s' % (indent, em_normal.join(lvals))
-        else:
-            lvals = ''
+                lvals_list.append(tpl_name_val % (name, value))
+        if lvals_list:
+            lvals = '%s%s' % (indent, em_normal.join(lvals_list))
 
         level = '%s %s\n' % (link, call)
 
@@ -1041,7 +1042,6 b' class VerboseTB(TBTools):'
     def format_exception(self, etype, evalue):
         colors = self.Colors  # just a shorthand + quicker name lookup
         colorsnormal = colors.Normal  # used a lot
-        indent = ' ' * INDENT_SIZE
         # Get (safely) a string form of the exception info
         try:
             etype_str, evalue_str = map(str, (etype, evalue))

IPython/core/usage.py

@@ -339,10 +339,3 b' default_banner_parts = ["Python %s\\n"%sys.version.split("\\n")[0],'
 ]
 
 default_banner = ''.join(default_banner_parts)
-
-# deprecated GUI banner
-
-default_gui_banner = '\n'.join([
-    'DEPRECATED: IPython.core.usage.default_gui_banner is deprecated and will be removed',
-    default_banner,
-])

IPython/extensions/autoreload.py

@@ -116,10 +116,9 b' import traceback'
 import types
 import weakref
 from importlib import import_module
+from importlib.util import source_from_cache
 from imp import reload
 
-from IPython.utils import openpy
-
 #------------------------------------------------------------------------------
 # Autoreload functionality
 #------------------------------------------------------------------------------
@@ -195,7 +194,7 b' class ModuleReloader(object):'
             py_filename = filename
         else:
             try:
-                py_filename = openpy.source_from_cache(filename)
+                py_filename = source_from_cache(filename)
             except ValueError:
                 return None, None
 

IPython/external/qt_loaders.py

@@ -109,47 +109,6 b' def loaded_api():'
 def has_binding(api):
     """Safely check for PyQt4/5, PySide or PySide2, without importing submodules
 
-    Supports Python <= 3.3
-
-       Parameters
-       ----------
-       api : str [ 'pyqtv1' | 'pyqt' | 'pyqt5' | 'pyside' | 'pyside2' | 'pyqtdefault']
-            Which module to check for
-
-       Returns
-       -------
-       True if the relevant module appears to be importable
-    """
-    # we can't import an incomplete pyside and pyqt4
-    # this will cause a crash in sip (#1431)
-    # check for complete presence before importing
-    module_name = api_to_module[api]
-
-    import imp
-    try:
-        #importing top level PyQt4/PySide module is ok...
-        mod = import_module(module_name)
-        #...importing submodules is not
-        imp.find_module('QtCore', mod.__path__)
-        imp.find_module('QtGui', mod.__path__)
-        imp.find_module('QtSvg', mod.__path__)
-        if api in (QT_API_PYQT5, QT_API_PYSIDE2):
-            # QT5 requires QtWidgets too
-            imp.find_module('QtWidgets', mod.__path__)
-
-        #we can also safely check PySide version
-        if api == QT_API_PYSIDE:
-            return check_version(mod.__version__, '1.0.3')
-        else:
-            return True
-    except ImportError:
-        return False
-
-def has_binding_new(api):
-    """Safely check for PyQt4/5, PySide or PySide2, without importing submodules
-
-    Supports Python >= 3.4
-
         Parameters
         ----------
         api : str [ 'pyqtv1' | 'pyqt' | 'pyqt5' | 'pyside' | 'pyside2' | 'pyqtdefault']
@@ -185,8 +144,6 b' def has_binding_new(api):'
 
     return True
 
-if sys.version_info >= (3, 4):
-    has_binding = has_binding_new
 
 def qtapi_version():
     """Return which QString API has been set, if any

IPython/lib/display.py

@@ -2,13 +2,14 b''
 
 Authors : MinRK, gregcaporaso, dannystaple
 """
+from html import escape as html_escape
 from os.path import exists, isfile, splitext, abspath, join, isdir
-from os import walk, sep
+from os import walk, sep, fsdecode
 
-from IPython.core.display import DisplayObject
+from IPython.core.display import DisplayObject, TextDisplayObject
 
 __all__ = ['Audio', 'IFrame', 'YouTubeVideo', 'VimeoVideo', 'ScribdDocument',
-           'FileLink', 'FileLinks']
+           'FileLink', 'FileLinks', 'Code']
 
 
 class Audio(DisplayObject):
@@ -334,15 +335,16 b' class FileLink(object):'
         if isdir(path):
             raise ValueError("Cannot display a directory using FileLink. "
               "Use FileLinks to display '%s'." % path)
-        self.path = path
+        self.path = fsdecode(path)
         self.url_prefix = url_prefix
         self.result_html_prefix = result_html_prefix
         self.result_html_suffix = result_html_suffix
 
     def _format_path(self):
-        fp = ''.join([self.url_prefix,self.path])
+        fp = ''.join([self.url_prefix, html_escape(self.path)])
         return ''.join([self.result_html_prefix,
-                        self.html_link_str % (fp, self.path),
+                        self.html_link_str % \
+                            (fp, html_escape(self.path, quote=False)),
                         self.result_html_suffix])
 
     def _repr_html_(self):
@@ -555,3 +557,52 b' class FileLinks(FileLink):'
         for dirname, subdirs, fnames in walked_dir:
             result_lines += self.terminal_display_formatter(dirname, fnames, self.included_suffixes)
         return '\n'.join(result_lines)
+
+
+class Code(TextDisplayObject):
+    """Display syntax-highlighted source code.
+
+    This uses Pygments to highlight the code for HTML and Latex output.
+
+    Parameters
+    ----------
+    data : str
+        The code as a string
+    url : str
+        A URL to fetch the code from
+    filename : str
+        A local filename to load the code from
+    language : str
+        The short name of a Pygments lexer to use for highlighting.
+        If not specified, it will guess the lexer based on the filename
+        or the code. Available lexers: http://pygments.org/docs/lexers/
+    """
+    def __init__(self, data=None, url=None, filename=None, language=None):
+        self.language = language
+        super().__init__(data=data, url=url, filename=filename)
+
+    def _get_lexer(self):
+        if self.language:
+            from pygments.lexers import get_lexer_by_name
+            return get_lexer_by_name(self.language)
+        elif self.filename:
+            from pygments.lexers import get_lexer_for_filename
+            return get_lexer_for_filename(self.filename)
+        else:
+            from pygments.lexers import guess_lexer
+            return guess_lexer(self.data)
+
+    def __repr__(self):
+        return self.data
+
+    def _repr_html_(self):
+        from pygments import highlight
+        from pygments.formatters import HtmlFormatter
+        fmt = HtmlFormatter()
+        style = '<style>{}</style>'.format(fmt.get_style_defs('.output_html'))
+        return style + highlight(self.data, self._get_lexer(), fmt)
+
+    def _repr_latex_(self):
+        from pygments import highlight
+        from pygments.formatters import LatexFormatter
+        return highlight(self.data, self._get_lexer(), LatexFormatter())

IPython/lib/guisupport.py

@@ -2,7 +2,7 b''
 """
 Support for creating GUI apps and starting event loops.
 
-IPython's GUI integration allows interative plotting and GUI usage in IPython
+IPython's GUI integration allows interactive plotting and GUI usage in IPython
 session. IPython has two different types of GUI integration:
 
 1. The terminal based IPython supports GUI event loops through Python's

IPython/lib/lexers.py

@@ -34,7 +34,9 b' This includes:'
 import re
 
 # Third party
-from pygments.lexers import BashLexer, PythonLexer, Python3Lexer
+from pygments.lexers import (
+    BashLexer, HtmlLexer, JavascriptLexer, RubyLexer, PerlLexer, PythonLexer,
+    Python3Lexer, TexLexer)
 from pygments.lexer import (
     Lexer, DelegatingLexer, RegexLexer, do_insertions, bygroups, using,
 )
@@ -51,19 +53,6 b" __all__ = ['build_ipy_lexer', 'IPython3Lexer', 'IPythonLexer',"
            'IPythonPartialTracebackLexer', 'IPythonTracebackLexer',
            'IPythonConsoleLexer', 'IPyLexer']
 
-ipython_tokens = [
-  (r"(?s)(\s*)(%%)(\w+)(.*)", bygroups(Text, Operator, Keyword, Text)),
-  (r'(?s)(^\s*)(%%!)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(BashLexer))),
-  (r"(%%?)(\w+)(\?\??)$",  bygroups(Operator, Keyword, Operator)),
-  (r"\b(\?\??)(\s*)$",  bygroups(Operator, Text)),
-  (r'(%)(sx|sc|system)(.*)(\n)', bygroups(Operator, Keyword,
-                                       using(BashLexer), Text)),
-  (r'(%)(\w+)(.*\n)', bygroups(Operator, Keyword, Text)),
-  (r'^(!!)(.+)(\n)', bygroups(Operator, using(BashLexer), Text)),
-  (r'(!)(?!=)(.+)(\n)', bygroups(Operator, using(BashLexer), Text)),
-  (r'^(\s*)(\?\??)(\s*%{0,2}[\w\.\*]*)', bygroups(Text, Operator, Text)),
-  (r'(\s*%{0,2}[\w\.\*]*)(\?\??)(\s*)$', bygroups(Text, Operator, Text)),
-]
 
 def build_ipy_lexer(python3):
     """Builds IPython lexers depending on the value of `python3`.
@@ -92,6 +81,36 b' def build_ipy_lexer(python3):'
         aliases = ['ipython2', 'ipython']
         doc = """IPython Lexer"""
 
+    ipython_tokens = [
+       (r'(?s)(\s*)(%%capture)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(PyLexer))),
+        (r'(?s)(\s*)(%%debug)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(PyLexer))),
+        (r'(?is)(\s*)(%%html)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(HtmlLexer))),
+        (r'(?s)(\s*)(%%javascript)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(JavascriptLexer))),
+        (r'(?s)(\s*)(%%js)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(JavascriptLexer))),
+        (r'(?s)(\s*)(%%latex)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(TexLexer))),
+        (r'(?s)(\s*)(%%pypy)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(PerlLexer))),
+        (r'(?s)(\s*)(%%prun)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(PyLexer))),
+        (r'(?s)(\s*)(%%pypy)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(PyLexer))),
+        (r'(?s)(\s*)(%%python)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(PyLexer))),
+        (r'(?s)(\s*)(%%python2)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(PythonLexer))),
+        (r'(?s)(\s*)(%%python3)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(Python3Lexer))),
+        (r'(?s)(\s*)(%%ruby)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(RubyLexer))),
+        (r'(?s)(\s*)(%%time)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(PyLexer))),
+        (r'(?s)(\s*)(%%timeit)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(PyLexer))),
+        (r'(?s)(\s*)(%%writefile)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(PyLexer))),
+        (r"(?s)(\s*)(%%)(\w+)(.*)", bygroups(Text, Operator, Keyword, Text)),
+        (r'(?s)(^\s*)(%%!)([^\n]*\n)(.*)', bygroups(Text, Operator, Text, using(BashLexer))),
+        (r"(%%?)(\w+)(\?\??)$",  bygroups(Operator, Keyword, Operator)),
+        (r"\b(\?\??)(\s*)$",  bygroups(Operator, Text)),
+        (r'(%)(sx|sc|system)(.*)(\n)', bygroups(Operator, Keyword,
+                                                using(BashLexer), Text)),
+        (r'(%)(\w+)(.*\n)', bygroups(Operator, Keyword, Text)),
+        (r'^(!!)(.+)(\n)', bygroups(Operator, using(BashLexer), Text)),
+        (r'(!)(?!=)(.+)(\n)', bygroups(Operator, using(BashLexer), Text)),
+        (r'^(\s*)(\?\??)(\s*%{0,2}[\w\.\*]*)', bygroups(Text, Operator, Text)),
+        (r'(\s*%{0,2}[\w\.\*]*)(\?\??)(\s*)$', bygroups(Text, Operator, Text)),
+    ]
+
     tokens = PyLexer.tokens.copy()
     tokens['root'] = ipython_tokens + tokens['root']
 
@@ -217,7 +236,7 b' class IPythonConsoleLexer(Lexer):'
 
             ---------------------------------------------------------------------------
             Exception                                 Traceback (most recent call last)
-            <ipython-input-1-fca2ab0ca76b> in <module>()
+            <ipython-input-1-fca2ab0ca76b> in <module>
             ----> 1 raise Exception
 
             Exception:

IPython/lib/pretty.py

@@ -77,11 +77,13 b' Inheritance diagram:'
             Portions (c) 2009 by Robert Kern.
 :license: BSD License.
 """
+
 from contextlib import contextmanager
+import datetime
+import os
+import re
 import sys
 import types
-import re
-import datetime
 from collections import deque
 from io import StringIO
 from warnings import warn
@@ -742,7 +744,6 b' _type_pprinters = {'
     tuple:                      _seq_pprinter_factory('(', ')'),
     list:                       _seq_pprinter_factory('[', ']'),
     dict:                       _dict_pprinter_factory('{', '}'),
-    
     set:                        _set_pprinter_factory('{', '}'),
     frozenset:                  _set_pprinter_factory('frozenset({', '})'),
     super:                      _super_pprint,
@@ -751,12 +752,17 b' _type_pprinters = {'
     types.FunctionType:         _function_pprint,
     types.BuiltinFunctionType:  _function_pprint,
     types.MethodType:           _repr_pprint,
-    
     datetime.datetime:          _repr_pprint,
     datetime.timedelta:         _repr_pprint,
     _exception_base:            _exception_pprint
 }
 
+# render os.environ like a dict
+_env_type = type(os.environ)
+# future-proof in case os.environ becomes a plain dict?
+if _env_type is not dict:
+    _type_pprinters[_env_type] = _dict_pprinter_factory('environ{', '}')
+
 try:
     # In PyPy, types.DictProxyType is dict, setting the dictproxy printer
     # using dict.setdefault avoids overwritting the dict printer
@@ -768,7 +774,7 b' except AttributeError: # Python 3'
     _type_pprinters[types.MappingProxyType] = \
         _dict_pprinter_factory('mappingproxy({', '})')
     _type_pprinters[slice] = _repr_pprint
-    
+
 try:
     _type_pprinters[long] = _repr_pprint
     _type_pprinters[unicode] = _repr_pprint

IPython/lib/tests/test_display.py

@@ -14,6 +14,11 b''
 #-----------------------------------------------------------------------------
 from tempfile import NamedTemporaryFile, mkdtemp
 from os.path import split, join as pjoin, dirname
+import sys
+try:
+    import pathlib
+except ImportError:
+    pass
 
 # Third-party imports
 import nose.tools as nt
@@ -33,6 +38,9 b' from IPython.testing.decorators import skipif_not_numpy'
 def test_instantiation_FileLink():
     """FileLink: Test class can be instantiated"""
     fl = display.FileLink('example.txt')
+    # TODO: remove if when only Python >= 3.6 is supported
+    if sys.version_info >= (3, 6):
+        fl = display.FileLink(pathlib.PurePath('example.txt'))
 
 def test_warning_on_non_existant_path_FileLink():
     """FileLink: Calling _repr_html_ on non-existant files returns a warning
@@ -175,3 +183,7 b' def test_recursive_FileLinks():'
 def test_audio_from_file():
     path = pjoin(dirname(__file__), 'test.wav')
     display.Audio(filename=path)
+
+def test_code_from_file():
+    c = display.Code(filename=__file__)
+    assert c._repr_html_().startswith('<style>')

IPython/lib/tests/test_lexers.py

@@ -127,3 +127,50 b' class TestLexers(TestCase):'
             (Token.Text, '\n'),
         ]
         self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
+
+        fragment = '%%writefile -a foo.py\nif a == b:\n    pass'
+        tokens = [
+            (Token.Operator, '%%writefile'),
+            (Token.Text, ' -a foo.py\n'),
+            (Token.Keyword, 'if'),
+            (Token.Text, ' '),
+            (Token.Name, 'a'),
+            (Token.Text, ' '),
+            (Token.Operator, '=='),
+            (Token.Text, ' '),
+            (Token.Name, 'b'),
+            (Token.Punctuation, ':'),
+            (Token.Text, '\n'),
+            (Token.Text, '    '),
+            (Token.Keyword, 'pass'),
+            (Token.Text, '\n'),
+        ]
+        self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
+
+        fragment = '%%timeit\nmath.sin(0)'
+        tokens = [
+            (Token.Operator, '%%timeit\n'),
+            (Token.Name, 'math'),
+            (Token.Operator, '.'),
+            (Token.Name, 'sin'),
+            (Token.Punctuation, '('),
+            (Token.Literal.Number.Integer, '0'),
+            (Token.Punctuation, ')'),
+            (Token.Text, '\n'),
+        ]
+
+        fragment = '%%HTML\n<div>foo</div>'
+        tokens = [
+            (Token.Operator, '%%HTML'),
+            (Token.Text, '\n'),
+            (Token.Punctuation, '<'),
+            (Token.Name.Tag, 'div'),
+            (Token.Punctuation, '>'),
+            (Token.Text, 'foo'),
+            (Token.Punctuation, '<'),
+            (Token.Punctuation, '/'),
+            (Token.Name.Tag, 'div'),
+            (Token.Punctuation, '>'),
+            (Token.Text, '\n'),
+        ]
+        self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))

IPython/lib/tests/test_pretty.py

@@ -6,6 +6,7 b''
 
 
 from collections import Counter, defaultdict, deque, OrderedDict
+import os
 import types
 import string
 import unittest
@@ -406,6 +407,15 b' def test_mappingproxy():'
     for obj, expected in cases:
         nt.assert_equal(pretty.pretty(obj), expected)
 
+
+def test_pretty_environ():
+    dict_repr = pretty.pretty(dict(os.environ))
+    # reindent to align with 'environ' prefix
+    dict_indented = dict_repr.replace('\n', '\n' + (' ' * len('environ')))
+    env_repr = pretty.pretty(os.environ)
+    nt.assert_equals(env_repr, 'environ' + dict_indented)
+
+
 def test_function_pretty():
     "Test pretty print of function"
     # posixpath is a pure python module, its interface is consistent

IPython/terminal/debugger.py

@@ -8,15 +8,14 b' from .ptutils import IPythonPTCompleter'
 from .shortcuts import suspend_to_bg, cursor_in_leading_ws
 
 from prompt_toolkit.enums import DEFAULT_BUFFER
-from prompt_toolkit.filters import (Condition, HasFocus, HasSelection,
-    ViInsertMode, EmacsInsertMode)
-from prompt_toolkit.keys import Keys
-from prompt_toolkit.key_binding.manager import KeyBindingManager
+from prompt_toolkit.filters import (Condition, has_focus, has_selection,
+    vi_insert_mode, emacs_insert_mode)
+from prompt_toolkit.key_binding import KeyBindings
 from prompt_toolkit.key_binding.bindings.completion import display_completions_like_readline
-from prompt_toolkit.token import Token
-from prompt_toolkit.shortcuts import create_prompt_application
-from prompt_toolkit.interface import CommandLineInterface
+from pygments.token import Token
+from prompt_toolkit.shortcuts.prompt import PromptSession
 from prompt_toolkit.enums import EditingMode
+from prompt_toolkit.formatted_text import PygmentsTokens
 
 
 class TerminalPdb(Pdb):
@@ -26,46 +25,40 b' class TerminalPdb(Pdb):'
         self.pt_init()
 
     def pt_init(self):
-        def get_prompt_tokens(cli):
+        def get_prompt_tokens():
             return [(Token.Prompt, self.prompt)]
 
-        def patch_stdout(**kwargs):
-            return self.pt_cli.patch_stdout_context(**kwargs)
-
         if self._ptcomp is None:
             compl = IPCompleter(shell=self.shell,
                                         namespace={},
                                         global_namespace={},
                                         parent=self.shell,
                                        )
-            self._ptcomp = IPythonPTCompleter(compl, patch_stdout=patch_stdout)
+            self._ptcomp = IPythonPTCompleter(compl)
 
-        kbmanager = KeyBindingManager.for_prompt()
-        supports_suspend = Condition(lambda cli: hasattr(signal, 'SIGTSTP'))
-        kbmanager.registry.add_binding(Keys.ControlZ, filter=supports_suspend
-                                      )(suspend_to_bg)
+        kb = KeyBindings()
+        supports_suspend = Condition(lambda: hasattr(signal, 'SIGTSTP'))
+        kb.add('c-z', filter=supports_suspend)(suspend_to_bg)
 
         if self.shell.display_completions == 'readlinelike':
-            kbmanager.registry.add_binding(Keys.ControlI,
-                                 filter=(HasFocus(DEFAULT_BUFFER)
-                                         & ~HasSelection()
-                                         & ViInsertMode() | EmacsInsertMode()
-                                         & ~cursor_in_leading_ws
-                                         ))(display_completions_like_readline)
-        multicolumn = (self.shell.display_completions == 'multicolumn')
-
-        self._pt_app = create_prompt_application(
+            kb.add('tab', filter=(has_focus(DEFAULT_BUFFER)
+                                  & ~has_selection
+                                  & vi_insert_mode | emacs_insert_mode
+                                  & ~cursor_in_leading_ws
+                              ))(display_completions_like_readline)
+
+        self.pt_app = PromptSession(
+                            message=(lambda: PygmentsTokens(get_prompt_tokens())),
                             editing_mode=getattr(EditingMode, self.shell.editing_mode.upper()),
-                            key_bindings_registry=kbmanager.registry,
+                            key_bindings=kb,
                             history=self.shell.debugger_history,
-                            completer= self._ptcomp,
+                            completer=self._ptcomp,
                             enable_history_search=True,
                             mouse_support=self.shell.mouse_support,
-                            get_prompt_tokens=get_prompt_tokens,
-                            display_completions_in_columns=multicolumn,
-                            style=self.shell.style
+                            complete_style=self.shell.pt_complete_style,
+                            style=self.shell.style,
+                            inputhook=self.shell.inputhook,
         )
-        self.pt_cli = CommandLineInterface(self._pt_app, eventloop=self.shell._eventloop)
 
     def cmdloop(self, intro=None):
         """Repeatedly issue a prompt, accept input, parse an initial prefix
@@ -92,7 +85,7 b' class TerminalPdb(Pdb):'
                     self._ptcomp.ipy_completer.namespace = self.curframe_locals
                     self._ptcomp.ipy_completer.global_namespace = self.curframe.f_globals
                     try:
-                        line = self.pt_cli.run(reset_current_buffer=True).text
+                        line = self.pt_app.prompt() # reset_current_buffer=True)
                     except EOFError:
                         line = 'EOF'
                 line = self.precmd(line)

IPython/terminal/interactiveshell.py

@@ -15,15 +15,16 b' from traitlets import ('
     Any,
 )
 
-from prompt_toolkit.document import Document
 from prompt_toolkit.enums import DEFAULT_BUFFER, EditingMode
 from prompt_toolkit.filters import (HasFocus, Condition, IsDone)
+from prompt_toolkit.formatted_text import PygmentsTokens
 from prompt_toolkit.history import InMemoryHistory
-from prompt_toolkit.shortcuts import create_prompt_application, create_eventloop, create_prompt_layout, create_output
-from prompt_toolkit.interface import CommandLineInterface
-from prompt_toolkit.key_binding.manager import KeyBindingManager
 from prompt_toolkit.layout.processors import ConditionalProcessor, HighlightMatchingBracketProcessor
-from prompt_toolkit.styles import PygmentsStyle, DynamicStyle
+from prompt_toolkit.output import ColorDepth
+from prompt_toolkit.patch_stdout import patch_stdout
+from prompt_toolkit.shortcuts import PromptSession, CompleteStyle, print_formatted_text
+from prompt_toolkit.styles import DynamicStyle, merge_styles
+from prompt_toolkit.styles.pygments import style_from_pygments_cls, style_from_pygments_dict
 
 from pygments.styles import get_style_by_name
 from pygments.style import Style
@@ -34,7 +35,7 b' from .magics import TerminalMagics'
 from .pt_inputhooks import get_inputhook_name_and_func
 from .prompts import Prompts, ClassicPrompts, RichPromptDisplayHook
 from .ptutils import IPythonPTCompleter, IPythonPTLexer
-from .shortcuts import register_ipython_shortcuts
+from .shortcuts import create_ipython_shortcuts
 
 DISPLAY_BANNER_DEPRECATED = object()
 
@@ -88,15 +89,11 b" _use_simple_prompt = ('IPY_TEST_SIMPLE_PROMPT' in os.environ) or (not _is_tty)"
 
 class TerminalInteractiveShell(InteractiveShell):
     space_for_menu = Integer(6, help='Number of line at the bottom of the screen '
-                                                  'to reserve for the completion menu'
+                                     'to reserve for the completion menu'
                             ).tag(config=True)
 
-    def _space_for_menu_changed(self, old, new):
-        self._update_layout()
-
-    pt_cli = None
+    pt_app = None
     debugger_history = None
-    _pt_app = None
 
     simple_prompt = Bool(_use_simple_prompt,
         help="""Use `raw_input` for the REPL, without completion and prompt colors.
@@ -167,9 +164,9 b' class TerminalInteractiveShell(InteractiveShell):'
     def _prompts_default(self):
         return self.prompts_class(self)
 
-    @observe('prompts')
-    def _(self, change):
-        self._update_layout()
+#    @observe('prompts')
+#    def _(self, change):
+#        self._update_layout()
 
     @default('displayhook_class')
     def _displayhook_class_default(self):
@@ -241,10 +238,7 b' class TerminalInteractiveShell(InteractiveShell):'
             return
 
         # Set up keyboard shortcuts
-        kbmanager = KeyBindingManager.for_prompt(
-            enable_open_in_editor=self.extra_open_editor_shortcuts,
-        )
-        register_ipython_shortcuts(kbmanager.registry, self)
+        key_bindings = create_ipython_shortcuts(self)
 
         # Pre-populate history from IPython's history database
         history = InMemoryHistory()
@@ -254,7 +248,7 b' class TerminalInteractiveShell(InteractiveShell):'
             # Ignore blank lines and consecutive duplicates
             cell = cell.rstrip()
             if cell and (cell != last_cell):
-                history.append(cell)
+                history.append_string(cell)
                 last_cell = cell
 
         self._style = self._make_style_from_name_or_cls(self.highlighting_style)
@@ -262,24 +256,18 b' class TerminalInteractiveShell(InteractiveShell):'
 
         editing_mode = getattr(EditingMode, self.editing_mode.upper())
 
-        def patch_stdout(**kwargs):
-            return self.pt_cli.patch_stdout_context(**kwargs)
-
-        self._pt_app = create_prompt_application(
+        self.pt_app = PromptSession(
                             editing_mode=editing_mode,
-                            key_bindings_registry=kbmanager.registry,
+                            key_bindings=key_bindings,
                             history=history,
-                            completer=IPythonPTCompleter(shell=self,
-                                                    patch_stdout=patch_stdout),
-                            enable_history_search=self.enable_history_search,
+                            completer=IPythonPTCompleter(shell=self),
+                            enable_history_search = self.enable_history_search,
                             style=self.style,
+                            include_default_pygments_style=False,
                             mouse_support=self.mouse_support,
-                            **self._layout_options()
-        )
-        self._eventloop = create_eventloop(self.inputhook)
-        self.pt_cli = CommandLineInterface(
-            self._pt_app, eventloop=self._eventloop,
-            output=create_output(true_color=self.true_color))
+                            enable_open_in_editor=self.extra_open_editor_shortcuts,
+                            color_depth=(ColorDepth.TRUE_COLOR if self.true_color else None),
+                            **self._extra_prompt_options())
 
     def _make_style_from_name_or_cls(self, name_or_cls):
         """
@@ -340,44 +328,60 b' class TerminalInteractiveShell(InteractiveShell):'
                 Token.OutPromptNum: '#ff0000 bold',
             }
         style_overrides.update(self.highlighting_style_overrides)
-        style = PygmentsStyle.from_defaults(pygments_style_cls=style_cls,
-                                            style_dict=style_overrides)
+        style = merge_styles([
+            style_from_pygments_cls(style_cls),
+            style_from_pygments_dict(style_overrides),
+        ])
 
         return style
 
-    def _layout_options(self):
+    @property
+    def pt_complete_style(self):
+        return {
+            'multicolumn': CompleteStyle.MULTI_COLUMN,
+            'column': CompleteStyle.COLUMN,
+            'readlinelike': CompleteStyle.READLINE_LIKE,
+        }[self.display_completions]
+
+    def _extra_prompt_options(self):
         """
         Return the current layout option for the current Terminal InteractiveShell
         """
+        def get_message():
+            return PygmentsTokens(self.prompts.in_prompt_tokens())
+
         return {
+                'complete_in_thread': False,
                 'lexer':IPythonPTLexer(),
                 'reserve_space_for_menu':self.space_for_menu,
-                'get_prompt_tokens':self.prompts.in_prompt_tokens,
-                'get_continuation_tokens':self.prompts.continuation_prompt_tokens,
-                'multiline':True,
-                'display_completions_in_columns': (self.display_completions == 'multicolumn'),
+                'message': get_message,
+                'prompt_continuation': (
+                    lambda width, lineno, is_soft_wrap:
+                        PygmentsTokens(self.prompts.continuation_prompt_tokens(width))),
+                'multiline': True,
+                'complete_style': self.pt_complete_style,
 
                 # Highlight matching brackets, but only when this setting is
                 # enabled, and only when the DEFAULT_BUFFER has the focus.
-                'extra_input_processors': [ConditionalProcessor(
+                'input_processors': [ConditionalProcessor(
                         processor=HighlightMatchingBracketProcessor(chars='[](){}'),
                         filter=HasFocus(DEFAULT_BUFFER) & ~IsDone() &
-                            Condition(lambda cli: self.highlight_matching_brackets))],
+                            Condition(lambda: self.highlight_matching_brackets))],
                 }
 
-    def _update_layout(self):
-        """
-        Ask for a re computation of the application layout, if for example ,
-        some configuration options have changed.
-        """
-        if self._pt_app:
-            self._pt_app.layout = create_prompt_layout(**self._layout_options())
-
     def prompt_for_code(self):
-        with self.pt_cli.patch_stdout_context(raw=True):
-            document = self.pt_cli.run(
-                pre_run=self.pre_prompt, reset_current_buffer=True)
-        return document.text
+        if self.rl_next_input:
+            default = self.rl_next_input
+            self.rl_next_input = None
+        else:
+            default = ''
+
+        with patch_stdout(raw=True):
+            text = self.pt_app.prompt(
+                default=default,
+#                pre_run=self.pre_prompt,# reset_current_buffer=True,
+                **self._extra_prompt_options())
+        return text
 
     def enable_win_unicode_console(self):
         if sys.version_info >= (3, 6):
@@ -437,22 +441,6 b' class TerminalInteractiveShell(InteractiveShell):'
 
     rl_next_input = None
 
-    def pre_prompt(self):
-        if self.rl_next_input:
-            # We can't set the buffer here, because it will be reset just after
-            # this. Adding a callable to pre_run_callables does what we need
-            # after the buffer is reset.
-            s = self.rl_next_input
-            def set_doc():
-                self.pt_cli.application.buffer.document = Document(s)
-            if hasattr(self.pt_cli, 'pre_run_callables'):
-                self.pt_cli.pre_run_callables.append(set_doc)
-            else:
-                # Older version of prompt_toolkit; it's OK to set the document
-                # directly here.
-                set_doc()
-            self.rl_next_input = None
-
     def interact(self, display_banner=DISPLAY_BANNER_DEPRECATED):
 
         if display_banner is not DISPLAY_BANNER_DEPRECATED:
@@ -515,8 +503,9 b' class TerminalInteractiveShell(InteractiveShell):'
             return
 
         tokens = self.prompts.rewrite_prompt_tokens()
-        if self.pt_cli:
-            self.pt_cli.print_tokens(tokens)
+        if self.pt_app:
+            print_formatted_text(PygmentsTokens(tokens), end='',
+                                 style=self.pt_app.app.style)
             print(cmd)
         else:
             prompt = ''.join(s for t, s in tokens)
@@ -531,7 +520,7 b' class TerminalInteractiveShell(InteractiveShell):'
         elif self._prompts_before:
             self.prompts = self._prompts_before
             self._prompts_before = None
-        self._update_layout()
+#        self._update_layout()
 
 
 InteractiveShellABC.register(TerminalInteractiveShell)

IPython/terminal/prompts.py

@@ -5,13 +5,15 b' import sys'
 
 from IPython.core.displayhook import DisplayHook
 
-from prompt_toolkit.layout.utils import token_list_width
+from prompt_toolkit.formatted_text import fragment_list_width, PygmentsTokens
+from prompt_toolkit.shortcuts import print_formatted_text
+
 
 class Prompts(object):
     def __init__(self, shell):
         self.shell = shell
 
-    def in_prompt_tokens(self, cli=None):
+    def in_prompt_tokens(self):
         return [
             (Token.Prompt, 'In ['),
             (Token.PromptNum, str(self.shell.execution_count)),
@@ -19,9 +21,9 b' class Prompts(object):'
         ]
 
     def _width(self):
-        return token_list_width(self.in_prompt_tokens())
+        return fragment_list_width(self.in_prompt_tokens())
 
-    def continuation_prompt_tokens(self, cli=None, width=None):
+    def continuation_prompt_tokens(self, width=None):
         if width is None:
             width = self._width()
         return [
@@ -42,12 +44,12 b' class Prompts(object):'
         ]
 
 class ClassicPrompts(Prompts):
-    def in_prompt_tokens(self, cli=None):
+    def in_prompt_tokens(self):
         return [
             (Token.Prompt, '>>> '),
         ]
 
-    def continuation_prompt_tokens(self, cli=None, width=None):
+    def continuation_prompt_tokens(self, width=None):
         return [
             (Token.Prompt, '... ')
         ]
@@ -73,7 +75,9 b' class RichPromptDisplayHook(DisplayHook):'
                 # Ask for a newline before multiline output
                 self.prompt_end_newline = False
 
-            if self.shell.pt_cli:
-                self.shell.pt_cli.print_tokens(tokens)
+            if self.shell.pt_app:
+                print_formatted_text(PygmentsTokens(tokens),
+                    style=self.shell.pt_app.app.style, end='',
+                )
             else:
                 sys.stdout.write(prompt_txt)

IPython/terminal/ptutils.py

@@ -14,8 +14,9 b' from IPython.core.completer import ('
     provisionalcompleter, cursor_to_position,
     _deduplicate_completions)
 from prompt_toolkit.completion import Completer, Completion
-from prompt_toolkit.layout.lexers import Lexer
-from prompt_toolkit.layout.lexers import PygmentsLexer
+from prompt_toolkit.lexers import Lexer
+from prompt_toolkit.lexers import PygmentsLexer
+from prompt_toolkit.patch_stdout import patch_stdout
 
 import pygments.lexers as pygments_lexers
 
@@ -52,14 +53,11 b' def _adjust_completion_text_based_on_context(text, body, offset):'
 
 class IPythonPTCompleter(Completer):
     """Adaptor to provide IPython completions to prompt_toolkit"""
-    def __init__(self, ipy_completer=None, shell=None, patch_stdout=None):
+    def __init__(self, ipy_completer=None, shell=None):
         if shell is None and ipy_completer is None:
             raise TypeError("Please pass shell=an InteractiveShell instance.")
         self._ipy_completer = ipy_completer
         self.shell = shell
-        if patch_stdout is None:
-            raise TypeError("Please pass patch_stdout")
-        self.patch_stdout = patch_stdout
 
     @property
     def ipy_completer(self):
@@ -75,7 +73,7 b' class IPythonPTCompleter(Completer):'
         # is imported). This context manager ensures that doesn't interfere with
         # the prompt.
 
-        with self.patch_stdout(), provisionalcompleter():
+        with patch_stdout(), provisionalcompleter():
             body = document.text
             cursor_row = document.cursor_position_row
             cursor_col = document.cursor_position_col
@@ -143,7 +141,7 b' class IPythonPTLexer(Lexer):'
             'latex': PygmentsLexer(l.TexLexer),
         }
 
-    def lex_document(self, cli, document):
+    def lex_document(self, document):
         text = document.text.lstrip()
 
         lexer = self.python_lexer
@@ -157,4 +155,4 b' class IPythonPTLexer(Lexer):'
                     lexer = l
                     break
 
-        return lexer.lex_document(cli, document)
+        return lexer.lex_document(document)

IPython/terminal/shortcuts.py

@@ -12,91 +12,78 b' import sys'
 from typing import Callable
 
 
+from prompt_toolkit.application.current import get_app
 from prompt_toolkit.enums import DEFAULT_BUFFER, SEARCH_BUFFER
-from prompt_toolkit.filters import (HasFocus, HasSelection, Condition,
-    ViInsertMode, EmacsInsertMode, HasCompletions)
-from prompt_toolkit.filters.cli import ViMode, ViNavigationMode
-from prompt_toolkit.keys import Keys
+from prompt_toolkit.filters import (has_focus, has_selection, Condition,
+    vi_insert_mode, emacs_insert_mode, has_completions, vi_mode)
 from prompt_toolkit.key_binding.bindings.completion import display_completions_like_readline
+from prompt_toolkit.key_binding import KeyBindings
 
 from IPython.utils.decorators import undoc
 
 @undoc
 @Condition
-def cursor_in_leading_ws(cli):
-    before = cli.application.buffer.document.current_line_before_cursor
+def cursor_in_leading_ws():
+    before = get_app().current_buffer.document.current_line_before_cursor
     return (not before) or before.isspace()
 
-def register_ipython_shortcuts(registry, shell):
+
+def create_ipython_shortcuts(shell):
     """Set up the prompt_toolkit keyboard shortcuts for IPython"""
-    insert_mode = ViInsertMode() | EmacsInsertMode()
+
+    kb = KeyBindings()
+    insert_mode = vi_insert_mode | emacs_insert_mode
 
     if getattr(shell, 'handle_return', None):
         return_handler = shell.handle_return(shell)
     else:
         return_handler = newline_or_execute_outer(shell)
 
-    # Ctrl+J == Enter, seemingly
-    registry.add_binding(Keys.ControlJ,
-                         filter=(HasFocus(DEFAULT_BUFFER)
-                                 & ~HasSelection()
-                                 & insert_mode
+    kb.add('enter', filter=(has_focus(DEFAULT_BUFFER)
+                            & ~has_selection
+                            & insert_mode
                         ))(return_handler)
 
-    registry.add_binding(Keys.ControlBackslash)(force_exit)
+    kb.add('c-\\')(force_exit)
 
-    registry.add_binding(Keys.ControlP,
-                         filter=(ViInsertMode() & HasFocus(DEFAULT_BUFFER)
-                        ))(previous_history_or_previous_completion)
+    kb.add('c-p', filter=(vi_insert_mode & has_focus(DEFAULT_BUFFER))
+                )(previous_history_or_previous_completion)
 
-    registry.add_binding(Keys.ControlN,
-                         filter=(ViInsertMode() & HasFocus(DEFAULT_BUFFER)
-                        ))(next_history_or_next_completion)
+    kb.add('c-n', filter=(vi_insert_mode & has_focus(DEFAULT_BUFFER))
+                )(next_history_or_next_completion)
 
-    registry.add_binding(Keys.ControlG,
-                         filter=(HasFocus(DEFAULT_BUFFER) & HasCompletions()
-                        ))(dismiss_completion)
+    kb.add('c-g', filter=(has_focus(DEFAULT_BUFFER) & has_completions)
+                )(dismiss_completion)
 
-    registry.add_binding(Keys.ControlC, filter=HasFocus(DEFAULT_BUFFER)
-                        )(reset_buffer)
+    kb.add('c-c', filter=has_focus(DEFAULT_BUFFER))(reset_buffer)
 
-    registry.add_binding(Keys.ControlC, filter=HasFocus(SEARCH_BUFFER)
-                        )(reset_search_buffer)
+    kb.add('c-c', filter=has_focus(SEARCH_BUFFER))(reset_search_buffer)
 
-    supports_suspend = Condition(lambda cli: hasattr(signal, 'SIGTSTP'))
-    registry.add_binding(Keys.ControlZ, filter=supports_suspend
-                        )(suspend_to_bg)
+    supports_suspend = Condition(lambda: hasattr(signal, 'SIGTSTP'))
+    kb.add('c-z', filter=supports_suspend)(suspend_to_bg)
 
     # Ctrl+I == Tab
-    registry.add_binding(Keys.ControlI,
-                         filter=(HasFocus(DEFAULT_BUFFER)
-                                 & ~HasSelection()
-                                 & insert_mode
-                                 & cursor_in_leading_ws
+    kb.add('tab', filter=(has_focus(DEFAULT_BUFFER)
+                          & ~has_selection
+                          & insert_mode
+                          & cursor_in_leading_ws
                         ))(indent_buffer)
+    kb.add('c-o', filter=(has_focus(DEFAULT_BUFFER)
+                  & emacs_insert_mode))(newline_autoindent_outer(shell.input_transformer_manager))
 
-    registry.add_binding(Keys.ControlO,
-                         filter=(HasFocus(DEFAULT_BUFFER)
-                                & EmacsInsertMode()))(newline_autoindent_outer(shell.input_transformer_manager))
-
-    registry.add_binding(Keys.F2,
-                         filter=HasFocus(DEFAULT_BUFFER)
-                        )(open_input_in_editor)
+    kb.add('f2', filter=has_focus(DEFAULT_BUFFER))(open_input_in_editor)
 
     if shell.display_completions == 'readlinelike':
-        registry.add_binding(Keys.ControlI,
-                             filter=(HasFocus(DEFAULT_BUFFER)
-                                     & ~HasSelection()
-                                     & insert_mode
-                                     & ~cursor_in_leading_ws
-                            ))(display_completions_like_readline)
+        kb.add('c-i', filter=(has_focus(DEFAULT_BUFFER)
+                              & ~has_selection
+                              & insert_mode
+                              & ~cursor_in_leading_ws
+                        ))(display_completions_like_readline)
 
     if sys.platform == 'win32':
-        registry.add_binding(Keys.ControlV,
-                             filter=(
-                             HasFocus(
-                             DEFAULT_BUFFER) & ~ViMode()
-                            ))(win_paste)
+        kb.add('c-v', filter=(has_focus(DEFAULT_BUFFER) & ~vi_mode))(win_paste)
+
+    return kb
 
 
 def newline_or_execute_outer(shell):
@@ -127,8 +114,8 b' def newline_or_execute_outer(shell):'
             b.insert_text('\n' + indent)
             return
 
-        if (status != 'incomplete') and b.accept_action.is_returnable:
-            b.accept_action.validate_and_handle(event.cli, b)
+        if (status != 'incomplete') and b.accept_handler:
+            b.validate_and_handle()
         else:
             b.insert_text('\n' + indent)
     return newline_or_execute
@@ -170,10 +157,10 b' def reset_search_buffer(event):'
     if event.current_buffer.document.text:
         event.current_buffer.reset()
     else:
-        event.cli.push_focus(DEFAULT_BUFFER)
+        event.app.layout.focus(DEFAULT_BUFFER)
 
 def suspend_to_bg(event):
-    event.cli.suspend_to_background()
+    event.app.suspend_to_background()
 
 def force_exit(event):
     """
@@ -232,8 +219,8 b' def newline_autoindent_outer(inputsplitter) -> Callable[..., None]:'
 
 
 def open_input_in_editor(event):
-    event.cli.current_buffer.tempfile_suffix = ".py"
-    event.cli.current_buffer.open_in_editor(event.cli)
+    event.app.current_buffer.tempfile_suffix = ".py"
+    event.app.current_buffer.open_in_editor()
 
 
 if sys.platform == 'win32':

IPython/testing/iptest.py

@@ -69,7 +69,7 b' if version_info < (6,):'
     warnings.filterwarnings(
         'ignore', message='.*Please use assertEqual instead', category=Warning, module='IPython.*')
 
-if version_info < (7,):
+if version_info < (8,):
     warnings.filterwarnings('ignore', message='.*Completer.complete.*',
                             category=PendingDeprecationWarning, module='.*')
 else:

IPython/testing/tests/test_ipunittest.py

@@ -46,29 +46,26 b' Authors'
 #-----------------------------------------------------------------------------
 
 from IPython.testing.ipunittest import ipdoctest, ipdocstring
-from IPython.utils.py3compat import doctest_refactor_print
 
 #-----------------------------------------------------------------------------
 # Test classes and functions
 #-----------------------------------------------------------------------------
 @ipdoctest
-@doctest_refactor_print
 def simple_dt():
     """
-    >>> print 1+1
+    >>> print(1+1)
     2
     """
 
 
 @ipdoctest
-@doctest_refactor_print
 def ipdt_flush():
     """
-In [20]: print 1
+In [20]: print(1)
 1
 
 In [26]: for i in range(4):
-   ....:     print i
+   ....:     print(i)
    ....:     
    ....: 
 0
@@ -82,14 +79,13 b' Out[27]: 7'
 
 
 @ipdoctest
-@doctest_refactor_print
 def ipdt_indented_test():
     """
-    In [20]: print 1
+    In [20]: print(1)
     1
 
     In [26]: for i in range(4):
-       ....:     print i
+       ....:     print(i)
        ....:     
        ....: 
     0
@@ -110,14 +106,13 b' class Foo(object):'
     """
 
     @ipdocstring
-    @doctest_refactor_print
     def ipdt_method(self):
         """
-        In [20]: print 1
+        In [20]: print(1)
         1
 
         In [26]: for i in range(4):
-           ....:     print i
+           ....:     print(i)
            ....:     
            ....: 
         0
@@ -129,9 +124,8 b' class Foo(object):'
         Out[27]: 7
         """
 
-    @doctest_refactor_print
     def normaldt_method(self):
         """
-        >>> print 1+1
+        >>> print(1+1)
         2
         """

IPython/testing/tools.py

@@ -270,20 +270,25 b' class TempFileMixin(object):'
     def mktmp(self, src, ext='.py'):
         """Make a valid python temp file."""
         fname, f = temp_pyfile(src, ext)
-        self.tmpfile = f
+        if not hasattr(self, 'tmps'):
+            self.tmps=[]
+        self.tmps.append((f, fname))
         self.fname = fname
 
     def tearDown(self):
-        if hasattr(self, 'tmpfile'):
-            # If the tmpfile wasn't made because of skipped tests, like in
-            # win32, there's nothing to cleanup.
-            self.tmpfile.close()
-            try:
-                os.unlink(self.fname)
-            except:
-                # On Windows, even though we close the file, we still can't
-                # delete it.  I have no clue why
-                pass
+        # If the tmpfile wasn't made because of skipped tests, like in
+        # win32, there's nothing to cleanup.
+        if hasattr(self, 'tmps'):
+            for f,fname in self.tmps:
+                # If the tmpfile wasn't made because of skipped tests, like in
+                # win32, there's nothing to cleanup.
+                f.close()
+                try:
+                    os.unlink(fname)
+                except:
+                    # On Windows, even though we close the file, we still can't
+                    # delete it.  I have no clue why
+                    pass
 
     def __enter__(self):
         return self

IPython/utils/PyColorize.py

@@ -29,7 +29,7 b' scan Python source code and re-emit it with no changes to its original'
 formatting (which is the hard part).
 """
 
-__all__ = ['ANSICodeColors','Parser']
+__all__ = ['ANSICodeColors', 'Parser']
 
 _scheme_default = 'Linux'
 
@@ -43,7 +43,7 b' import tokenize'
 
 generate_tokens = tokenize.generate_tokens
 
-from IPython.utils.coloransi import TermColors, InputTermColors ,ColorScheme, ColorSchemeTable
+from IPython.utils.coloransi import TermColors, InputTermColors,ColorScheme, ColorSchemeTable
 from .colorable import Colorable
 from io import StringIO
 
@@ -185,8 +185,11 b' class Parser(Colorable):'
 
         super(Parser, self).__init__(parent=parent)
 
-        self.color_table = color_table and color_table or ANSICodeColors
+        self.color_table = color_table if color_table else ANSICodeColors
         self.out = out
+        self.pos = None
+        self.lines = None
+        self.raw = None
         if not style:
             self.style = self.default_style
         else:
@@ -213,7 +216,7 b' class Parser(Colorable):'
 
         string_output = 0
         if out == 'str' or self.out == 'str' or \
-           isinstance(self.out,StringIO):
+           isinstance(self.out, StringIO):
             # XXX - I don't really like this state handling logic, but at this
             # point I don't want to make major changes, so adding the
             # isinstance() check is the simplest I can do to ensure correct
@@ -223,15 +226,16 b' class Parser(Colorable):'
             string_output = 1
         elif out is not None:
             self.out = out
+        else:
+            raise ValueError('`out` or `self.out` should be file-like or the value `"str"`')
 
         # Fast return of the unmodified input for NoColor scheme
         if self.style == 'NoColor':
             error = False
             self.out.write(raw)
             if string_output:
-                return raw,error
-            else:
-                return None,error
+                return raw, error
+            return None, error
 
         # local shorthands
         colors = self.color_table[self.style].colors
@@ -245,9 +249,10 b' class Parser(Colorable):'
         pos = 0
         raw_find = self.raw.find
         lines_append = self.lines.append
-        while 1:
+        while True:
             pos = raw_find('\n', pos) + 1
-            if not pos: break
+            if not pos:
+                break
             lines_append(pos)
         lines_append(len(self.raw))
 
@@ -275,12 +280,13 b' class Parser(Colorable):'
             return (output, error)
         return (None, error)
 
-    def __call__(self, toktype, toktext, start_pos, end_pos, line):
-        """ Token handler, with syntax highlighting."""
-        (srow,scol) = start_pos
-        (erow,ecol) = end_pos
+
+    def _inner_call_(self, toktype, toktext, start_pos):
+        """like call but write to a temporary buffer"""
+        buff = StringIO()
+        srow, scol = start_pos
         colors = self.colors
-        owrite = self.out.write
+        owrite = buff.write
 
         # line separator, so this works across platforms
         linesep = os.linesep
@@ -297,7 +303,8 b' class Parser(Colorable):'
         # skip indenting tokens
         if toktype in [token.INDENT, token.DEDENT]:
             self.pos = newpos
-            return
+            buff.seek(0)
+            return buff.read()
 
         # map token type to a color group
         if token.LPAR <= toktype <= token.OP:
@@ -306,8 +313,6 b' class Parser(Colorable):'
             toktype = _KEYWORD
         color = colors.get(toktype, colors[_TEXT])
 
-        #print '<%s>' % toktext,    # dbg
-
         # Triple quoted strings must be handled carefully so that backtracking
         # in pagers works correctly. We need color terminators on _each_ line.
         if linesep in toktext:
@@ -316,3 +321,11 b' class Parser(Colorable):'
 
         # send text
         owrite('%s%s%s' % (color,toktext,colors.normal))
+        buff.seek(0)
+        return buff.read()
+
+
+    def __call__(self, toktype, toktext, start_pos, end_pos, line):
+        """ Token handler, with syntax highlighting."""
+        self.out.write(
+            self._inner_call_(toktype, toktext, start_pos))

IPython/utils/data.py

@@ -23,12 +23,6 b' def uniq_stable(elems):'
     return [x for x in elems if x not in seen and not seen.add(x)]
 
 
-def flatten(seq):
-    """Flatten a list of lists (NOT recursive, only works for 2d lists)."""
-
-    return [x for subseq in seq for x in subseq]
-    
-
 def chop(seq, size):
     """Chop a sequence into chunks of the given size."""
     return [seq[i:i+size] for i in range(0,len(seq),size)]

IPython/utils/frame.py

@@ -21,7 +21,6 b' from IPython.utils import py3compat'
 # Code
 #-----------------------------------------------------------------------------
 
[e-mail]_refactor_print
 def extract_vars(*names,**kw):
     """Extract a set of variables by name from another frame.
 

IPython/utils/io.py

@@ -17,7 +17,6 b' from warnings import warn'
 
 from IPython.utils.decorators import undoc
 from .capture import CapturedIO, capture_output
-from .py3compat import input
 
 @undoc
 class IOStream:
@@ -211,35 +210,34 b" def temp_pyfile(src, ext='.py'):"
     f.flush()
     return fname, f
 
+@undoc
 def atomic_writing(*args, **kwargs):
     """DEPRECATED: moved to notebook.services.contents.fileio"""
-    warn("IPython.utils.io.atomic_writing has moved to notebook.services.contents.fileio", stacklevel=2)
+    warn("IPython.utils.io.atomic_writing has moved to notebook.services.contents.fileio since IPython 4.0", DeprecationWarning, stacklevel=2)
     from notebook.services.contents.fileio import atomic_writing
     return atomic_writing(*args, **kwargs)
 
+@undoc
 def raw_print(*args, **kw):
-    """Raw print to sys.__stdout__, otherwise identical interface to print()."""
+    """DEPRECATED: Raw print to sys.__stdout__, otherwise identical interface to print()."""
+    warn("IPython.utils.io.raw_print has been deprecated since IPython 7.0", DeprecationWarning, stacklevel=2)
 
     print(*args, sep=kw.get('sep', ' '), end=kw.get('end', '\n'),
           file=sys.__stdout__)
     sys.__stdout__.flush()
 
-
+@undoc
 def raw_print_err(*args, **kw):
-    """Raw print to sys.__stderr__, otherwise identical interface to print()."""
+    """DEPRECATED: Raw print to sys.__stderr__, otherwise identical interface to print()."""
+    warn("IPython.utils.io.raw_print_err has been deprecated since IPython 7.0", DeprecationWarning, stacklevel=2)
 
     print(*args, sep=kw.get('sep', ' '), end=kw.get('end', '\n'),
           file=sys.__stderr__)
     sys.__stderr__.flush()
 
-
-# Short aliases for quick debugging, do NOT use these in production code.
-rprint = raw_print
-rprinte = raw_print_err
-
-
+@undoc
 def unicode_std_stream(stream='stdout'):
     """DEPRECATED, moved to nbconvert.utils.io"""
-    warn("IPython.utils.io.unicode_std_stream has moved to nbconvert.utils.io", stacklevel=2)
+    warn("IPython.utils.io.unicode_std_stream has moved to nbconvert.utils.io since IPython 4.0", DeprecationWarning, stacklevel=2)
     from nbconvert.utils.io import unicode_std_stream
     return unicode_std_stream(stream)

IPython/utils/openpy.py

@@ -103,19 +103,3 b" def read_py_url(url, errors='replace', skip_encoding_cookie=True):"
     response = urlopen(url)
     buffer = io.BytesIO(response.read())
     return source_to_unicode(buffer, errors, skip_encoding_cookie)
-
-def _list_readline(x):
-    """Given a list, returns a readline() function that returns the next element
-    with each call.
-    """
-    x = iter(x)
-    def readline():
-        return next(x)
-    return readline
-
-# Code for going between .py files and cached .pyc files ----------------------
-try: 
-    from importlib.util import source_from_cache, cache_from_source
-except ImportError :
-    ## deprecated since 3.4
-    from imp import source_from_cache, cache_from_source

IPython/utils/sysinfo.py

@@ -21,7 +21,7 b' import sys'
 import subprocess
 
 from IPython.core import release
-from IPython.utils import py3compat, _sysinfo, encoding
+from IPython.utils import _sysinfo, encoding
 
 #-----------------------------------------------------------------------------
 # Code
@@ -98,7 +98,6 b' def get_sys_info():'
     path = p.realpath(p.dirname(p.abspath(p.join(__file__, '..'))))
     return pkg_info(path)
 
[e-mail]_refactor_print
 def sys_info():
     """Return useful information about IPython and the system, as a string.
 
@@ -106,7 +105,7 b' def sys_info():'
     --------
     ::
     
-        In [2]: print sys_info()
+        In [2]: print(sys_info())
         {'commit_hash': '144fdae',      # random
          'commit_source': 'repository',
          'ipython_path': '/home/fperez/usr/lib/python2.6/site-packages/IPython',

IPython/utils/tests/test_io.py

@@ -18,7 +18,6 b' import nose.tools as nt'
 
 from IPython.testing.decorators import skipif, skip_win32
 from IPython.utils.io import IOStream, Tee, capture_output
-from IPython.utils.py3compat import doctest_refactor_print
 from IPython.utils.tempdir import TemporaryDirectory
 
 
@@ -59,7 +58,7 b' class TeeTestCase(unittest.TestCase):'
 def test_io_init():
     """Test that io.stdin/out/err exist at startup"""
     for name in ('stdin', 'stdout', 'stderr'):
-        cmd = doctest_refactor_print("from IPython.utils import io;print io.%s.__class__"%name)
+        cmd = "from IPython.utils import io;print(io.%s.__class__)"%name
         p = Popen([sys.executable, '-c', cmd],
                     stdout=PIPE)
         p.wait()

IPython/utils/tests/test_openpy.py

@@ -29,11 +29,3 b' def test_source_to_unicode():'
 
     source_no_cookie = openpy.source_to_unicode(source_bytes, skip_encoding_cookie=True)
     nt.assert_not_in(u'coding: iso-8859-5', source_no_cookie)
-
-def test_list_readline():
-    l = ['a', 'b']
-    readline = openpy._list_readline(l)
-    nt.assert_equal(readline(), 'a')
-    nt.assert_equal(readline(), 'b')
-    with nt.assert_raises(StopIteration):
-        readline()
  No newline at end of file

IPython/utils/tests/test_text.py

@@ -18,11 +18,7 b' import random'
 import sys
 
 import nose.tools as nt
-try:
-    from pathlib import Path
-except ImportError:
-    # for Python 3.3
-    from pathlib2 import Path
+from pathlib import Path
 
 from IPython.utils import text
 
@@ -137,13 +133,9 b' def eval_formatter_no_slicing_check(f):'
     
     s = f.format('{stuff[slice(1,4)]}', **ns)
     nt.assert_equal(s, 'ell')
-    
-    if sys.version_info >= (3, 4):
-        # String formatting has changed in Python 3.4, so this now works.
-        s = f.format("{a[:]}", a=[1, 2])
-        nt.assert_equal(s, "[1, 2]")
-    else:
-        nt.assert_raises(SyntaxError, f.format, "{a[:]}")
+
+    s = f.format("{a[:]}", a=[1, 2])
+    nt.assert_equal(s, "[1, 2]")
 
 def test_eval_formatter():
     f = text.EvalFormatter()

IPython/utils/text.py

@@ -13,11 +13,7 b' import re'
 import sys
 import textwrap
 from string import Formatter
-try:
-    from pathlib import Path
-except ImportError:
-    # for Python 3.3
-    from pathlib2 import Path
+from pathlib import Path
 
 from IPython.utils import py3compat
 
@@ -665,7 +661,7 b' def compute_item_matrix(items, row_first=False, empty=None, *args, **kwargs) :'
     empty : (default None)
         default value to fill list if needed
     separator_size : int (default=2)
-        How much caracters will be used as a separation between each columns.
+        How much characters will be used as a separation between each columns.
     displaywidth : int (default=80)
         The width of the area onto which the columns should enter
 

MANIFEST.in

@@ -1,7 +1,16 b''
 include README.rst
 include COPYING.rst
+include LICENSE
 include setupbase.py
 include setupegg.py
+include MANIFEST.in
+include tox.ini
+include .mailmap
+
+recursive-exclude tools *
+exclude tools
+exclude CONTRIBUTING.md
+exclude .editorconfig
 
 graft setupext
 
@@ -29,6 +38,7 b' prune docs/dist'
 # Patterns to exclude from any directory
 global-exclude *~
 global-exclude *.flc
+global-exclude *.yml
 global-exclude *.pyc
 global-exclude *.pyo
 global-exclude .dircopy.log

README.rst

@@ -23,7 +23,9 b' contribute to the project.'
 
 **IPython versions and Python Support**
 
-**IPython 6** requires Python version 3.3 and above.
+**IPython 7.0** requires Python version 3.4 and above.
+
+**IPython 6.x** requires Python version 3.3 and above.
 
 **IPython 5.x LTS** is the compatible release for Python 2.7.
 If you require Python 2 support, you **must** use IPython 5.x LTS. Please
@@ -31,7 +33,7 b' update your project configurations and requirements as necessary.'
 
 
 The Notebook, Qt console and a number of other pieces are now parts of *Jupyter*.
-See the `Jupyter installation docs <http://jupyter.readthedocs.io/en/latest/install.html>`__
+See the `Jupyter installation docs <https://jupyter.readthedocs.io/en/latest/install.html>`__
 if you want to use these.
 
 
@@ -41,7 +43,7 b' Development and Instant running'
 ===============================
 
 You can find the latest version of the development documentation on `readthedocs
-<http://ipython.readthedocs.io/en/latest/>`_.
+<https://ipython.readthedocs.io/en/latest/>`_.
 
 You can run IPython from this directory without even installing it system-wide
 by typing at the terminal::
@@ -49,7 +51,7 b' by typing at the terminal::'
    $ python -m IPython
 
 Or see the `development installation docs
-<http://ipython.readthedocs.io/en/latest/install/install.html#installing-the-development-version>`_
+<https://ipython.readthedocs.io/en/latest/install/install.html#installing-the-development-version>`_
 for the latest revision on read the docs.
 
 Documentation and installation instructions for older version of IPython can be
@@ -68,14 +70,14 b' Support version.'
 
 If you are encountering this error message you are likely trying to install or
 use IPython from source. You need to checkout the remote 5.x branch. If you are
-using git the following should work:
+using git the following should work::
 
   $ git fetch origin
   $ git checkout 5.x
 
 If you encounter this error message with a regular install of IPython, then you
 likely need to update your package manager, for example if you are using `pip`
-check the version of pip with
+check the version of pip with::
 
   $ pip --version
 

appveyor.yml

@@ -8,8 +8,8 b' environment:'
      PYTHON_VERSION: "3.6.x"
      PYTHON_ARCH: "32"
 
-   - PYTHON: "C:\\Python33-x64"
-     PYTHON_VERSION: "3.3.x"
+   - PYTHON: "C:\\Python34-x64"
+     PYTHON_VERSION: "3.4.x"
      PYTHON_ARCH: "64"
 
    - PYTHON: "C:\\Python36-x64"
@@ -21,7 +21,7 b' init:'
 
 install:
   - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%"
-  - "%CMD_IN_ENV%  pip install setuptools>=18.5 --upgrade"
+  - "%CMD_IN_ENV%  python -m pip install --upgrade setuptools pip"
   - "%CMD_IN_ENV%  pip install nose coverage"
   - "%CMD_IN_ENV%  pip install .[test]"
   - "%CMD_IN_ENV%  mkdir results"

docs/README.rst

@@ -48,7 +48,7 b' API documentation. This build target skips that.'
 You can run ``make help`` to see information on all possible make targets.
 
 To save time,
-the make targets above only proceess the files that have been changed since the
+the make targets above only process the files that have been changed since the
 previous docs build.
 To remove the previous docs build you can use ``make clean``.
 You can also combine ``clean`` with other `make` commands;

docs/autogen_shortcuts.py

@@ -1,7 +1,6 b''
 from os.path import abspath, dirname, join
 
-from IPython.terminal.interactiveshell import KeyBindingManager
-
+from IPython.terminal.shortcuts import create_ipython_shortcuts
 
 def name(c):
     s = c.__class__.__name__
@@ -42,8 +41,14 b' def multi_filter_str(flt):'
 log_filters = {'_AndList': 'And', '_OrList': 'Or'}
 log_invert =  {'_Invert'}
 
-kbm = KeyBindingManager.for_prompt()
-ipy_bindings = kbm.registry.key_bindings
+class _DummyTerminal(object):
+    """Used as a buffer to get prompt_toolkit bindings
+    """
+    handle_return = None
+    input_splitter = None
+    display_completions = None
+
+ipy_bindings = create_ipython_shortcuts(_DummyTerminal()).bindings
 
 dummy_docs = []  # ignore bindings without proper documentation
 

docs/source/config/integrating.rst

@@ -42,6 +42,13 b' For example::'
         def _repr_html_(self):
             return "<h1>" + self.text + "</h1>"
 
+We often want to provide frontends with guidance on how to display the data. To
+support this, ``_repr_*_()`` methods can also return a ``(data, metadata)``
+tuple where ``metadata`` is a dictionary containing arbitrary key-value pairs for
+the frontend to interpret. An example use case is ``_repr_jpeg_()``, which can
+be set to return a jpeg image and a ``{'height': 400, 'width': 600}`` dictionary
+to inform the frontend how to size the image.
+
 There are also two more powerful display methods:
 
 .. class:: MyObject

docs/source/interactive/plotting.rst

@@ -11,7 +11,7 b' In order for this to be possible, you need to use the ``display()`` function,'
 that should be available by default on IPython 5.4+ and 6.1+, or that you can
 import with ``from IPython.display import display``. Then use ``display(<your
 object>)`` instead of ``print()``, and if possible your object will be displayed
-with a richer representation. In the terminal of course, there wont be much
+with a richer representation. In the terminal of course, there won't be much
 difference as object are most of the time represented by text, but in notebook
 and similar interface you will get richer outputs.
 

docs/source/interactive/reference.rst

@@ -308,6 +308,9 b' one of (note that the modes are given unquoted):'
     * [append:] well, that says it.
     * [rotate:] create rotating logs log_name.1~, log_name.2~, etc.
 
+Adding the '-o' flag to '%logstart' magic (as in '%logstart -o [log_name [log_mode]]')
+will also include output from iPython in the log file.
+
 The :magic:`logoff` and :magic:`logon` functions allow you to temporarily stop and
 resume logging to a file which had previously been started with
 %logstart. They will fail (with an explanation) if you try to use them
@@ -607,12 +610,66 b' startup files, and everything, just as if it were a normal IPython session.'
 For information on setting configuration options when running IPython from
 python, see :ref:`configure_start_ipython`.
 
-It is also possible to embed an IPython shell in a namespace in your Python code.
-This allows you to evaluate dynamically the state of your code,
-operate with your variables, analyze them, etc. Note however that
-any changes you make to values while in the shell do not propagate back
-to the running code, so it is safe to modify your values because you
-won't break your code in bizarre ways by doing so.
+It is also possible to embed an IPython shell in a namespace in your Python
+code. This allows you to evaluate dynamically the state of your code, operate
+with your variables, analyze them, etc. For example, if you run the following
+code snippet::
+
+  import IPython
+
+  a = 42
+  IPython.embed()
+
+and within the IPython shell, you reassign `a` to `23` to do further testing of 
+some sort, you can then exit::
+
+  >>> IPython.embed()
+  Python 3.6.2 (default, Jul 17 2017, 16:44:45) 
+  Type 'copyright', 'credits' or 'license' for more information
+  IPython 6.2.0.dev -- An enhanced Interactive Python. Type '?' for help.
+
+  In [1]: a = 23
+
+  In [2]: exit()
+
+Once you exit and print `a`, the value 23 will be shown::
+
+
+  In: print(a)
+  23
+
+It's important to note that the code run in the embedded IPython shell will 
+*not* change the state of your code and variables, **unless** the shell is 
+contained within the global namespace. In the above example, `a` is changed 
+because this is true.
+
+To further exemplify this, consider the following example::
+
+  import IPython
+  def do():
+      a = 42
+      print(a)
+      IPython.embed()
+      print(a)
+
+Now if call the function and complete the state changes as we did above, the
+value `42` will be printed. Again, this is because it's not in the global
+namespace:: 
+
+  do()
+
+Running a file with the above code can lead to the following session::
+
+  >>> do()
+  42
+  Python 3.6.2 (default, Jul 17 2017, 16:44:45) 
+  Type 'copyright', 'credits' or 'license' for more information
+  IPython 6.2.0.dev -- An enhanced Interactive Python. Type '?' for help.
+
+  In [1]: a = 23
+
+  In [2]: exit()
+  42
 
 .. note::
 
@@ -711,7 +768,7 b' context line to show. This allows to a many line of context on shallow stack tra'
     In[6]: foo(1)
     # ...
     ipdb> where 8
-    <ipython-input-6-9e45007b2b59>(1)<module>()
+    <ipython-input-6-9e45007b2b59>(1)<module>
     ----> 1 foo(1)
 
     <ipython-input-5-7baadc3d1465>(5)foo()
@@ -740,7 +797,7 b' And less context on shallower Stack Trace:'
 .. code::
 
     ipdb> where 1
-    <ipython-input-13-afa180a57233>(1)<module>()
+    <ipython-input-13-afa180a57233>(1)<module>
     ----> 1 foo(7)
 
     <ipython-input-5-7baadc3d1465>(5)foo()

docs/source/overview.rst

@@ -217,29 +217,30 b' running, use the ``%connect_info`` magic to get the unique connection file,'
 which will be something like ``--existing kernel-19732.json`` but with
 different numbers which correspond to the Process ID of the kernel.
 
-You can read more about using `jupyter qtconsole 
+You can read more about using `jupyter qtconsole
 <http://jupyter.org/qtconsole/>`_, and
 `jupyter notebook <http://jupyter-notebook.readthedocs.io/en/latest/>`_. There
-is also a :ref:`message spec <messaging>` which documents the protocol for 
+is also a :ref:`message spec <messaging>` which documents the protocol for
 communication between kernels
 and clients.
 
 .. seealso::
-    
+
     `Frontend/Kernel Model`_ example notebook
 
 
 Interactive parallel computing
 ==============================
 
-    
+
 This functionality is optional and now part of the `ipyparallel
 <http://ipyparallel.readthedocs.io/>`_ project.
 
 Portability and Python requirements
 -----------------------------------
 
-Version 6.0+ supports compatibility with Python 3.3 and higher.
+Version 7.0+ supports Python 3.4 and higher.
+Versions 6.x support Python 3.3 and higher.
 Versions 2.0 to 5.x work with Python 2.7.x releases and Python 3.3 and higher.
 Version 1.0 additionally worked with Python 2.6 and 3.2.
 Version 0.12 was the first version to fully support Python 3.

docs/source/whatsnew/github-stats-0.11.rst

@@ -384,7 +384,7 b' Regular issues (285):'
 * `326 <https://github.com/ipython/ipython/issues/326>`_: Update docs and examples for parallel stuff to reflect movement away from Twisted
 * `341 <https://github.com/ipython/ipython/issues/341>`_: FIx Parallel Magics for newparallel
 * `338 <https://github.com/ipython/ipython/issues/338>`_: Usability improvements to Qt console
-* `142 <https://github.com/ipython/ipython/issues/142>`_: unexpected auto-indenting when varibles names that start with 'pass' 
+* `142 <https://github.com/ipython/ipython/issues/142>`_: unexpected auto-indenting when variables names that start with 'pass' 
 * `296 <https://github.com/ipython/ipython/issues/296>`_: Automatic PDB via %pdb doesn't work
 * `337 <https://github.com/ipython/ipython/issues/337>`_: exit( and quit( in Qt console produces phantom signature/docstring popup, even though quit() or exit() raises NameError
 * `318 <https://github.com/ipython/ipython/issues/318>`_: %debug broken in master: invokes missing save_history() method

docs/source/whatsnew/github-stats-1.0.rst

@@ -580,7 +580,7 b' Pull Requests (793):'
 * :ghpull:`3576`: Added support for markdown in heading cells when they are nbconverted.
 * :ghpull:`3575`: tweak `run -d` message to 'continue execution'
 * :ghpull:`3569`: add PYTHONSTARTUP to startup files
-* :ghpull:`3567`: Trigger a single event on js app initilized
+* :ghpull:`3567`: Trigger a single event on js app initialized
 * :ghpull:`3565`: style.min.css should always exist...
 * :ghpull:`3531`: allow markdown in heading cells
 * :ghpull:`3577`: Simplify codemirror ipython-mode
@@ -1195,7 +1195,7 b' Issues (691):'
 * :ghissue:`3957`: Notebook help page broken in Firefox
 * :ghissue:`3894`: nbconvert test failure
 * :ghissue:`3887`: 1.0.0a1 shows blank screen in both firefox and chrome (windows 7)
-* :ghissue:`3703`: `nbconvert`: Output options -- names and documentataion
+* :ghissue:`3703`: `nbconvert`: Output options -- names and documentation
 * :ghissue:`3931`: Tab completion not working during debugging in the notebook
 * :ghissue:`3936`: Ipcluster plugin is not working with Ipython 1.0dev
 * :ghissue:`3941`: IPython Notebook kernel crash on Win7x64
@@ -1264,7 +1264,7 b' Issues (691):'
 * :ghissue:`3737`: ipython nbconvert crashes with ValueError: Invalid format string.
 * :ghissue:`3730`: nbconvert: unhelpful error when pandoc isn't installed
 * :ghissue:`3718`: markdown cell cursor misaligned in notebook
-* :ghissue:`3710`: mutiple input fields for %debug in the notebook after resetting the kernel
+* :ghissue:`3710`: multiple input fields for %debug in the notebook after resetting the kernel
 * :ghissue:`3713`: PyCharm has problems with IPython working inside PyPy created by virtualenv
 * :ghissue:`3712`: Code completion: Complete on dictionary keys
 * :ghissue:`3680`: --pylab and --matplotlib flag
@@ -1405,7 +1405,7 b' Issues (691):'
 * :ghissue:`3519`: IPython Parallel map mysteriously turns pandas Series into numpy ndarray
 * :ghissue:`3345`: IPython embedded shells ask if I want to exit, but I set confirm_exit = False
 * :ghissue:`3509`: IPython won't close without asking "Are you sure?" in Firefox 
-* :ghissue:`3471`: Notebook jinja2/markupsafe depedencies in manual
+* :ghissue:`3471`: Notebook jinja2/markupsafe dependencies in manual
 * :ghissue:`3502`: Notebook broken in master
 * :ghissue:`3302`: autoreload does not work in ipython 0.13.x, python 3.3
 * :ghissue:`3475`: no warning when leaving/closing notebook on master without saved changes
@@ -1477,7 +1477,7 b' Issues (691):'
 * :ghissue:`3380`: simple call to kernel
 * :ghissue:`3379`: TaskRecord key 'started' not set
 * :ghissue:`3241`: notebook connection time out
-* :ghissue:`3334`: magic interpreter interpretes non magic commands?
+* :ghissue:`3334`: magic interpreter interprets non magic commands?
 * :ghissue:`3326`: python3.3: Type error when launching SGE cluster in IPython notebook
 * :ghissue:`3349`: pip3 doesn't run 2to3?
 * :ghissue:`3347`: Longlist support in ipdb

docs/source/whatsnew/github-stats-5.rst

@@ -3,6 +3,28 b''
 Issues closed in the 5.x development cycle
 ==========================================
 
+Issues closed in 5.6
+--------------------
+
+GitHub stats for 2017/09/15 - 2018/04/02 (tag: 5.5.0)
+
+These lists are automatically generated, and may be incomplete or contain duplicates.
+
+We closed 2 issues and merged 28 pull requests.
+The full list can be seen `on GitHub <https://github.com/ipython/ipython/issues?q=milestone%3A5.6>`__
+
+The following 10 authors contributed 47 commits.
+
+* Benjamin Ragan-Kelley
+* Henry Fredrick Schreiner
+* Joris Van den Bossche
+* Matthias Bussonnier
+* Mradul Dubey
+* Roshan Rao
+* Samuel Lelièvre
+* Teddy Rendahl
+* Thomas A Caswell
+* Thomas Kluyver
 
 Issues closed in 5.4
 --------------------

docs/source/whatsnew/github-stats-6.rst

@@ -1,10 +1,58 b''
 Issues closed in the 6.x development cycle
 ==========================================
 
-Issues closed in 6.2
+Issues closed in 6.3
 --------------------
 
 
+GitHub stats for 2017/09/15 - 2018/04/02 (tag: 6.2.0)
+
+These lists are automatically generated, and may be incomplete or contain duplicates.
+
+We closed 10 issues and merged 50 pull requests.
+The full list can be seen `on GitHub <https://github.com/ipython/ipython/issues?q=milestone%3A6.3>`__
+
+The following 35 authors contributed 253 commits.
+
+* Anatoly Techtonik
+* Antony Lee
+* Benjamin Ragan-Kelley
+* Corey McCandless
+* Craig Citro
+* Cristian Ciupitu
+* David Cottrell
+* David Straub
+* Doug Latornell
+* Fabio Niephaus
+* Gergely Nagy
+* Henry Fredrick Schreiner
+* Hugo
+* Ismael Venegas Castelló
+* Ivan Gonzalez
+* J Forde
+* Jeremy Sikes
+* Joris Van den Bossche
+* Lesley Cordero
+* luzpaz
+* madhu94
+* Matthew R. Scott
+* Matthias Bussonnier
+* Matthias Geier
+* Olesya Baranova
+* Peter Williams
+* Rastislav Barlik
+* Roshan Rao
+* rs2
+* Samuel Lelièvre
+* Shailyn javier Ortiz jimenez
+* Sjoerd de Vries
+* Teddy Rendahl
+* Thomas A Caswell
+* Thomas Kluyver
+
+Issues closed in 6.2
+--------------------
+
 GitHub stats for 2017/05/31 - 2017/09/15 (tag: 6.1.0)
 
 These lists are automatically generated, and may be incomplete or contain duplicates.

docs/source/whatsnew/index.rst

@@ -43,5 +43,3 b' development work they do here in a user friendly format.'
    version0.10
    version0.9
    version0.8
-
-

docs/source/whatsnew/version0.12.rst

@@ -293,7 +293,7 b' Backwards incompatible changes'
   deprecated, but continue to work.
 
 * For embedding a shell, note that the parameters ``user_global_ns`` and
-  ``global_ns`` have been deprectated in favour of ``user_module`` and
+  ``global_ns`` have been deprecated in favour of ``user_module`` and
   ``module`` respsectively.  The new parameters expect a module-like object,
   rather than a namespace dict.  The old parameters remain for backwards
   compatibility, although ``user_global_ns`` is now ignored. The ``user_ns``

docs/source/whatsnew/version5.rst

@@ -3,6 +3,33 b''
 ============
 
 
+.. _whatsnew570:
+
+IPython 5.7
+===========
+
+* Fix IPython trying to import non-existing matplotlib backends :ghpull:`11087`
+* fix for display hook not publishing object metadata :ghpull:`11101`
+
+.. _whatsnew560:
+
+IPython 5.6
+===========
+
+* In Python 3.6 and above, dictionaries preserve the order items were added to
+  them. On these versions, IPython will display dictionaries in their native
+  order, rather than sorting by the keys (:ghpull:`10958`).
+* :class:`~.IPython.display.ProgressBar` can now be used as an iterator
+  (:ghpull:`10813`).
+* The shell object gains a :meth:`~.InteractiveShell.check_complete` method,
+  to allow a smoother transition to new input processing machinery planned for
+  IPython 7 (:ghpull:`11044`).
+* IPython should start faster, as it no longer looks for all available pygments
+  styles on startup (:ghpull:`10859`).
+
+You can see all the PR marked for the `5.6. milestone <https://github.com/ipython/ipython/pulls?utf8=%E2%9C%93&q=is%3Apr+milestone%3A5.6+is%3Aclosed+NOT+%22Backport+PR%22+>`_,
+and all the `backport versions <https://github.com/ipython/ipython/pulls?utf8=%E2%9C%93&q=is%3Apr%20milestone%3A5.6%20is%3Aclosed%20%22Backport%20PR%22%20>`__.
+
 .. _whatsnew550:
 
 IPython 5.5

docs/source/whatsnew/version6.rst

@@ -2,6 +2,55 b''
  6.x Series
 ============
 
+.. _whatsnew640:
+
+IPython 6.4.0
+=============
+
+Everything new in :ref:`IPython 5.7 <whatsnew570>`
+
+* Fix display object not emitting metadata :ghpull:`11106`
+* Comments failing Jedi test :ghpull:`11110` 
+
+
+.. _whatsnew631:
+
+IPython 6.3.1
+=============
+
+This is a bugfix release to switch the default completions back to IPython's
+own completion machinery. We discovered some problems with the completions
+from Jedi, including completing column names on pandas data frames.
+
+You can switch the completions source with the config option
+:configtrait:`Completer.use_jedi`.
+
+.. _whatsnew630:
+
+IPython 6.3
+===========
+
+IPython 6.3 contains all the bug fixes and features in
+:ref:`IPython 5.6 <whatsnew560>`. In addition:
+
+* A new display class :class:`IPython.display.Code` can be used to display
+  syntax highlighted code in a notebook (:ghpull:`10978`).
+* The :cellmagic:`html` magic now takes a ``--isolated`` option to put the
+  content in an iframe (:ghpull:`10962`).
+* The code to find completions using the Jedi library has had various
+  adjustments. This is still a work in progress, but we hope this version has
+  fewer annoyances (:ghpull:`10956`, :ghpull:`10969`, :ghpull:`10999`,
+  :ghpull:`11035`, :ghpull:`11063`, :ghpull:`11065`).
+* The *post* event callbacks are now always called, even when the execution failed
+  (for example because of a ``SyntaxError``).
+* The execution info and result objects are now made available in the
+  corresponding *pre* or *post* ``*_run_cell`` :doc:`event callbacks </config/callbacks>`
+  in a backward compatible manner (:ghissue:`10774` and :ghpull:`10795`).
+* Performance with very long code cells (hundreds of lines) is greatly improved
+  (:ghpull:`10898`). Further improvements are planned for IPython 7.
+
+You can see all `pull requests for the 6.3 milestone
+<https://github.com/ipython/ipython/pulls?utf8=%E2%9C%93&q=is%3Apr+milestone%3A6.3+is%3Aclosed>`__.
 
 .. _whatsnew620:
 

examples/IPython Kernel/Script Magics.ipynb

@@ -436,7 +436,7 b''
     "\n",
     "    c.ScriptMagics.scripts = ['R', 'pypy', 'myprogram']\n",
     "\n",
-    "And if any of these programs do not apear on your default PATH, then you would also need to specify their location with:\n",
+    "And if any of these programs do not appear on your default PATH, then you would also need to specify their location with:\n",
     "\n",
     "    c.ScriptMagics.script_paths = {'myprogram': '/opt/path/to/myprogram'}"
    ]

setup.py

@@ -1,4 +1,4 b''
-#!/usr/bin/env python
+#!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 """Setup script for IPython.
 
@@ -26,7 +26,7 b' import sys'
 #
 # This check is also made in IPython/__init__, don't forget to update both when
 # changing Python version requirements.
-if sys.version_info < (3, 3):
+if sys.version_info < (3, 4):
     pip_message = 'This may be due to an out of date pip. Make sure you have pip >= 9.0.1.'
     try:
         import pip
@@ -42,9 +42,9 b' if sys.version_info < (3, 3):'
 
 
     error = """
-IPython 6.0+ does not support Python 2.6, 2.7, 3.0, 3.1, or 3.2.
+IPython 7.0+ supports Python 3.4 and above.
 When using Python 2.7, please install IPython 5.x LTS Long Term Support version.
-Beginning with IPython 6.0, Python 3.3 and above is required.
+Python 3.3 was supported up to IPython 6.x.
 
 See IPython `README.rst` file for more information:
 
@@ -175,7 +175,7 b' extras_require = dict('
     parallel = ['ipyparallel'],
     qtconsole = ['qtconsole'],
     doc = ['Sphinx>=1.3'],
-    test = ['nose>=0.10.1', 'requests', 'testpath', 'pygments', 'nbformat', 'ipykernel'],
+    test = ['nose>=0.10.1', 'requests', 'testpath', 'pygments', 'nbformat', 'ipykernel', 'numpy'],
     terminal = [],
     kernel = ['ipykernel'],
     nbformat = ['nbformat'],
@@ -190,7 +190,7 b' install_requires = ['
     'pickleshare',
     'simplegeneric>0.8',
     'traitlets>=4.2',
-    'prompt_toolkit>=1.0.15,<2.0.0',
+    'prompt_toolkit>=2.0.0,<2.1.0',
     'pygments',
     'backcall',
 ]
@@ -200,9 +200,7 b' install_requires = ['
 # but requires pip >= 6. pip < 6 ignores these.
 
 extras_require.update({
-    'test:python_version >= "3.4"': ['numpy'],
-    ':python_version == "3.3"': ['pathlib2'],
-    ':python_version <= "3.4"': ['typing'],
+    ':python_version == "3.4"': ['typing'],
     ':sys_platform != "win32"': ['pexpect'],
     ':sys_platform == "darwin"': ['appnope'],
     ':sys_platform == "win32"': ['colorama'],
@@ -232,7 +230,7 b' for key, deps in extras_require.items():'
 extras_require['all'] = everything
 
 if 'setuptools' in sys.modules:
-    setuptools_extra_args['python_requires'] = '>=3.3'
+    setuptools_extra_args['python_requires'] = '>=3.4'
     setuptools_extra_args['zip_safe'] = False
     setuptools_extra_args['entry_points'] = {
         'console_scripts': find_entry_points(),

setupext/install_data_ext.py

@@ -65,7 +65,7 b' class install_data_ext(install_data):'
             files = lof[2]
             if len(files) == 0:
                 # If there are no files listed, the user must be
-                # trying to create an empty directory, so add the the
+                # trying to create an empty directory, so add the
                 # directory to the list of output files.
                 self.outfiles.append(dir)
             else:

tools/build_release

@@ -1,4 +1,4 b''
-#!/usr/bin/env python
+#!/usr/bin/env python3
 """IPython release build script.
 """
 import os

tools/git-mpr.py

@@ -58,7 +58,7 b' def merge_pr(num):'
         cmd = "git pull "+repo+" "+branch
         not_merged[str(num)] = cmd
         print("==============================================================================")
-        print("Something went wrong merging this branch, you can try it manually by runngin :")
+        print("Something went wrong merging this branch, you can try it manually by running :")
         print(cmd)
         print("==============================================================================")
         
@@ -118,7 +118,7 b' def main(*args):'
 
     if not_merged :
         print('*************************************************************************************')
-        print('the following branch have not been merged automatically, considere doing it by hand :')
+        print('The following branch has not been merged automatically, consider doing it by hand   :')
         for num, cmd in not_merged.items() :
             print( "PR {num}: {cmd}".format(num=num, cmd=cmd))
         print('*************************************************************************************')

tools/github_stats.py

@@ -212,7 +212,7 b' if __name__ == "__main__":'
 
     print("We closed %d issues and merged %d pull requests." % (n_issues, n_pulls))
     if milestone:
-        print("The full list can be seen `on GitHub <https://github.com/{project}/issues?q=milestone%3A{milestone}+>`__".format(project=project,milestone=milestone)
+        print("The full list can be seen `on GitHub <https://github.com/{project}/issues?q=milestone%3A{milestone}>`__".format(project=project,milestone=milestone)
         )
     
     print()

tools/release

@@ -1,4 +1,4 b''
-#!/usr/bin/env python
+#!/usr/bin/env python3
 """IPython release script.
 
 This should ONLY be run at real release time.