Prototype async REPL using IPython, take III

This is a squash and a rebase of a large number of commits from Min and
I. For simplicity of managing it, history has been reduced to a single
commit, but more historical versions can be found, in particular in
PR 11155, or commit aedb5d6d3a441dcdb7180ac9b5cc03f91329117b to be more
exact.

IPython/core/async_helpers.py

@@ -0,0 +1,88
+"""
+Async helper function that are invalid syntax on Python 3.5 and below.
+
+Known limitation and possible improvement.
+
+Top level code that contain a return statement (instead of, or in addition to
+await) will be detected as requiring being wrapped in async calls. This should
+be prevented as early return will not work.
+"""
+
+
+
+import ast
+import sys
+import inspect
+from textwrap import dedent, indent
+from types import CodeType
+
+
+def _asyncio_runner(coro):
+    """
+    Handler for asyncio autoawait
+    """
+    import asyncio
+    return asyncio.get_event_loop().run_until_complete(coro)
+
+
+def _curio_runner(coroutine):
+    """
+    handler for curio autoawait
+    """
+    import curio
+    return curio.run(coroutine)
+
+
+if sys.version_info > (3, 5):
+    # nose refuses to avoid this file and async def is invalidsyntax
+    s = dedent('''
+    def _trio_runner(function):
+        import trio
+        async def loc(coro):
+            """
+            We need the dummy no-op async def to protect from
+            trio's internal. See https://github.com/python-trio/trio/issues/89
+            """
+            return await coro
+        return trio.run(loc, function)
+    ''')
+    exec(s, globals(), locals())
+
+
+def _asyncify(code: str) -> str:
+    """wrap code in async def definition.
+
+    And setup a bit of context to run it later.
+    """
+    res = dedent("""
+        async def __wrapper__():
+            try:
+                {usercode}
+            finally:
+                locals()
+    """).format(usercode=indent(code, ' ' * 8)[8:])
+    return res
+
+
+def _should_be_async(cell: str) -> bool:
+    """Detect if a block of code need to be wrapped in an `async def`
+
+    Attempt to parse the block of code, it it compile we're fine.
+    Otherwise we  wrap if and try to compile.
+
+    If it works, assume it should be async. Otherwise Return False.
+
+    Not handled yet: If the block of code has a return statement as  the top
+    level, it will be seen as async. This is a know limitation.
+    """
+
+    try:
+        ast.parse(cell)
+        return False
+    except SyntaxError:
+        try:
+            ast.parse(_asyncify(cell))
+        except SyntaxError:
+            return False
+        return True
+    return False

IPython/core/tests/test_async_helpers.py

@@ -0,0 +1,52
+"""
+Test for async helpers. 
+
+Should only trigger on python 3.5+ or will have syntax errors.
+"""
+
+import sys
+import nose.tools as nt
+from textwrap import dedent
+from unittest import TestCase
+
+ip = get_ipython()
+iprc = lambda x: ip.run_cell(dedent(x))
+
+if sys.version_info > (3,5):
+    from IPython.core.async_helpers import _should_be_async
+
+    class AsyncTest(TestCase):
+
+        def test_should_be_async(self):
+            nt.assert_false(_should_be_async("False"))
+            nt.assert_true(_should_be_async("await bar()"))
+            nt.assert_true(_should_be_async("x = await bar()"))
+            nt.assert_false(_should_be_async(dedent("""
+                async def awaitable():
+                    pass
+            """)))
+
+        def test_execute(self):
+            iprc("""
+            import asyncio
+            await asyncio.sleep(0.001)
+            """)
+
+        def test_autoawait(self):
+            ip.run_cell('%autoawait False')
+            ip.run_cell('%autoawait True')
+            iprc('''
+                from asyncio import sleep
+                await.sleep(0.1)
+            ''')
+
+        def test_autoawait_curio(self):
+            ip.run_cell('%autoawait curio')
+
+        def test_autoawait_trio(self):
+            ip.run_cell('%autoawait trio')
+
+        def tearDown(self):
+            ip.loop_runner = 'asyncio'
+
+

docs/source/interactive/autoawait.rst

@@ -0,0 +1,186
+
+.. autoawait:
+
+Asynchronous in REPL: Autoawait
+===============================
+
+Starting with IPython 6.0, and when user Python 3.6 and above, IPython offer the
+ability to run asynchronous code from the REPL. constructs which are
+:exc:`SyntaxError` s in the Python REPL can be used seamlessly in IPython.
+
+When a supported libray is used, IPython will automatically `await` Futures
+and Coroutines in the REPL. This will happen if an :ref:`await <await>` (or `async`) is
+use at top level scope, or if any structure valid only in `async def
+<https://docs.python.org/3/reference/compound_stmts.html#async-def>`_ function
+context are present. For example, the following being a syntax error in the
+Python REPL::
+
+    Python 3.6.0 
+    [GCC 4.2.1]
+    Type "help", "copyright", "credits" or "license" for more information.
+    >>> import aiohttp
+    >>> result = aiohttp.get('https://api.github.com')
+    >>> response = await result
+      File "<stdin>", line 1
+        response = await result
+                              ^
+    SyntaxError: invalid syntax
+
+Should behave as expected in the IPython REPL::
+
+    Python 3.6.0
+    Type 'copyright', 'credits' or 'license' for more information
+    IPython 6.0.0.dev -- An enhanced Interactive Python. Type '?' for help.
+
+    In [1]: import aiohttp
+       ...: result = aiohttp.get('https://api.github.com')
+
+    In [2]: response = await result
+    <pause for a few 100s ms>
+
+    In [3]: await response.json()
+    Out[3]:
+    {'authorizations_url': 'https://api.github.com/authorizations',
+     'code_search_url': 'https://api.github.com/search/code?q={query}...',
+    ...
+    }
+
+
+You can use the ``c.InteractiveShell.autoawait`` configuration option and set it
+to :any:`False` to deactivate automatic wrapping of asynchronous code. You can also
+use the :magic:`%autoawait` magic to toggle the behavior at runtime::
+
+    In [1]: %autoawait False
+
+    In [2]: %autoawait
+    IPython autoawait is `Off`, and set to use `IPython.core.interactiveshell._asyncio_runner`
+
+
+
+By default IPython will assume integration with Python's provided
+:mod:`asyncio`, but integration with other libraries is provided. In particular
+we provide experimental integration with the ``curio`` and ``trio`` library.
+
+You can switch current integration by using the
+``c.InteractiveShell.loop_runner`` option or the ``autoawait <name
+integration>`` magic.
+
+For example::
+
+    In [1]: %autoawait trio
+
+    In [2]: import trio
+
+    In [3]: async def child(i):
+       ...:     print("   child %s goes to sleep"%i)
+       ...:     await trio.sleep(2)
+       ...:     print("   child %s wakes up"%i)
+
+    In [4]: print('parent start')
+       ...: async with trio.open_nursery() as n:
+       ...:     for i in range(5):
+       ...:         n.spawn(child, i)
+       ...: print('parent end')
+    parent start
+       child 2 goes to sleep
+       child 0 goes to sleep
+       child 3 goes to sleep
+       child 1 goes to sleep
+       child 4 goes to sleep
+       <about 2 seconds pause>
+       child 2 wakes up
+       child 1 wakes up
+       child 0 wakes up
+       child 3 wakes up
+       child 4 wakes up
+    parent end
+
+
+In the above example, ``async with`` at top level scope is a syntax error in
+Python.
+
+Using this mode can have unexpected consequences if used in interaction with
+other features of IPython and various registered extensions. In particular if you
+are a direct or indirect user of the AST transformers, these may not apply to
+your code.
+
+The default loop, or runner does not run in the background, so top level
+asynchronous code must finish for the REPL to allow you to enter more code. As
+with usual Python semantic, the awaitables are started only when awaited for the
+first time. That is to say, in first example, no network request is done between
+``In[1]`` and ``In[2]``.
+
+
+Internals
+=========
+
+As running asynchronous code is not supported in interactive REPL as of Python
+3.6 we have to rely to a number of complex workaround to allow this to happen.
+It is interesting to understand how this works in order to understand potential
+bugs, or provide a custom runner.
+
+Among the many approaches that are at our disposition, we find only one that
+suited out need. Under the hood we :ct the code object from a async-def function
+and run it in global namesapace after modifying the ``__code__`` object.::
+
+    async def inner_async():
+        locals().update(**global_namespace)
+        #
+        # here is user code
+        #
+        return last_user_statement
+    codeobj = modify(inner_async.__code__)
+    coroutine = eval(codeobj, user_ns)
+    display(loop_runner(coroutine))
+
+
+
+The first thing you'll notice is that unlike classical ``exec``, there is only
+one name space. Second, user code runs in a function scope, and not a module
+scope.
+
+On top of the above there are significant modification to the AST of
+``function``, and ``loop_runner`` can be arbitrary complex. So there is a
+significant overhead to this kind of code.
+
+By default the generated coroutine function will be consumed by Asyncio's
+``loop_runner = asyncio.get_evenloop().run_until_complete()`` method. It is
+though possible to provide your own.
+
+A loop runner is a *synchronous*  function responsible from running a coroutine
+object.
+
+The runner is responsible from ensuring that ``coroutine`` run to completion,
+and should return the result of executing the coroutine. Let's write a
+runner for ``trio`` that print a message when used as an exercise, ``trio`` is
+special as it usually prefer to run a function object and make a coroutine by
+itself, we can get around this limitation by wrapping it in an async-def without
+parameters and passing this value to ``trio``::
+
+
+    In [1]: import trio
+       ...: from types import CoroutineType
+       ...:
+       ...: def trio_runner(coro:CoroutineType):
+       ...:     print('running asynchronous code')
+       ...:     async def corowrap(coro):
+       ...:         return await coro
+       ...:     return trio.run(corowrap, coro)
+
+We can set it up by passing it to ``%autoawait``::
+
+    In [2]: %autoawait trio_runner
+
+    In [3]: async def async_hello(name):
+       ...:     await trio.sleep(1)
+       ...:     print(f'Hello {name} world !')
+       ...:     await trio.sleep(1)
+
+    In [4]: await async_hello('async')
+    running asynchronous code
+    Hello async world !
+
+
+Asynchronous programming in python (and in particular in the REPL) is still a
+relatively young subject. Feel free to contribute improvements to this codebase
+and give us feedback.

docs/source/whatsnew/pr/await-repl.rst

@@ -0,0 +1,55
+Await REPL
+----------
+
+:ghpull:`10390` introduced the ability to ``await`` Futures and
+Coroutines in the REPL. For example::
+
+    Python 3.6.0
+    Type 'copyright', 'credits' or 'license' for more information
+    IPython 6.0.0.dev -- An enhanced Interactive Python. Type '?' for help.
+
+    In [1]: import aiohttp
+       ...: result = aiohttp.get('https://api.github.com')
+
+    In [2]: response = await result
+    <pause for a few 100s ms>
+
+    In [3]: await response.json()
+    Out[3]:
+    {'authorizations_url': 'https://api.github.com/authorizations',
+     'code_search_url': 'https://api.github.com/search/code?q={query}{&page,per_page,sort,order}',
+    ...
+    }
+
+
+Integration is by default with `asyncio`, but other libraries can be configured, 
+like ``curio`` or ``trio``, to improve concurrency in the REPL::
+
+    In [1]: %autoawait trio
+
+    In [2]: import trio
+
+    In [3]: async def child(i):
+       ...:     print("   child %s goes to sleep"%i)
+       ...:     await trio.sleep(2)
+       ...:     print("   child %s wakes up"%i)
+
+    In [4]: print('parent start')
+       ...: async with trio.open_nursery() as n:
+       ...:     for i in range(3):
+       ...:         n.spawn(child, i)
+       ...: print('parent end')
+    parent start
+       child 2 goes to sleep
+       child 0 goes to sleep
+       child 1 goes to sleep
+       <about 2 seconds pause>
+       child 2 wakes up
+       child 1 wakes up
+       child 0 wakes up
+    parent end
+
+See :ref:`autoawait` for more information.
+
+
+

IPython/core/interactiveshell.py

@@ -13,6 +13,7
 
 import abc
 import ast
+import asyncio
 import atexit
 import builtins as builtin_mod
 import functools
@@ -30,6 +31,7 from io import open as io_open
 from pickleshare import PickleShareDB
 
 from traitlets.config.configurable import SingletonConfigurable
+from traitlets.utils.importstring import import_item
 from IPython.core import oinspect
 from IPython.core import magic
 from IPython.core import page
@@ -73,7 +75,7 from IPython.utils.text import format_screen, LSString, SList, DollarFormatter
 from IPython.utils.tempdir import TemporaryDirectory
 from traitlets import (
     Integer, Bool, CaselessStrEnum, Enum, List, Dict, Unicode, Instance, Type,
-    observe, default,
+    observe, default, validate, Any
 )
 from warnings import warn
 from logging import error
@@ -113,6 +115,102 else:
     _single_targets_nodes = (ast.AugAssign, )
 
 #-----------------------------------------------------------------------------
+# Await Helpers
+#-----------------------------------------------------------------------------
+
+def removed_co_newlocals(function:types.FunctionType) -> types.FunctionType:
+    """Return a function that do not create a new local scope. 
+
+    Given a function, create a clone of this function where the co_newlocal flag
+    has been removed, making this function code actually run in the sourounding
+    scope. 
+
+    We need this in order to run asynchronous code in user level namespace.
+    """
+    from types import CodeType, FunctionType
+    CO_NEWLOCALS = 0x0002
+    code = function.__code__
+    new_code = CodeType(
+        code.co_argcount, 
+        code.co_kwonlyargcount,
+        code.co_nlocals, 
+        code.co_stacksize, 
+        code.co_flags & ~CO_NEWLOCALS,
+        code.co_code, 
+        code.co_consts,
+        code.co_names, 
+        code.co_varnames, 
+        code.co_filename, 
+        code.co_name, 
+        code.co_firstlineno, 
+        code.co_lnotab, 
+        code.co_freevars, 
+        code.co_cellvars
+    )
+    return FunctionType(new_code, globals(), function.__name__, function.__defaults__)
+
+
+if sys.version_info > (3,5):
+    from .async_helpers import (_asyncio_runner, _curio_runner, _trio_runner,
+                                _should_be_async, _asyncify
+                                )
+else :
+    _asyncio_runner = _curio_runner = _trio_runner = None
+
+    def _should_be_async(whatever:str)->bool:
+        return False
+
+
+def _ast_asyncify(cell:str, wrapper_name:str) -> ast.Module:
+    """
+    Parse a cell with top-level await and modify the AST to be able to run it later.
+
+    Parameter
+    ---------
+
+    cell: str
+        The code cell to asyncronify
+    wrapper_name: str
+        The name of the function to be used to wrap the passed `cell`. It is
+        advised to **not** use a python identifier in order to not pollute the
+        global namespace in which the function will be ran.
+
+    Return
+    ------
+
+    A module object AST containing **one** function named `wrapper_name`.
+
+    The given code is wrapped in a async-def function, parsed into an AST, and
+    the resulting function definition AST is modified to return the last
+    expression.
+
+    The last expression or await node is moved into a return statement at the
+    end of the function, and removed from its original location. If the last
+    node is not Expr or Await nothing is done.
+
+    The function `__code__` will need to be later modified  (by
+    ``removed_co_newlocals``) in a subsequent step to not create new `locals()`
+    meaning that the local and global scope are the same, ie as if the body of
+    the function was at module level.
+
+    Lastly a call to `locals()` is made just before the last expression of the
+    function, or just after the last assignment or statement to make sure the
+    global dict is updated as python function work with a local fast cache which
+    is updated only on `local()` calls.
+    """
+
+    from ast import Expr, Await, Return
+    tree = ast.parse(_asyncify(cell))
+
+    function_def = tree.body[0]
+    function_def.name = wrapper_name
+    try_block = function_def.body[0]
+    lastexpr = try_block.body[-1]
+    if isinstance(lastexpr, (Expr, Await)):
+        try_block.body[-1] = Return(lastexpr.value)
+    ast.fix_missing_locations(tree)
+    return tree
+#-----------------------------------------------------------------------------
 # Globals
 #-----------------------------------------------------------------------------
 
@@ -258,6 +356,40 class InteractiveShell(SingletonConfigurable):
         """
     ).tag(config=True)
 
+    autoawait = Bool(True, help=
+        """
+        Automatically run await statement in the top level repl.
+        """
+    ).tag(config=True)
+
+    loop_runner_map ={
+        'asyncio':_asyncio_runner,
+        'curio':_curio_runner,
+        'trio':_trio_runner,
+    }
+
+    loop_runner = Any(default_value="IPython.core.interactiveshell._asyncio_runner",
+        allow_none=True,
+        help="""Select the loop runner that will be used to execute top-level asynchronous code"""
+    ).tag(config=True)
+
+    @default('loop_runner')
+    def _default_loop_runner(self):
+        return import_item("IPython.core.interactiveshell._asyncio_runner")
+
+    @validate('loop_runner')
+    def _import_runner(self, proposal):
+        if isinstance(proposal.value, str):
+            if proposal.value in self.loop_runner_map:
+                return self.loop_runner_map[proposal.value]
+            runner = import_item(proposal.value)
+            if not callable(runner):
+                raise ValueError('loop_runner must be callable')
+            return runner
+        if not callable(proposal.value):
+            raise ValueError('loop_runner must be callable')
+        return proposal.value
+
     automagic = Bool(True, help=
         """
         Enable magic commands to be called without the leading %.
@@ -1449,6 +1581,7 class InteractiveShell(SingletonConfigurable):
         parent = None
         obj = None
 
+
         # Look for the given name by splitting it in parts.  If the head is
         # found, then we look for all the remaining parts as members, and only
         # declare success if we can find them all.
@@ -1984,7 +2117,6 class InteractiveShell(SingletonConfigurable):
         self.set_hook('complete_command', cd_completer, str_key = '%cd')
         self.set_hook('complete_command', reset_completer, str_key = '%reset')
 
-
     @skip_doctest
     def complete(self, text, line=None, cursor_pos=None):
         """Return the completed text and a list of completions.
@@ -2667,14 +2799,36 class InteractiveShell(SingletonConfigurable):
         return result
 
     def _run_cell(self, raw_cell, store_history, silent, shell_futures):
-        """Internal method to run a complete IPython cell.
+        """Internal method to run a complete IPython cell."""
+        return self.loop_runner(
+            self.run_cell_async(
+                raw_cell,
+                store_history=store_history,
+                silent=silent,
+                shell_futures=shell_futures,
+            )
+        )
+
+    @asyncio.coroutine
+    def run_cell_async(self, raw_cell, store_history=False, silent=False, shell_futures=True):
+        """Run a complete IPython cell asynchronously.
 
         Parameters
         ----------
         raw_cell : str
+          The code (including IPython code such as %magic functions) to run.
         store_history : bool
+          If True, the raw and translated cell will be stored in IPython's
+          history. For user code calling back into IPython's machinery, this
+          should be set to False.
         silent : bool
+          If True, avoid side-effects, such as implicit displayhooks and
+          and logging.  silent=True forces store_history=False.
         shell_futures : bool
+          If True, the code will share future statements with the interactive
+          shell. It will both be affected by previous __future__ imports, and
+          any __future__ imports in the code will affect the shell. If False,
+          __future__ imports are not shared in either direction.
 
         Returns
         -------
@@ -2749,13 +2903,33 class InteractiveShell(SingletonConfigurable):
         # compiler
         compiler = self.compile if shell_futures else CachingCompiler()
 
+        _run_async = False
+
         with self.builtin_trap:
             cell_name = self.compile.cache(cell, self.execution_count)
 
             with self.display_trap:
                 # Compile to bytecode
                 try:
-                    code_ast = compiler.ast_parse(cell, filename=cell_name)
+                    if self.autoawait and _should_be_async(cell):
+                        # the code AST below will not be user code: we wrap it
+                        # in an `async def`. This will likely make some AST
+                        # transformer below miss some transform opportunity and
+                        # introduce a small coupling to run_code (in which we
+                        # bake some assumptions of what _ast_asyncify returns.
+                        # they are ways around (like grafting part of the ast
+                        # later:
+                        #    - Here, return code_ast.body[0].body[1:-1], as well
+                        #    as last expression in  return statement which is
+                        #    the user code part.
+                        #    - Let it go through the AST transformers, and graft
+                        #    - it back after the AST transform
+                        # But that seem unreasonable, at least while we
+                        # do not need it.
+                        code_ast = _ast_asyncify(cell, 'async-def-wrapper')
+                        _run_async = True
+                    else:
+                        code_ast = compiler.ast_parse(cell, filename=cell_name)
                 except self.custom_exceptions as e:
                     etype, value, tb = sys.exc_info()
                     self.CustomTB(etype, value, tb)
@@ -2780,9 +2954,11 class InteractiveShell(SingletonConfigurable):
                 self.displayhook.exec_result = result
 
                 # Execute the user code
-                interactivity = 'none' if silent else self.ast_node_interactivity
-                has_raised = self.run_ast_nodes(code_ast.body, cell_name,
-                   interactivity=interactivity, compiler=compiler, result=result)
+                interactivity = "none" if silent else self.ast_node_interactivity
+                if _run_async:
+                    interactivity = 'async'
+                has_raised = yield from self.run_ast_nodes(code_ast.body, cell_name,
+                       interactivity=interactivity, compiler=compiler, result=result)
                 
                 self.last_execution_succeeded = not has_raised
                 self.last_execution_result = result
@@ -2826,12 +3002,12 class InteractiveShell(SingletonConfigurable):
             except Exception:
                 warn("AST transformer %r threw an error. It will be unregistered." % transformer)
                 self.ast_transformers.remove(transformer)
-        
+
         if self.ast_transformers:
             ast.fix_missing_locations(node)
         return node
-                
 
+    @asyncio.coroutine
     def run_ast_nodes(self, nodelist:ListType[AST], cell_name:str, interactivity='last_expr',
                         compiler=compile, result=None):
         """Run a sequence of AST nodes. The execution mode depends on the
@@ -2852,6 +3028,12 class InteractiveShell(SingletonConfigurable):
           are not displayed) 'last_expr_or_assign' will run the last expression
           or the last assignment. Other values for this parameter will raise a
           ValueError.
+
+          Experimental value: 'async' Will try to run top level interactive
+          async/await code in default runner, this will not respect the
+          interactivty setting and will only run the last node if it is an
+          expression. 
+
         compiler : callable
           A function with the same interface as the built-in compile(), to turn
           the AST nodes into code objects. Default is the built-in compile().
@@ -2880,6 +3062,7 class InteractiveShell(SingletonConfigurable):
                     nodelist.append(nnode)
             interactivity = 'last_expr'
 
+        _async = False
         if interactivity == 'last_expr':
             if isinstance(nodelist[-1], ast.Expr):
                 interactivity = "last"
@@ -2892,20 +3075,32 class InteractiveShell(SingletonConfigurable):
             to_run_exec, to_run_interactive = nodelist[:-1], nodelist[-1:]
         elif interactivity == 'all':
             to_run_exec, to_run_interactive = [], nodelist
+        elif interactivity == 'async':
+            _async = True
         else:
             raise ValueError("Interactivity was %r" % interactivity)
         try:
-            for i, node in enumerate(to_run_exec):
-                mod = ast.Module([node])
-                code = compiler(mod, cell_name, "exec")
-                if self.run_code(code, result):
-                    return True
-
-            for i, node in enumerate(to_run_interactive):
-                mod = ast.Interactive([node])
-                code = compiler(mod, cell_name, "single")
-                if self.run_code(code, result):
+            if _async:
+                # If interactivity is async the semantics of run_code are
+                # completely different Skip usual machinery.
+                mod = ast.Module(nodelist)
+                async_wrapper_code = compiler(mod, 'cell_name', 'exec')
+                exec(async_wrapper_code, self.user_global_ns, self.user_ns)
+                async_code = removed_co_newlocals(self.user_ns.pop('async-def-wrapper')).__code__
+                if (yield from self.run_code(async_code, result, async_=True)):
                     return True
+            else:
+                for i, node in enumerate(to_run_exec):
+                    mod = ast.Module([node])
+                    code = compiler(mod, cell_name, "exec")
+                    if (yield from self.run_code(code, result)):
+                        return True
+
+                for i, node in enumerate(to_run_interactive):
+                    mod = ast.Interactive([node])
+                    code = compiler(mod, cell_name, "single")
+                    if (yield from self.run_code(code, result)):
+                        return True
 
             # Flush softspace
             if softspace(sys.stdout, 0):
@@ -2928,7 +3123,23 class InteractiveShell(SingletonConfigurable):
 
         return False
 
-    def run_code(self, code_obj, result=None):
+    def _async_exec(self, code_obj: types.CodeType, user_ns: dict):
+        """
+        Evaluate an asynchronous code object using a code runner
+
+        Fake asynchronous execution of code_object in a namespace via a proxy namespace.
+
+        Returns coroutine object, which can be executed via async loop runner
+
+        WARNING: The semantics of `async_exec` are quite different from `exec`,
+        in particular you can only pass a single namespace. It also return a
+        handle to the value of the last things returned by code_object.
+        """
+
+        return eval(code_obj, user_ns)
+
+    @asyncio.coroutine
+    def run_code(self, code_obj, result=None, *, async_=False):
         """Execute a code object.
 
         When an exception occurs, self.showtraceback() is called to display a
@@ -2940,6 +3151,8 class InteractiveShell(SingletonConfigurable):
           A compiled code object, to be executed
         result : ExecutionResult, optional
           An object to store exceptions that occur during execution.
+        async_ :  Bool (Experimental)
+          Attempt to run top-level asynchronous code in a default loop.
 
         Returns
         -------
@@ -2957,8 +3170,12 class InteractiveShell(SingletonConfigurable):
         try:
             try:
                 self.hooks.pre_run_code_hook()
-                #rprint('Running code', repr(code_obj)) # dbg
-                exec(code_obj, self.user_global_ns, self.user_ns)
+                if async_:
+                    last_expr = (yield from self._async_exec(code_obj, self.user_ns))
+                    code = compile('last_expr', 'fake', "single")
+                    exec(code, {'last_expr': last_expr})
+                else:
+                    exec(code_obj, self.user_global_ns, self.user_ns)
             finally:
                 # Reset our crash handler in place
                 sys.excepthook = old_excepthook

IPython/core/magics/basic.py

@@ -2,19 +2,20
 
 
 import argparse
-import textwrap
+from logging import error
 import io
-import sys
 from pprint import pformat
+import textwrap
+import sys
+from warnings import warn
 
+from traitlets.utils.importstring import import_item
 from IPython.core import magic_arguments, page
 from IPython.core.error import UsageError
 from IPython.core.magic import Magics, magics_class, line_magic, magic_escapes
 from IPython.utils.text import format_screen, dedent, indent
 from IPython.testing.skipdoctest import skip_doctest
 from IPython.utils.ipstruct import Struct
-from warnings import warn
-from logging import error
 
 
 class MagicsDisplay(object):
@@ -379,6 +380,64 Currently the magic system has the following functions:""",
             xmode_switch_err('user')
 
     @line_magic
+    def autoawait(self, parameter_s):
+        """
+        Allow to change the status of the autoawait option.
+
+        This allow you to set a specific asynchronous code runner.
+
+        If no value is passed, print the currently used asynchronous integration
+        and whether it is activated.
+
+        It can take a number of value evaluated in the following order:
+
+        - False/false/off deactivate autoawait integration
+        - True/true/on activate autoawait integration using configured default
+          loop
+        - asyncio/curio/trio activate autoawait integration and use integration
+          with said library.
+
+        If the passed parameter does not match any of the above and is a python
+        identifier, get said object from user namespace and set it as the
+        runner, and activate autoawait.
+
+        If the object is a fully qualified object name, attempt to import it and
+        set it as the runner, and activate autoawait."""
+
+        param = parameter_s.strip()
+        d = {True: "on", False: "off"}
+
+        if not param:
+            print("IPython autoawait is `{}`, and set to use `{}`".format(
+                d[self.shell.autoawait],
+                self.shell.loop_runner
+            ))
+            return None
+
+        if param.lower() in ('false', 'off'):
+            self.shell.autoawait = False
+            return None
+        if param.lower() in ('true', 'on'):
+            self.shell.autoawait = True
+            return None
+
+        if param in self.shell.loop_runner_map:
+            self.shell.loop_runner = param
+            self.shell.autoawait = True
+            return None
+
+        if param in self.shell.user_ns :
+            self.shell.loop_runner = self.shell.user_ns[param]
+            self.shell.autoawait = True
+            return None
+
+        runner = import_item(param)
+
+        self.shell.loop_runner = runner
+        self.shell.autoawait = True
+
+
+    @line_magic
     def pip(self, args=''):
         """
         Intercept usage of ``pip`` in IPython and direct user to run command outside of IPython.

IPython/terminal/embed.py

@@ -19,6 +19,23 from IPython.terminal.ipapp import load_default_config
 from traitlets import Bool, CBool, Unicode
 from IPython.utils.io import ask_yes_no
 
+from contextlib import contextmanager
+
+_sentinel = object()
+@contextmanager
+def new_context():
+    import trio._core._run as tcr
+    old_runner = getattr(tcr.GLOBAL_RUN_CONTEXT, 'runner', _sentinel)
+    old_task = getattr(tcr.GLOBAL_RUN_CONTEXT, 'task', None)
+    if old_runner is not _sentinel:
+        del tcr.GLOBAL_RUN_CONTEXT.runner
+    tcr.GLOBAL_RUN_CONTEXT.task = None
+    yield
+    if old_runner is not _sentinel:
+        tcr.GLOBAL_RUN_CONTEXT.runner = old_runner
+    tcr.GLOBAL_RUN_CONTEXT.task = old_task
+
+
 class KillEmbedded(Exception):pass
 
 # kept for backward compatibility as IPython 6 was released with
@@ -366,6 +383,9 def embed(**kwargs):
         config = load_default_config()
         config.InteractiveShellEmbed = config.TerminalInteractiveShell
         kwargs['config'] = config
+    using = kwargs.get('using', 'trio')
+    if using :
+        kwargs['config'].update({'TerminalInteractiveShell':{'loop_runner':using, 'colors':'NoColor'}})
     #save ps1/ps2 if defined
     ps1 = None
     ps2 = None
@@ -380,11 +400,12 def embed(**kwargs):
         cls = type(saved_shell_instance)
         cls.clear_instance()
     frame = sys._getframe(1)
-    shell = InteractiveShellEmbed.instance(_init_location_id='%s:%s' % (
-        frame.f_code.co_filename, frame.f_lineno), **kwargs)
-    shell(header=header, stack_depth=2, compile_flags=compile_flags,
-          _call_location_id='%s:%s' % (frame.f_code.co_filename, frame.f_lineno))
-    InteractiveShellEmbed.clear_instance()
+    with new_context():
+        shell = InteractiveShellEmbed.instance(_init_location_id='%s:%s' % (
+            frame.f_code.co_filename, frame.f_lineno), **kwargs)
+        shell(header=header, stack_depth=2, compile_flags=compile_flags,
+            _call_location_id='%s:%s' % (frame.f_code.co_filename, frame.f_lineno))
+        InteractiveShellEmbed.clear_instance()
     #restore previous instance
     if saved_shell_instance is not None:
         cls = type(saved_shell_instance)

IPython/terminal/tests/test_embed.py

@@ -72,6 +72,7 def test_nest_embed():
 
     child = pexpect.spawn(sys.executable, ['-m', 'IPython', '--colors=nocolor'],
                           env=env)
+    child.timeout = 5
     child.expect(ipy_prompt)
     child.sendline("import IPython")
     child.expect(ipy_prompt)
@@ -86,7 +87,8 def test_nest_embed():
     except pexpect.TIMEOUT as e:
         print(e)
         #child.interact()
-    child.sendline("embed1 = get_ipython()"); child.expect(ipy_prompt)
+    child.sendline("embed1 = get_ipython()")
+    child.expect(ipy_prompt)
     child.sendline("print('true' if embed1 is not ip0 else 'false')")
     assert(child.expect(['true\r\n', 'false\r\n']) == 0)
     child.expect(ipy_prompt)
@@ -103,7 +105,8 def test_nest_embed():
     except pexpect.TIMEOUT as e:
         print(e)
         #child.interact()
-    child.sendline("embed2 = get_ipython()"); child.expect(ipy_prompt)
+    child.sendline("embed2 = get_ipython()")
+    child.expect(ipy_prompt)
     child.sendline("print('true' if embed2 is not embed1 else 'false')")
     assert(child.expect(['true\r\n', 'false\r\n']) == 0)
     child.expect(ipy_prompt)

docs/source/conf.py

@@ -143,7 +143,8 today_fmt = '%B %d, %Y'
 
 # Exclude these glob-style patterns when looking for source files. They are
 # relative to the source/ directory.
-exclude_patterns = ['whatsnew/pr']
+exclude_patterns = ['whatsnew/pr/antigravity-feature.*', 
+                    'whatsnew/pr/incompat-switching-to-perl.*']
 
 
 # If true, '()' will be appended to :func: etc. cross-reference text.

docs/source/interactive/index.rst

@@ -21,6 +21,7 done some work in the classic Python REPL.
    plotting
    reference
    shell
+   autoawait
    tips
    python-ipython-diff
    magics

docs/source/whatsnew/development.rst

@@ -11,6 +11,135 This document describes in-flight development work.
     `docs/source/whatsnew/pr` folder
 
 
+Released .... ...., 2017
+
+
+Need to be updated:
+
+.. toctree::
+   :maxdepth: 2
+   :glob: 
+
+   pr/*
+
+IPython 6 feature a major improvement in the completion machinery which is now
+capable of completing non-executed code. It is also the first version of IPython
+to stop compatibility with Python 2, which is still supported on the bugfix only
+5.x branch. Read below to have a non-exhaustive list of new features.
+
+Make sure you have pip > 9.0 before upgrading. 
+You should be able to update by using:
+
+.. code::
+
+    pip install ipython --upgrade
+
+New completion API and Interface
+--------------------------------
+
+The completer Completion API has seen an overhaul, and the new completer have
+plenty of improvement both from the end users of terminal IPython or for
+consumers of the API.
+
+This new API is capable of pulling completions from :any:`jedi`, thus allowing
+type inference on non-executed code. If :any:`jedi` is installed completion like
+the following are now becoming possible without code evaluation:
+
+    >>> data = ['Number of users', 123_456]
+    ... data[0].<tab>
+
+That is to say, IPython is now capable of inferring that `data[0]` is a string,
+and will suggest completions like `.capitalize`. The completion power of IPython
+will increase with new Jedi releases, and a number of bugs and more completions
+are already available on development version of :any:`jedi` if you are curious.
+
+With the help of prompt toolkit, types of completions can be shown in the
+completer interface:
+
+.. image:: ../_images/jedi_type_inference_60.png
+    :alt: Jedi showing ability to do type inference
+    :align: center
+    :width: 400px
+    :target: ../_images/jedi_type_inference_60.png
+
+The appearance of the completer is controlled by the
+``c.TerminalInteractiveShell.display_completions`` option that will show the
+type differently depending on the value among ``'column'``, ``'multicolumn'``
+and ``'readlinelike'``
+
+The use of Jedi also full fill a number of request and fix a number of bugs
+like case insensitive completion, completion after division operator: See
+:ghpull:`10182`.
+
+Extra patches and updates will be needed to the :mod:`ipykernel` package for
+this feature to be available to other clients like jupyter Notebook, Lab,
+Nteract, Hydrogen...
+
+The use of Jedi can is barely noticeable on recent enough machines, but can be
+feel on older ones,  in cases were Jedi behavior need to be adjusted, the amount
+of time given to Jedi to compute type inference can be adjusted with
+``c.IPCompleter.jedi_compute_type_timeout``, with object whose type were not
+inferred will be shown as ``<unknown>``. Jedi can also be completely deactivated
+by using the ``c.Completer.use_jedi=False`` option.
+
+
+The old ``Completer.complete()`` API is waiting deprecation and should be
+replaced replaced by ``Completer.completions()`` in a near future. Feedback on
+the current state of the API and suggestions welcome.
+
+Python 3 only codebase
+----------------------
+
+One of the large challenges in IPython 6.0 has been the adoption of a pure
+Python 3 code base, which lead us to great length to upstream patches in pip,
+pypi and warehouse to make sure Python 2 system still upgrade to the latest
+compatible Python version compatible.
+
+We remind our Python 2 users that IPython 5 is still compatible with Python 2.7,
+still maintained and get regular releases. Using pip 9+, upgrading IPython will
+automatically upgrade to the latest version compatible with your system.
+
+.. warning::
+
+  If you are on a system using an older verison of pip on Python 2, pip may
+  still install IPython 6.0 on your system, and IPython will refuse to start. 
+  You can fix this by ugrading pip, and reinstalling ipython, or forcing pip to
+  install an earlier version: ``pip install 'ipython<6'``
+
+The ability to use only Python 3 on the code base of IPython has bring a number
+of advantage. Most of the newly written code make use of `optional function type
+anotation <https://www.python.org/dev/peps/pep-0484/>`_ leading to clearer code
+and better documentation.
+
+The total size of the repository has also for a first time between releases
+(excluding the big split for 4.0) decreased by about 1500 lines, potentially
+quite a bit more codewide as some documents like this one are append only and
+are about 300 lines long.
+
+The removal as of Python2/Python3 shim layer has made the code quite clearer and
+more idiomatic in a number of location, and much friendlier to work with and
+understand. We hope to further embrace Python 3 capability in the next release
+cycle and introduce more of the Python 3 only idioms (yield from, kwarg only,
+general unpacking) in the code base of IPython, and see if we can take advantage
+of these as well to improve user experience with better error messages and
+hints.
+
+
+Miscs improvements
+------------------
+
+
+- The :cellmagic:`capture` magic can now capture the result of a cell (from an
+  expression on the last line), as well as printed and displayed output.
+  :ghpull:`9851`.
+
+- Pressing Ctrl-Z in the terminal debugger now suspends IPython, as it already
+  does in the main terminal prompt.
+
+- autoreload can now reload ``Enum``. See :ghissue:`10232` and :ghpull:`10316`
+
+- IPython.display has gained a :any:`GeoJSON <IPython.display.GeoJSON>` object.
+  :ghpull:`10288` and :ghpull:`10253`
 
 .. DO NOT EDIT THIS LINE BEFORE RELEASE. FEATURE INSERTION POINT.
 

setup.py

@@ -201,6 +201,7 install_requires = [
 
 extras_require.update({
     ':python_version == "3.4"': ['typing'],
+    ':python_version >= "3.5"': ['trio', 'curio'],
     ':sys_platform != "win32"': ['pexpect'],
     ':sys_platform == "darwin"': ['appnope'],
     ':sys_platform == "win32"': ['colorama'],