docs cleanup, reformat code, remove dead code.

IPython/core/async_helpers.py

@@ -1,28 +1,35 b''
 """
 Async helper function that are invalid syntax on Python 3.5 and below.
 
-Known limitation and possible improvement.
+This code is best effort, and may have edge cases not behaving as expected. In
+particular it contain a number of heuristics to detect whether code is
+effectively async and need to run in an event loop or not.
 
-Top level code that contain a return statement (instead of, or in addition to
-await) will be detected as requiring being wrapped in async calls. This should
-be prevented as early return will not work.
+Some constructs (like top-level `return`, or `yield`) are taken care of
+explicitly to actually raise a SyntaxError and stay as close as possible to
+Python semantics.
 """
 
 
 import ast
 import sys
-import inspect
 from textwrap import dedent, indent
-from types import CodeType
 
 
-def _asyncio_runner(coro):
-    """
-    Handler for asyncio autoawait
-    """
-    import asyncio
+class _AsyncIORunner:
+
+    def __call__(self, coro):
+        """
+        Handler for asyncio autoawait
+        """
+        import asyncio
+
+        return asyncio.get_event_loop().run_until_complete(coro)
 
-    return asyncio.get_event_loop().run_until_complete(coro)
+    def __str__(self):
+        return 'asyncio'
+
+_asyncio_runner = _AsyncIORunner()
 
 
 def _curio_runner(coroutine):
@@ -36,12 +43,14 b' def _curio_runner(coroutine):'
 
 def _trio_runner(async_fn):
     import trio
+
     async def loc(coro):
         """
         We need the dummy no-op async def to protect from
         trio's internal. See https://github.com/python-trio/trio/issues/89
         """
         return await coro
+
     return trio.run(loc, async_fn)
 
 
@@ -60,7 +69,9 b' def _pseudo_sync_runner(coro):'
         return exc.value
     else:
         # TODO: do not raise but return an execution result with the right info.
-        raise RuntimeError("{coro_name!r} needs a real async loop".format(coro_name=coro.__name__))
+        raise RuntimeError(
+            "{coro_name!r} needs a real async loop".format(coro_name=coro.__name__)
+        )
 
 
 def _asyncify(code: str) -> str:
@@ -69,7 +80,7 b' def _asyncify(code: str) -> str:'
     And setup a bit of context to run it later.
     """
     res = dedent(
-    """
+        """
     async def __wrapper__():
         try:
     {usercode}
@@ -86,12 +97,13 b' class _AsyncSyntaxErrorVisitor(ast.NodeVisitor):'
     the implementation involves wrapping the repl in an async function, it
     is erroneously allowed (e.g. yield or return at the top level)
     """
+
     def generic_visit(self, node):
         func_types = (ast.FunctionDef, ast.AsyncFunctionDef)
         invalid_types = (ast.Return, ast.Yield, ast.YieldFrom)
 
         if isinstance(node, func_types):
-            return      # Don't recurse into functions
+            return  # Don't recurse into functions
         elif isinstance(node, invalid_types):
             raise SyntaxError()
         else:

IPython/core/interactiveshell.py

@@ -2806,7 +2806,7 b' class InteractiveShell(SingletonConfigurable):'
                 self.events.trigger('post_run_cell', result)
         return result
 
-    def _run_cell(self, raw_cell, store_history, silent, shell_futures):
+    def _run_cell(self, raw_cell:str, store_history:bool, silent:bool, shell_futures:bool):
         """Internal method to run a complete IPython cell."""
         coro = self.run_cell_async(
             raw_cell,
@@ -2815,11 +2815,16 b' class InteractiveShell(SingletonConfigurable):'
             shell_futures=shell_futures,
         )
 
+        # run_cell_async is async, but may not actually need and eventloop.
+        # when this is the case, we want to run it using the pseudo_sync_runner
+        # so that code can invoke eventloops (for example via the %run , and
+        # `%paste` magic.
         try:
             interactivity = coro.send(None)
         except StopIteration as exc:
             return exc.value
 
+        # if code was not async, sending `None` was actually executing the code.
         if isinstance(interactivity, ExecutionResult):
             return interactivity
 
@@ -3159,7 +3164,6 b' class InteractiveShell(SingletonConfigurable):'
 
         return eval(code_obj, user_ns)
 
-    @asyncio.coroutine
     def run_code(self, code_obj, result=None, *, async_=False):
         """Execute a code object.
 

IPython/core/magics/basic.py

@@ -632,7 +632,12 b' class AsyncMagics(BasicMagics):'
         runner, and activate autoawait. 
 
         If the object is a fully qualified object name, attempt to import it and
-        set it as the runner, and activate autoawait."""
+        set it as the runner, and activate autoawait.
+        
+        
+        The exact behavior of autoawait is experimental and subject to change
+        across version of IPython and Python.
+        """
 
         param = parameter_s.strip()
         d = {True: "on", False: "off"}

IPython/terminal/embed.py

@@ -19,23 +19,6 b' from IPython.terminal.ipapp import load_default_config'
 from traitlets import Bool, CBool, Unicode
 from IPython.utils.io import ask_yes_no
 
-from contextlib import contextmanager
-
-_sentinel = object()
-@contextmanager
-def new_context():
-    import trio._core._run as tcr
-    old_runner = getattr(tcr.GLOBAL_RUN_CONTEXT, 'runner', _sentinel)
-    old_task = getattr(tcr.GLOBAL_RUN_CONTEXT, 'task', None)
-    if old_runner is not _sentinel:
-        del tcr.GLOBAL_RUN_CONTEXT.runner
-    tcr.GLOBAL_RUN_CONTEXT.task = None
-    yield
-    if old_runner is not _sentinel:
-        tcr.GLOBAL_RUN_CONTEXT.runner = old_runner
-    tcr.GLOBAL_RUN_CONTEXT.task = old_task
-
-
 class KillEmbedded(Exception):pass
 
 # kept for backward compatibility as IPython 6 was released with
@@ -400,12 +383,11 b' def embed(**kwargs):'
         cls = type(saved_shell_instance)
         cls.clear_instance()
     frame = sys._getframe(1)
-    with new_context():
-        shell = InteractiveShellEmbed.instance(_init_location_id='%s:%s' % (
-            frame.f_code.co_filename, frame.f_lineno), **kwargs)
-        shell(header=header, stack_depth=2, compile_flags=compile_flags,
-            _call_location_id='%s:%s' % (frame.f_code.co_filename, frame.f_lineno))
-        InteractiveShellEmbed.clear_instance()
+    shell = InteractiveShellEmbed.instance(_init_location_id='%s:%s' % (
+        frame.f_code.co_filename, frame.f_lineno), **kwargs)
+    shell(header=header, stack_depth=2, compile_flags=compile_flags,
+        _call_location_id='%s:%s' % (frame.f_code.co_filename, frame.f_lineno))
+    InteractiveShellEmbed.clear_instance()
     #restore previous instance
     if saved_shell_instance is not None:
         cls = type(saved_shell_instance)

docs/source/interactive/autoawait.rst

@@ -3,6 +3,11 b''
 Asynchronous in REPL: Autoawait
 ===============================
 
+.. note::
+
+   This feature is experimental and behavior can change betwen python and
+   IPython version without prior deprecation.
+
 Starting with IPython 7.0, and when user Python 3.6 and above, IPython offer the
 ability to run asynchronous code from the REPL. Constructs which are
 :exc:`SyntaxError` s in the Python REPL can be used seamlessly in IPython.
@@ -12,10 +17,10 b' notebook interface or any other frontend using the Jupyter protocol will need to'
 use a newer version of IPykernel. The details of how async code runs in
 IPykernel will differ between IPython, IPykernel and their versions.
 
-When a supported library is used, IPython will automatically `await` Futures and
-Coroutines in the REPL. This will happen if an :ref:`await <await>` (or any
-other async constructs like async-with, async-for) is use at top level scope, or
-if any structure valid only in `async def
+When a supported library is used, IPython will automatically allow Futures and
+Coroutines in the REPL to be ``await`` ed. This will happen if an :ref:`await
+<await>` (or any other async constructs like async-with, async-for) is use at
+top level scope, or if any structure valid only in `async def
 <https://docs.python.org/3/reference/compound_stmts.html#async-def>`_ function
 context are present. For example, the following being a syntax error in the
 Python REPL::
@@ -58,15 +63,13 b' use the :magic:`%autoawait` magic to toggle the behavior at runtime::'
     In [1]: %autoawait False
 
     In [2]: %autoawait
-    IPython autoawait is `Off`, and set to use `IPython.core.interactiveshell._asyncio_runner`
+    IPython autoawait is `Off`, and set to use `asyncio`
 
 
 
 By default IPython will assume integration with Python's provided
 :mod:`asyncio`, but integration with other libraries is provided. In particular
-we provide experimental integration with the ``curio`` and ``trio`` library, the
-later one being necessary if you require the ability to do nested call of
-IPython's ``embed()`` functionality.
+we provide experimental integration with the ``curio`` and ``trio`` library.
 
 You can switch current integration by using the
 ``c.InteractiveShell.loop_runner`` option or the ``autoawait <name
@@ -111,7 +114,7 b' other features of IPython and various registered extensions. In particular if yo'
 are a direct or indirect user of the AST transformers, these may not apply to
 your code.
 
-When Using command line IPython, the default loop (or runner) does not process
+When using command line IPython, the default loop (or runner) does not process
 in the background, so top level asynchronous code must finish for the REPL to
 allow you to enter more code. As with usual Python semantic, the awaitables are
 started only when awaited for the first time. That is to say, in first example,
@@ -122,25 +125,20 b' Effects on IPython.embed()'
 ==========================
 
 IPython core being asynchronous, the use of ``IPython.embed()`` will now require
-a loop to run. In order to allow ``IPython.embed()`` to be nested, as most event
-loops can't be nested, ``IPython.embed()`` default to a pseudo-synchronous mode,
-where async code is not allowed. This mode is available in classical IPython
-using ``%autoawait sync``
-
-
-
-This affect the ability to nest ``IPython.embed()`` which may
-require you to install alternate IO libraries  like ``curio`` and ``trio`` 
-
+a loop to run. By default IPython will use a fake coroutine runner which should
+allow ``IPython.embed()`` to be nested. Though this will prevent usage of the
+``autoawait`` feature when using IPython embed. 
 
+You can set explicitly a coroutine runner for ``embed()`` if you desire to run
+asynchronous code, the exact behavior is though undefined.
 
 Internals
 =========
 
 As running asynchronous code is not supported in interactive REPL (as of Python
-3.7) we have to rely to a number of complex workaround to allow this to happen.
-It is interesting to understand how this works in order to comprehend potential
-bugs, or provide a custom runner.
+3.7) we have to rely to a number of complex workaround and heuristic to allow
+this to happen. It is interesting to understand how this works in order to
+comprehend potential bugs, or provide a custom runner.
 
 Among the many approaches that are at our disposition, we find only one that
 suited out need. Under the hood we use the code object from a async-def function
@@ -168,8 +166,10 b' On top of the above there are significant modification to the AST of'
 significant overhead to this kind of code.
 
 By default the generated coroutine function will be consumed by Asyncio's
-``loop_runner = asyncio.get_evenloop().run_until_complete()`` method. It is
-though possible to provide your own.
+``loop_runner = asyncio.get_evenloop().run_until_complete()`` method if
+``async`` mode is deemed necessary, otherwise the coroutine will just be
+exhausted in a simple runner. It is though possible to change the default
+runner.
 
 A loop runner is a *synchronous*  function responsible from running a coroutine
 object.
@@ -208,3 +208,6 b' We can set it up by passing it to ``%autoawait``::'
 Asynchronous programming in python (and in particular in the REPL) is still a
 relatively young subject. We expect some code to not behave as you expect, so
 feel free to contribute improvements to this codebase and give us feedback.
+
+We invite you to thoroughly test this feature and report any unexpected behavior
+as well as propose any improvement.

docs/source/whatsnew/pr/await-repl.rst

@@ -23,6 +23,10 b' yourself. To know more read the :ref:`autoawait` section of our docs, see'
     ...
     }
 
+.. note::
+
+   Async integration is experimental code, behavior may change or be removed
+   between Python and IPython versions without warnings.
 
 Integration is by default with `asyncio`, but other libraries can be configured, 
 like ``curio`` or ``trio``, to improve concurrency in the REPL::
@@ -57,13 +61,33 b' See :ref:`autoawait` for more information.'
 Asynchronous code in a Notebook interface or any other frontend using the
 Jupyter Protocol will need further updates of the IPykernel package.
 
+Non-Asynchronous code 
+---------------------
+
+As the internal API of IPython are now asynchronous, IPython need to run under
+an even loop. In order to allow many workflow, (like using the ``%run`` magic,
+or copy_pasting code that explicitly starts/stop event loop), when top-level code
+is detected as not being asynchronous, IPython code is advanced via a
+pseudo-synchronous runner, and will not may not advance pending tasks.
 
 Change to Nested Embed
 ----------------------
 
-The introduction of the ability to run async code had ripple effect on the
-ability to use nested IPython. You may need to install the ``trio`` library
-(version 0.5 at the time of this writing) to
-have this feature working. 
+The introduction of the ability to run async code had some effect on the
+``IPython.embed()`` API. By default embed will not allow you to run asynchronous
+code unless a event loop is specified.
+
+Expected Future changes
+-----------------------
+
+We expect more internal but public IPython function to become ``async``, and
+will likely end up having a persisting event loop while IPython is running.
 
+Thanks
+------
 
+This took more than a year in the making, and the code was rebased a number of
+time leading to commit authorship that may have been lost in the final
+Pull-Request. Huge thanks to many people for contribution, discussion, code,
+documentation, use-case: dalejung, danielballan, ellisonbg, fperez, gnestor,
+minrk, njsmith, pganssle, tacaswell, takluyver , vidartf ... And many other.