Merge pull request #11713 from Carreau/async-exec

Async exec in Python 3.8

IPython/core/async_helpers.py

@@ -13,6 +13,7 b' Python semantics.'
 
 import ast
 import sys
+import inspect
 from textwrap import dedent, indent
 
 
@@ -98,6 +99,8 b' class _AsyncSyntaxErrorVisitor(ast.NodeVisitor):'
     is erroneously allowed (e.g. yield or return at the top level)
     """
     def __init__(self):
+        if sys.version_info >= (3,8):
+            raise ValueError('DEPRECATED in Python 3.8+')
         self.depth = 0
         super().__init__()
 
@@ -146,12 +149,16 b' def _should_be_async(cell: str) -> bool:'
     Not handled yet: If the block of code has a return statement as  the top
     level, it will be seen as async. This is a know limitation.
     """
-
+    if sys.version_info > (3, 8):
+        try:
+            code = compile(cell, "<>", "exec", flags=getattr(ast,'PyCF_ALLOW_TOP_LEVEL_AWAIT', 0x0))
+            return inspect.CO_COROUTINE & code.co_flags == inspect.CO_COROUTINE
+        except SyntaxError:
+            return False
     try:
         # we can't limit ourself to ast.parse, as it __accepts__ to parse on
         # 3.7+, but just does not _compile_
-        compile(cell, "<>", "exec")
-        return False
+        code = compile(cell, "<>", "exec")
     except SyntaxError:
         try:
             parse_tree = _async_parse_cell(cell)

IPython/core/compilerop.py

@@ -35,6 +35,7 b' import hashlib'
 import linecache
 import operator
 import time
+from contextlib import contextmanager
 
 #-----------------------------------------------------------------------------
 # Constants
@@ -134,6 +135,21 b' class CachingCompiler(codeop.Compile):'
         linecache._ipython_cache[name] = entry
         return name
 
+    @contextmanager
+    def extra_flags(self, flags):
+        ## bits that we'll set to 1
+        turn_on_bits = ~self.flags & flags
+
+
+        self.flags = self.flags | flags
+        try:
+            yield
+        finally:
+            # turn off only the bits we turned on so that something like
+            # __future__ that set flags stays. 
+            self.flags &= ~turn_on_bits
+
+
 def check_linecache_ipython(*args):
     """Call linecache.checkcache() safely protecting our cached values.
     """

IPython/core/interactiveshell.py

@@ -17,6 +17,7 b' import asyncio'
 import atexit
 import builtins as builtin_mod
 import functools
+import inspect
 import os
 import re
 import runpy
@@ -165,7 +166,6 b' def removed_co_newlocals(function:types.FunctionType) -> types.FunctionType:'
 # we still need to run things using the asyncio eventloop, but there is no
 # async integration
 from .async_helpers import (_asyncio_runner,  _asyncify, _pseudo_sync_runner)
-
 if sys.version_info > (3, 5):
     from .async_helpers import _curio_runner, _trio_runner, _should_be_async
 else :
@@ -214,6 +214,8 b' def _ast_asyncify(cell:str, wrapper_name:str) -> ast.Module:'
     """
 
     from ast import Expr, Await, Return
+    if sys.version_info >= (3,8):
+        return ast.parse(cell)
     tree = ast.parse(_asyncify(cell))
 
     function_def = tree.body[0]
@@ -2909,8 +2911,7 b' class InteractiveShell(SingletonConfigurable):'
             return False
         return _should_be_async(cell)
 
-    @asyncio.coroutine
-    def run_cell_async(self, raw_cell: str, store_history=False, silent=False, shell_futures=True) -> ExecutionResult:
+    async def run_cell_async(self, raw_cell: str, store_history=False, silent=False, shell_futures=True) -> ExecutionResult:
         """Run a complete IPython cell asynchronously.
 
         Parameters
@@ -3002,23 +3003,26 b' class InteractiveShell(SingletonConfigurable):'
             with self.display_trap:
                 # Compile to bytecode
                 try:
-                    if self.autoawait and _should_be_async(cell):
-                        # the code AST below will not be user code: we wrap it
-                        # in an `async def`. This will likely make some AST
-                        # transformer below miss some transform opportunity and
-                        # introduce a small coupling to run_code (in which we
-                        # bake some assumptions of what _ast_asyncify returns.
-                        # they are ways around (like grafting part of the ast
-                        # later:
-                        #    - Here, return code_ast.body[0].body[1:-1], as well
-                        #    as last expression in  return statement which is
-                        #    the user code part.
-                        #    - Let it go through the AST transformers, and graft
-                        #    - it back after the AST transform
-                        # But that seem unreasonable, at least while we
-                        # do not need it.
-                        code_ast = _ast_asyncify(cell, 'async-def-wrapper')
-                        _run_async = True
+                    if sys.version_info < (3,8) and self.autoawait:
+                        if _should_be_async(cell):
+                            # the code AST below will not be user code: we wrap it
+                            # in an `async def`. This will likely make some AST
+                            # transformer below miss some transform opportunity and
+                            # introduce a small coupling to run_code (in which we
+                            # bake some assumptions of what _ast_asyncify returns.
+                            # they are ways around (like grafting part of the ast
+                            # later:
+                            #    - Here, return code_ast.body[0].body[1:-1], as well
+                            #    as last expression in  return statement which is
+                            #    the user code part.
+                            #    - Let it go through the AST transformers, and graft
+                            #    - it back after the AST transform
+                            # But that seem unreasonable, at least while we
+                            # do not need it.
+                            code_ast = _ast_asyncify(cell, 'async-def-wrapper')
+                            _run_async = True
+                        else:
+                            code_ast = compiler.ast_parse(cell, filename=cell_name)
                     else:
                         code_ast = compiler.ast_parse(cell, filename=cell_name)
                 except self.custom_exceptions as e:
@@ -3049,7 +3053,7 b' class InteractiveShell(SingletonConfigurable):'
                 if _run_async:
                     interactivity = 'async'
 
-                has_raised = yield from self.run_ast_nodes(code_ast.body, cell_name,
+                has_raised = await self.run_ast_nodes(code_ast.body, cell_name,
                        interactivity=interactivity, compiler=compiler, result=result)
 
                 self.last_execution_succeeded = not has_raised
@@ -3129,8 +3133,7 b' class InteractiveShell(SingletonConfigurable):'
             ast.fix_missing_locations(node)
         return node
 
-    @asyncio.coroutine
-    def run_ast_nodes(self, nodelist:ListType[AST], cell_name:str, interactivity='last_expr',
+    async def run_ast_nodes(self, nodelist:ListType[AST], cell_name:str, interactivity='last_expr',
                         compiler=compile, result=None):
         """Run a sequence of AST nodes. The execution mode depends on the
         interactivity parameter.
@@ -3169,6 +3172,7 b' class InteractiveShell(SingletonConfigurable):'
         """
         if not nodelist:
             return
+
         if interactivity == 'last_expr_or_assign':
             if isinstance(nodelist[-1], _assign_nodes):
                 asg = nodelist[-1]
@@ -3198,10 +3202,15 b' class InteractiveShell(SingletonConfigurable):'
         elif interactivity == 'all':
             to_run_exec, to_run_interactive = [], nodelist
         elif interactivity == 'async':
+            to_run_exec, to_run_interactive = [], nodelist
             _async = True
         else:
             raise ValueError("Interactivity was %r" % interactivity)
+
         try:
+            if _async and sys.version_info > (3,8):
+                raise ValueError("This branch should never happen on Python 3.8 and above, "
+                                 "please try to upgrade IPython and open a bug report with your case.")
             if _async:
                 # If interactivity is async the semantics of run_code are
                 # completely different Skip usual machinery.
@@ -3209,19 +3218,34 b' class InteractiveShell(SingletonConfigurable):'
                 async_wrapper_code = compiler(mod, cell_name, 'exec')
                 exec(async_wrapper_code, self.user_global_ns, self.user_ns)
                 async_code = removed_co_newlocals(self.user_ns.pop('async-def-wrapper')).__code__
-                if (yield from self.run_code(async_code, result, async_=True)):
+                if (await self.run_code(async_code, result, async_=True)):
                     return True
             else:
-                for i, node in enumerate(to_run_exec):
-                    mod = Module([node], [])
-                    code = compiler(mod, cell_name, "exec")
-                    if (yield from self.run_code(code, result)):
-                        return True
-
-                for i, node in enumerate(to_run_interactive):
-                    mod = ast.Interactive([node])
-                    code = compiler(mod, cell_name, "single")
-                    if (yield from self.run_code(code, result)):
+                if sys.version_info > (3, 8):
+                    def compare(code):
+                        is_async = (inspect.CO_COROUTINE & code.co_flags == inspect.CO_COROUTINE)
+                        return is_async
+                else:
+                    def compare(code):
+                        return _async
+
+                # refactor that to just change the mod constructor.
+                to_run = []
+                for node in to_run_exec:
+                    to_run.append((node, 'exec'))
+
+                for node in to_run_interactive:
+                    to_run.append((node, 'single'))
+
+                for node,mode in to_run:
+                    if mode == 'exec':
+                        mod = Module([node], [])
+                    elif mode == 'single':
+                        mod = ast.Interactive([node])
+                    with compiler.extra_flags(getattr(ast, 'PyCF_ALLOW_TOP_LEVEL_AWAIT', 0x0) if self.autoawait else 0x0):
+                        code = compiler(mod, cell_name, mode)
+                        asy = compare(code)
+                    if (await self.run_code(code, result,  async_=asy)):
                         return True
 
             # Flush softspace
@@ -3260,8 +3284,7 b' class InteractiveShell(SingletonConfigurable):'
 
         return eval(code_obj, user_ns)
 
-    @asyncio.coroutine
-    def run_code(self, code_obj, result=None, *, async_=False):
+    async def run_code(self, code_obj, result=None, *, async_=False):
         """Execute a code object.
 
         When an exception occurs, self.showtraceback() is called to display a
@@ -3292,10 +3315,12 b' class InteractiveShell(SingletonConfigurable):'
         try:
             try:
                 self.hooks.pre_run_code_hook()
-                if async_:
-                    last_expr = (yield from self._async_exec(code_obj, self.user_ns))
+                if async_ and sys.version_info < (3,8):
+                    last_expr = (await self._async_exec(code_obj, self.user_ns))
                     code = compile('last_expr', 'fake', "single")
                     exec(code, {'last_expr': last_expr})
+                elif async_ :
+                    await eval(code_obj, self.user_global_ns, self.user_ns)
                 else:
                     exec(code_obj, self.user_global_ns, self.user_ns)
             finally: