From 0e6f90ec04d2e07de5e0c96988974294b3e493f8 2019-06-18 01:21:06 From: Matthias Bussonnier Date: 2019-06-18 01:21:06 Subject: [PATCH] Merge pull request #11713 from Carreau/async-exec Async exec in Python 3.8 --- diff --git a/IPython/core/async_helpers.py b/IPython/core/async_helpers.py index 46d2147..1a7d889 100644 --- a/IPython/core/async_helpers.py +++ b/IPython/core/async_helpers.py @@ -13,6 +13,7 @@ Python semantics. import ast import sys +import inspect from textwrap import dedent, indent @@ -98,6 +99,8 @@ class _AsyncSyntaxErrorVisitor(ast.NodeVisitor): is erroneously allowed (e.g. yield or return at the top level) """ def __init__(self): + if sys.version_info >= (3,8): + raise ValueError('DEPRECATED in Python 3.8+') self.depth = 0 super().__init__() @@ -146,12 +149,16 @@ def _should_be_async(cell: str) -> bool: Not handled yet: If the block of code has a return statement as the top level, it will be seen as async. This is a know limitation. """ - + if sys.version_info > (3, 8): + try: + code = compile(cell, "<>", "exec", flags=getattr(ast,'PyCF_ALLOW_TOP_LEVEL_AWAIT', 0x0)) + return inspect.CO_COROUTINE & code.co_flags == inspect.CO_COROUTINE + except SyntaxError: + return False try: # we can't limit ourself to ast.parse, as it __accepts__ to parse on # 3.7+, but just does not _compile_ - compile(cell, "<>", "exec") - return False + code = compile(cell, "<>", "exec") except SyntaxError: try: parse_tree = _async_parse_cell(cell) diff --git a/IPython/core/compilerop.py b/IPython/core/compilerop.py index 6a055f9..c4771af 100644 --- a/IPython/core/compilerop.py +++ b/IPython/core/compilerop.py @@ -35,6 +35,7 @@ import hashlib import linecache import operator import time +from contextlib import contextmanager #----------------------------------------------------------------------------- # Constants @@ -134,6 +135,21 @@ class CachingCompiler(codeop.Compile): linecache._ipython_cache[name] = entry return name + @contextmanager + def extra_flags(self, flags): + ## bits that we'll set to 1 + turn_on_bits = ~self.flags & flags + + + self.flags = self.flags | flags + try: + yield + finally: + # turn off only the bits we turned on so that something like + # __future__ that set flags stays. + self.flags &= ~turn_on_bits + + def check_linecache_ipython(*args): """Call linecache.checkcache() safely protecting our cached values. """ diff --git a/IPython/core/interactiveshell.py b/IPython/core/interactiveshell.py index 6b6bc20..9a25185 100644 --- a/IPython/core/interactiveshell.py +++ b/IPython/core/interactiveshell.py @@ -17,6 +17,7 @@ import asyncio import atexit import builtins as builtin_mod import functools +import inspect import os import re import runpy @@ -165,7 +166,6 @@ def removed_co_newlocals(function:types.FunctionType) -> types.FunctionType: # we still need to run things using the asyncio eventloop, but there is no # async integration from .async_helpers import (_asyncio_runner, _asyncify, _pseudo_sync_runner) - if sys.version_info > (3, 5): from .async_helpers import _curio_runner, _trio_runner, _should_be_async else : @@ -214,6 +214,8 @@ def _ast_asyncify(cell:str, wrapper_name:str) -> ast.Module: """ from ast import Expr, Await, Return + if sys.version_info >= (3,8): + return ast.parse(cell) tree = ast.parse(_asyncify(cell)) function_def = tree.body[0] @@ -2909,8 +2911,7 @@ class InteractiveShell(SingletonConfigurable): return False return _should_be_async(cell) - @asyncio.coroutine - def run_cell_async(self, raw_cell: str, store_history=False, silent=False, shell_futures=True) -> ExecutionResult: + async def run_cell_async(self, raw_cell: str, store_history=False, silent=False, shell_futures=True) -> ExecutionResult: """Run a complete IPython cell asynchronously. Parameters @@ -3002,23 +3003,26 @@ class InteractiveShell(SingletonConfigurable): with self.display_trap: # Compile to bytecode try: - if self.autoawait and _should_be_async(cell): - # the code AST below will not be user code: we wrap it - # in an `async def`. This will likely make some AST - # transformer below miss some transform opportunity and - # introduce a small coupling to run_code (in which we - # bake some assumptions of what _ast_asyncify returns. - # they are ways around (like grafting part of the ast - # later: - # - Here, return code_ast.body[0].body[1:-1], as well - # as last expression in return statement which is - # the user code part. - # - Let it go through the AST transformers, and graft - # - it back after the AST transform - # But that seem unreasonable, at least while we - # do not need it. - code_ast = _ast_asyncify(cell, 'async-def-wrapper') - _run_async = True + if sys.version_info < (3,8) and self.autoawait: + if _should_be_async(cell): + # the code AST below will not be user code: we wrap it + # in an `async def`. This will likely make some AST + # transformer below miss some transform opportunity and + # introduce a small coupling to run_code (in which we + # bake some assumptions of what _ast_asyncify returns. + # they are ways around (like grafting part of the ast + # later: + # - Here, return code_ast.body[0].body[1:-1], as well + # as last expression in return statement which is + # the user code part. + # - Let it go through the AST transformers, and graft + # - it back after the AST transform + # But that seem unreasonable, at least while we + # do not need it. + code_ast = _ast_asyncify(cell, 'async-def-wrapper') + _run_async = True + else: + code_ast = compiler.ast_parse(cell, filename=cell_name) else: code_ast = compiler.ast_parse(cell, filename=cell_name) except self.custom_exceptions as e: @@ -3049,7 +3053,7 @@ class InteractiveShell(SingletonConfigurable): if _run_async: interactivity = 'async' - has_raised = yield from self.run_ast_nodes(code_ast.body, cell_name, + has_raised = await self.run_ast_nodes(code_ast.body, cell_name, interactivity=interactivity, compiler=compiler, result=result) self.last_execution_succeeded = not has_raised @@ -3129,8 +3133,7 @@ class InteractiveShell(SingletonConfigurable): ast.fix_missing_locations(node) return node - @asyncio.coroutine - def run_ast_nodes(self, nodelist:ListType[AST], cell_name:str, interactivity='last_expr', + async def run_ast_nodes(self, nodelist:ListType[AST], cell_name:str, interactivity='last_expr', compiler=compile, result=None): """Run a sequence of AST nodes. The execution mode depends on the interactivity parameter. @@ -3169,6 +3172,7 @@ class InteractiveShell(SingletonConfigurable): """ if not nodelist: return + if interactivity == 'last_expr_or_assign': if isinstance(nodelist[-1], _assign_nodes): asg = nodelist[-1] @@ -3198,10 +3202,15 @@ class InteractiveShell(SingletonConfigurable): elif interactivity == 'all': to_run_exec, to_run_interactive = [], nodelist elif interactivity == 'async': + to_run_exec, to_run_interactive = [], nodelist _async = True else: raise ValueError("Interactivity was %r" % interactivity) + try: + if _async and sys.version_info > (3,8): + raise ValueError("This branch should never happen on Python 3.8 and above, " + "please try to upgrade IPython and open a bug report with your case.") if _async: # If interactivity is async the semantics of run_code are # completely different Skip usual machinery. @@ -3209,19 +3218,34 @@ class InteractiveShell(SingletonConfigurable): async_wrapper_code = compiler(mod, cell_name, 'exec') exec(async_wrapper_code, self.user_global_ns, self.user_ns) async_code = removed_co_newlocals(self.user_ns.pop('async-def-wrapper')).__code__ - if (yield from self.run_code(async_code, result, async_=True)): + if (await self.run_code(async_code, result, async_=True)): return True else: - for i, node in enumerate(to_run_exec): - mod = Module([node], []) - code = compiler(mod, cell_name, "exec") - if (yield from self.run_code(code, result)): - return True - - for i, node in enumerate(to_run_interactive): - mod = ast.Interactive([node]) - code = compiler(mod, cell_name, "single") - if (yield from self.run_code(code, result)): + if sys.version_info > (3, 8): + def compare(code): + is_async = (inspect.CO_COROUTINE & code.co_flags == inspect.CO_COROUTINE) + return is_async + else: + def compare(code): + return _async + + # refactor that to just change the mod constructor. + to_run = [] + for node in to_run_exec: + to_run.append((node, 'exec')) + + for node in to_run_interactive: + to_run.append((node, 'single')) + + for node,mode in to_run: + if mode == 'exec': + mod = Module([node], []) + elif mode == 'single': + mod = ast.Interactive([node]) + with compiler.extra_flags(getattr(ast, 'PyCF_ALLOW_TOP_LEVEL_AWAIT', 0x0) if self.autoawait else 0x0): + code = compiler(mod, cell_name, mode) + asy = compare(code) + if (await self.run_code(code, result, async_=asy)): return True # Flush softspace @@ -3260,8 +3284,7 @@ class InteractiveShell(SingletonConfigurable): return eval(code_obj, user_ns) - @asyncio.coroutine - def run_code(self, code_obj, result=None, *, async_=False): + async def run_code(self, code_obj, result=None, *, async_=False): """Execute a code object. When an exception occurs, self.showtraceback() is called to display a @@ -3292,10 +3315,12 @@ class InteractiveShell(SingletonConfigurable): try: try: self.hooks.pre_run_code_hook() - if async_: - last_expr = (yield from self._async_exec(code_obj, self.user_ns)) + if async_ and sys.version_info < (3,8): + last_expr = (await self._async_exec(code_obj, self.user_ns)) code = compile('last_expr', 'fake', "single") exec(code, {'last_expr': last_expr}) + elif async_ : + await eval(code_obj, self.user_global_ns, self.user_ns) else: exec(code_obj, self.user_global_ns, self.user_ns) finally: