From 6f33fcd449312e0df728e9ec6ed4185b103e34f2 2018-08-14 01:46:23 From: Matthias Bussonnier Date: 2018-08-14 01:46:23 Subject: [PATCH] Prototype async REPL using IPython, take III This is a squash and a rebase of a large number of commits from Min and I. For simplicity of managing it, history has been reduced to a single commit, but more historical versions can be found, in particular in PR 11155, or commit aedb5d6d3a441dcdb7180ac9b5cc03f91329117b to be more exact. --- diff --git a/IPython/core/async_helpers.py b/IPython/core/async_helpers.py new file mode 100644 index 0000000..f22d93e --- /dev/null +++ b/IPython/core/async_helpers.py @@ -0,0 +1,88 @@ +""" +Async helper function that are invalid syntax on Python 3.5 and below. + +Known limitation and possible improvement. + +Top level code that contain a return statement (instead of, or in addition to +await) will be detected as requiring being wrapped in async calls. This should +be prevented as early return will not work. +""" + + + +import ast +import sys +import inspect +from textwrap import dedent, indent +from types import CodeType + + +def _asyncio_runner(coro): + """ + Handler for asyncio autoawait + """ + import asyncio + return asyncio.get_event_loop().run_until_complete(coro) + + +def _curio_runner(coroutine): + """ + handler for curio autoawait + """ + import curio + return curio.run(coroutine) + + +if sys.version_info > (3, 5): + # nose refuses to avoid this file and async def is invalidsyntax + s = dedent(''' + def _trio_runner(function): + import trio + async def loc(coro): + """ + We need the dummy no-op async def to protect from + trio's internal. See https://github.com/python-trio/trio/issues/89 + """ + return await coro + return trio.run(loc, function) + ''') + exec(s, globals(), locals()) + + +def _asyncify(code: str) -> str: + """wrap code in async def definition. + + And setup a bit of context to run it later. + """ + res = dedent(""" + async def __wrapper__(): + try: + {usercode} + finally: + locals() + """).format(usercode=indent(code, ' ' * 8)[8:]) + return res + + +def _should_be_async(cell: str) -> bool: + """Detect if a block of code need to be wrapped in an `async def` + + Attempt to parse the block of code, it it compile we're fine. + Otherwise we wrap if and try to compile. + + If it works, assume it should be async. Otherwise Return False. + + Not handled yet: If the block of code has a return statement as the top + level, it will be seen as async. This is a know limitation. + """ + + try: + ast.parse(cell) + return False + except SyntaxError: + try: + ast.parse(_asyncify(cell)) + except SyntaxError: + return False + return True + return False diff --git a/IPython/core/interactiveshell.py b/IPython/core/interactiveshell.py index 9d17bdf..b754011 100644 --- a/IPython/core/interactiveshell.py +++ b/IPython/core/interactiveshell.py @@ -13,6 +13,7 @@ import abc import ast +import asyncio import atexit import builtins as builtin_mod import functools @@ -30,6 +31,7 @@ from io import open as io_open from pickleshare import PickleShareDB from traitlets.config.configurable import SingletonConfigurable +from traitlets.utils.importstring import import_item from IPython.core import oinspect from IPython.core import magic from IPython.core import page @@ -73,7 +75,7 @@ from IPython.utils.text import format_screen, LSString, SList, DollarFormatter from IPython.utils.tempdir import TemporaryDirectory from traitlets import ( Integer, Bool, CaselessStrEnum, Enum, List, Dict, Unicode, Instance, Type, - observe, default, + observe, default, validate, Any ) from warnings import warn from logging import error @@ -113,6 +115,102 @@ else: _single_targets_nodes = (ast.AugAssign, ) #----------------------------------------------------------------------------- +# Await Helpers +#----------------------------------------------------------------------------- + +def removed_co_newlocals(function:types.FunctionType) -> types.FunctionType: + """Return a function that do not create a new local scope. + + Given a function, create a clone of this function where the co_newlocal flag + has been removed, making this function code actually run in the sourounding + scope. + + We need this in order to run asynchronous code in user level namespace. + """ + from types import CodeType, FunctionType + CO_NEWLOCALS = 0x0002 + code = function.__code__ + new_code = CodeType( + code.co_argcount, + code.co_kwonlyargcount, + code.co_nlocals, + code.co_stacksize, + code.co_flags & ~CO_NEWLOCALS, + code.co_code, + code.co_consts, + code.co_names, + code.co_varnames, + code.co_filename, + code.co_name, + code.co_firstlineno, + code.co_lnotab, + code.co_freevars, + code.co_cellvars + ) + return FunctionType(new_code, globals(), function.__name__, function.__defaults__) + + +if sys.version_info > (3,5): + from .async_helpers import (_asyncio_runner, _curio_runner, _trio_runner, + _should_be_async, _asyncify + ) +else : + _asyncio_runner = _curio_runner = _trio_runner = None + + def _should_be_async(whatever:str)->bool: + return False + + +def _ast_asyncify(cell:str, wrapper_name:str) -> ast.Module: + """ + Parse a cell with top-level await and modify the AST to be able to run it later. + + Parameter + --------- + + cell: str + The code cell to asyncronify + wrapper_name: str + The name of the function to be used to wrap the passed `cell`. It is + advised to **not** use a python identifier in order to not pollute the + global namespace in which the function will be ran. + + Return + ------ + + A module object AST containing **one** function named `wrapper_name`. + + The given code is wrapped in a async-def function, parsed into an AST, and + the resulting function definition AST is modified to return the last + expression. + + The last expression or await node is moved into a return statement at the + end of the function, and removed from its original location. If the last + node is not Expr or Await nothing is done. + + The function `__code__` will need to be later modified (by + ``removed_co_newlocals``) in a subsequent step to not create new `locals()` + meaning that the local and global scope are the same, ie as if the body of + the function was at module level. + + Lastly a call to `locals()` is made just before the last expression of the + function, or just after the last assignment or statement to make sure the + global dict is updated as python function work with a local fast cache which + is updated only on `local()` calls. + """ + + from ast import Expr, Await, Return + tree = ast.parse(_asyncify(cell)) + + function_def = tree.body[0] + function_def.name = wrapper_name + try_block = function_def.body[0] + lastexpr = try_block.body[-1] + if isinstance(lastexpr, (Expr, Await)): + try_block.body[-1] = Return(lastexpr.value) + ast.fix_missing_locations(tree) + return tree +#----------------------------------------------------------------------------- # Globals #----------------------------------------------------------------------------- @@ -258,6 +356,40 @@ class InteractiveShell(SingletonConfigurable): """ ).tag(config=True) + autoawait = Bool(True, help= + """ + Automatically run await statement in the top level repl. + """ + ).tag(config=True) + + loop_runner_map ={ + 'asyncio':_asyncio_runner, + 'curio':_curio_runner, + 'trio':_trio_runner, + } + + loop_runner = Any(default_value="IPython.core.interactiveshell._asyncio_runner", + allow_none=True, + help="""Select the loop runner that will be used to execute top-level asynchronous code""" + ).tag(config=True) + + @default('loop_runner') + def _default_loop_runner(self): + return import_item("IPython.core.interactiveshell._asyncio_runner") + + @validate('loop_runner') + def _import_runner(self, proposal): + if isinstance(proposal.value, str): + if proposal.value in self.loop_runner_map: + return self.loop_runner_map[proposal.value] + runner = import_item(proposal.value) + if not callable(runner): + raise ValueError('loop_runner must be callable') + return runner + if not callable(proposal.value): + raise ValueError('loop_runner must be callable') + return proposal.value + automagic = Bool(True, help= """ Enable magic commands to be called without the leading %. @@ -1449,6 +1581,7 @@ class InteractiveShell(SingletonConfigurable): parent = None obj = None + # Look for the given name by splitting it in parts. If the head is # found, then we look for all the remaining parts as members, and only # declare success if we can find them all. @@ -1984,7 +2117,6 @@ class InteractiveShell(SingletonConfigurable): self.set_hook('complete_command', cd_completer, str_key = '%cd') self.set_hook('complete_command', reset_completer, str_key = '%reset') - @skip_doctest def complete(self, text, line=None, cursor_pos=None): """Return the completed text and a list of completions. @@ -2667,14 +2799,36 @@ class InteractiveShell(SingletonConfigurable): return result def _run_cell(self, raw_cell, store_history, silent, shell_futures): - """Internal method to run a complete IPython cell. + """Internal method to run a complete IPython cell.""" + return self.loop_runner( + self.run_cell_async( + raw_cell, + store_history=store_history, + silent=silent, + shell_futures=shell_futures, + ) + ) + + @asyncio.coroutine + def run_cell_async(self, raw_cell, store_history=False, silent=False, shell_futures=True): + """Run a complete IPython cell asynchronously. Parameters ---------- raw_cell : str + The code (including IPython code such as %magic functions) to run. store_history : bool + If True, the raw and translated cell will be stored in IPython's + history. For user code calling back into IPython's machinery, this + should be set to False. silent : bool + If True, avoid side-effects, such as implicit displayhooks and + and logging. silent=True forces store_history=False. shell_futures : bool + If True, the code will share future statements with the interactive + shell. It will both be affected by previous __future__ imports, and + any __future__ imports in the code will affect the shell. If False, + __future__ imports are not shared in either direction. Returns ------- @@ -2749,13 +2903,33 @@ class InteractiveShell(SingletonConfigurable): # compiler compiler = self.compile if shell_futures else CachingCompiler() + _run_async = False + with self.builtin_trap: cell_name = self.compile.cache(cell, self.execution_count) with self.display_trap: # Compile to bytecode try: - code_ast = compiler.ast_parse(cell, filename=cell_name) + if self.autoawait and _should_be_async(cell): + # the code AST below will not be user code: we wrap it + # in an `async def`. This will likely make some AST + # transformer below miss some transform opportunity and + # introduce a small coupling to run_code (in which we + # bake some assumptions of what _ast_asyncify returns. + # they are ways around (like grafting part of the ast + # later: + # - Here, return code_ast.body[0].body[1:-1], as well + # as last expression in return statement which is + # the user code part. + # - Let it go through the AST transformers, and graft + # - it back after the AST transform + # But that seem unreasonable, at least while we + # do not need it. + code_ast = _ast_asyncify(cell, 'async-def-wrapper') + _run_async = True + else: + code_ast = compiler.ast_parse(cell, filename=cell_name) except self.custom_exceptions as e: etype, value, tb = sys.exc_info() self.CustomTB(etype, value, tb) @@ -2780,9 +2954,11 @@ class InteractiveShell(SingletonConfigurable): self.displayhook.exec_result = result # Execute the user code - interactivity = 'none' if silent else self.ast_node_interactivity - has_raised = self.run_ast_nodes(code_ast.body, cell_name, - interactivity=interactivity, compiler=compiler, result=result) + interactivity = "none" if silent else self.ast_node_interactivity + if _run_async: + interactivity = 'async' + has_raised = yield from self.run_ast_nodes(code_ast.body, cell_name, + interactivity=interactivity, compiler=compiler, result=result) self.last_execution_succeeded = not has_raised self.last_execution_result = result @@ -2826,12 +3002,12 @@ class InteractiveShell(SingletonConfigurable): except Exception: warn("AST transformer %r threw an error. It will be unregistered." % transformer) self.ast_transformers.remove(transformer) - + if self.ast_transformers: ast.fix_missing_locations(node) return node - + @asyncio.coroutine def run_ast_nodes(self, nodelist:ListType[AST], cell_name:str, interactivity='last_expr', compiler=compile, result=None): """Run a sequence of AST nodes. The execution mode depends on the @@ -2852,6 +3028,12 @@ class InteractiveShell(SingletonConfigurable): are not displayed) 'last_expr_or_assign' will run the last expression or the last assignment. Other values for this parameter will raise a ValueError. + + Experimental value: 'async' Will try to run top level interactive + async/await code in default runner, this will not respect the + interactivty setting and will only run the last node if it is an + expression. + compiler : callable A function with the same interface as the built-in compile(), to turn the AST nodes into code objects. Default is the built-in compile(). @@ -2880,6 +3062,7 @@ class InteractiveShell(SingletonConfigurable): nodelist.append(nnode) interactivity = 'last_expr' + _async = False if interactivity == 'last_expr': if isinstance(nodelist[-1], ast.Expr): interactivity = "last" @@ -2892,20 +3075,32 @@ class InteractiveShell(SingletonConfigurable): to_run_exec, to_run_interactive = nodelist[:-1], nodelist[-1:] elif interactivity == 'all': to_run_exec, to_run_interactive = [], nodelist + elif interactivity == 'async': + _async = True else: raise ValueError("Interactivity was %r" % interactivity) try: - for i, node in enumerate(to_run_exec): - mod = ast.Module([node]) - code = compiler(mod, cell_name, "exec") - if self.run_code(code, result): - return True - - for i, node in enumerate(to_run_interactive): - mod = ast.Interactive([node]) - code = compiler(mod, cell_name, "single") - if self.run_code(code, result): + if _async: + # If interactivity is async the semantics of run_code are + # completely different Skip usual machinery. + mod = ast.Module(nodelist) + async_wrapper_code = compiler(mod, 'cell_name', 'exec') + exec(async_wrapper_code, self.user_global_ns, self.user_ns) + async_code = removed_co_newlocals(self.user_ns.pop('async-def-wrapper')).__code__ + if (yield from self.run_code(async_code, result, async_=True)): return True + else: + for i, node in enumerate(to_run_exec): + mod = ast.Module([node]) + code = compiler(mod, cell_name, "exec") + if (yield from self.run_code(code, result)): + return True + + for i, node in enumerate(to_run_interactive): + mod = ast.Interactive([node]) + code = compiler(mod, cell_name, "single") + if (yield from self.run_code(code, result)): + return True # Flush softspace if softspace(sys.stdout, 0): @@ -2928,7 +3123,23 @@ class InteractiveShell(SingletonConfigurable): return False - def run_code(self, code_obj, result=None): + def _async_exec(self, code_obj: types.CodeType, user_ns: dict): + """ + Evaluate an asynchronous code object using a code runner + + Fake asynchronous execution of code_object in a namespace via a proxy namespace. + + Returns coroutine object, which can be executed via async loop runner + + WARNING: The semantics of `async_exec` are quite different from `exec`, + in particular you can only pass a single namespace. It also return a + handle to the value of the last things returned by code_object. + """ + + return eval(code_obj, user_ns) + + @asyncio.coroutine + def run_code(self, code_obj, result=None, *, async_=False): """Execute a code object. When an exception occurs, self.showtraceback() is called to display a @@ -2940,6 +3151,8 @@ class InteractiveShell(SingletonConfigurable): A compiled code object, to be executed result : ExecutionResult, optional An object to store exceptions that occur during execution. + async_ : Bool (Experimental) + Attempt to run top-level asynchronous code in a default loop. Returns ------- @@ -2957,8 +3170,12 @@ class InteractiveShell(SingletonConfigurable): try: try: self.hooks.pre_run_code_hook() - #rprint('Running code', repr(code_obj)) # dbg - exec(code_obj, self.user_global_ns, self.user_ns) + if async_: + last_expr = (yield from self._async_exec(code_obj, self.user_ns)) + code = compile('last_expr', 'fake', "single") + exec(code, {'last_expr': last_expr}) + else: + exec(code_obj, self.user_global_ns, self.user_ns) finally: # Reset our crash handler in place sys.excepthook = old_excepthook diff --git a/IPython/core/magics/basic.py b/IPython/core/magics/basic.py index 87532e1..29434e9 100644 --- a/IPython/core/magics/basic.py +++ b/IPython/core/magics/basic.py @@ -2,19 +2,20 @@ import argparse -import textwrap +from logging import error import io -import sys from pprint import pformat +import textwrap +import sys +from warnings import warn +from traitlets.utils.importstring import import_item from IPython.core import magic_arguments, page from IPython.core.error import UsageError from IPython.core.magic import Magics, magics_class, line_magic, magic_escapes from IPython.utils.text import format_screen, dedent, indent from IPython.testing.skipdoctest import skip_doctest from IPython.utils.ipstruct import Struct -from warnings import warn -from logging import error class MagicsDisplay(object): @@ -379,6 +380,64 @@ Currently the magic system has the following functions:""", xmode_switch_err('user') @line_magic + def autoawait(self, parameter_s): + """ + Allow to change the status of the autoawait option. + + This allow you to set a specific asynchronous code runner. + + If no value is passed, print the currently used asynchronous integration + and whether it is activated. + + It can take a number of value evaluated in the following order: + + - False/false/off deactivate autoawait integration + - True/true/on activate autoawait integration using configured default + loop + - asyncio/curio/trio activate autoawait integration and use integration + with said library. + + If the passed parameter does not match any of the above and is a python + identifier, get said object from user namespace and set it as the + runner, and activate autoawait. + + If the object is a fully qualified object name, attempt to import it and + set it as the runner, and activate autoawait.""" + + param = parameter_s.strip() + d = {True: "on", False: "off"} + + if not param: + print("IPython autoawait is `{}`, and set to use `{}`".format( + d[self.shell.autoawait], + self.shell.loop_runner + )) + return None + + if param.lower() in ('false', 'off'): + self.shell.autoawait = False + return None + if param.lower() in ('true', 'on'): + self.shell.autoawait = True + return None + + if param in self.shell.loop_runner_map: + self.shell.loop_runner = param + self.shell.autoawait = True + return None + + if param in self.shell.user_ns : + self.shell.loop_runner = self.shell.user_ns[param] + self.shell.autoawait = True + return None + + runner = import_item(param) + + self.shell.loop_runner = runner + self.shell.autoawait = True + + + @line_magic def pip(self, args=''): """ Intercept usage of ``pip`` in IPython and direct user to run command outside of IPython. diff --git a/IPython/core/tests/test_async_helpers.py b/IPython/core/tests/test_async_helpers.py new file mode 100644 index 0000000..6c54208 --- /dev/null +++ b/IPython/core/tests/test_async_helpers.py @@ -0,0 +1,52 @@ +""" +Test for async helpers. + +Should only trigger on python 3.5+ or will have syntax errors. +""" + +import sys +import nose.tools as nt +from textwrap import dedent +from unittest import TestCase + +ip = get_ipython() +iprc = lambda x: ip.run_cell(dedent(x)) + +if sys.version_info > (3,5): + from IPython.core.async_helpers import _should_be_async + + class AsyncTest(TestCase): + + def test_should_be_async(self): + nt.assert_false(_should_be_async("False")) + nt.assert_true(_should_be_async("await bar()")) + nt.assert_true(_should_be_async("x = await bar()")) + nt.assert_false(_should_be_async(dedent(""" + async def awaitable(): + pass + """))) + + def test_execute(self): + iprc(""" + import asyncio + await asyncio.sleep(0.001) + """) + + def test_autoawait(self): + ip.run_cell('%autoawait False') + ip.run_cell('%autoawait True') + iprc(''' + from asyncio import sleep + await.sleep(0.1) + ''') + + def test_autoawait_curio(self): + ip.run_cell('%autoawait curio') + + def test_autoawait_trio(self): + ip.run_cell('%autoawait trio') + + def tearDown(self): + ip.loop_runner = 'asyncio' + + diff --git a/IPython/terminal/embed.py b/IPython/terminal/embed.py index ea5c4f1..fa0345c 100644 --- a/IPython/terminal/embed.py +++ b/IPython/terminal/embed.py @@ -19,6 +19,23 @@ from IPython.terminal.ipapp import load_default_config from traitlets import Bool, CBool, Unicode from IPython.utils.io import ask_yes_no +from contextlib import contextmanager + +_sentinel = object() +@contextmanager +def new_context(): + import trio._core._run as tcr + old_runner = getattr(tcr.GLOBAL_RUN_CONTEXT, 'runner', _sentinel) + old_task = getattr(tcr.GLOBAL_RUN_CONTEXT, 'task', None) + if old_runner is not _sentinel: + del tcr.GLOBAL_RUN_CONTEXT.runner + tcr.GLOBAL_RUN_CONTEXT.task = None + yield + if old_runner is not _sentinel: + tcr.GLOBAL_RUN_CONTEXT.runner = old_runner + tcr.GLOBAL_RUN_CONTEXT.task = old_task + + class KillEmbedded(Exception):pass # kept for backward compatibility as IPython 6 was released with @@ -366,6 +383,9 @@ def embed(**kwargs): config = load_default_config() config.InteractiveShellEmbed = config.TerminalInteractiveShell kwargs['config'] = config + using = kwargs.get('using', 'trio') + if using : + kwargs['config'].update({'TerminalInteractiveShell':{'loop_runner':using, 'colors':'NoColor'}}) #save ps1/ps2 if defined ps1 = None ps2 = None @@ -380,11 +400,12 @@ def embed(**kwargs): cls = type(saved_shell_instance) cls.clear_instance() frame = sys._getframe(1) - shell = InteractiveShellEmbed.instance(_init_location_id='%s:%s' % ( - frame.f_code.co_filename, frame.f_lineno), **kwargs) - shell(header=header, stack_depth=2, compile_flags=compile_flags, - _call_location_id='%s:%s' % (frame.f_code.co_filename, frame.f_lineno)) - InteractiveShellEmbed.clear_instance() + with new_context(): + shell = InteractiveShellEmbed.instance(_init_location_id='%s:%s' % ( + frame.f_code.co_filename, frame.f_lineno), **kwargs) + shell(header=header, stack_depth=2, compile_flags=compile_flags, + _call_location_id='%s:%s' % (frame.f_code.co_filename, frame.f_lineno)) + InteractiveShellEmbed.clear_instance() #restore previous instance if saved_shell_instance is not None: cls = type(saved_shell_instance) diff --git a/IPython/terminal/tests/test_embed.py b/IPython/terminal/tests/test_embed.py index 5d75ad0..de5b1e3 100644 --- a/IPython/terminal/tests/test_embed.py +++ b/IPython/terminal/tests/test_embed.py @@ -72,6 +72,7 @@ def test_nest_embed(): child = pexpect.spawn(sys.executable, ['-m', 'IPython', '--colors=nocolor'], env=env) + child.timeout = 5 child.expect(ipy_prompt) child.sendline("import IPython") child.expect(ipy_prompt) @@ -86,7 +87,8 @@ def test_nest_embed(): except pexpect.TIMEOUT as e: print(e) #child.interact() - child.sendline("embed1 = get_ipython()"); child.expect(ipy_prompt) + child.sendline("embed1 = get_ipython()") + child.expect(ipy_prompt) child.sendline("print('true' if embed1 is not ip0 else 'false')") assert(child.expect(['true\r\n', 'false\r\n']) == 0) child.expect(ipy_prompt) @@ -103,7 +105,8 @@ def test_nest_embed(): except pexpect.TIMEOUT as e: print(e) #child.interact() - child.sendline("embed2 = get_ipython()"); child.expect(ipy_prompt) + child.sendline("embed2 = get_ipython()") + child.expect(ipy_prompt) child.sendline("print('true' if embed2 is not embed1 else 'false')") assert(child.expect(['true\r\n', 'false\r\n']) == 0) child.expect(ipy_prompt) diff --git a/docs/source/conf.py b/docs/source/conf.py index 20ef8f9..7d99ebd 100755 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -143,7 +143,8 @@ today_fmt = '%B %d, %Y' # Exclude these glob-style patterns when looking for source files. They are # relative to the source/ directory. -exclude_patterns = ['whatsnew/pr'] +exclude_patterns = ['whatsnew/pr/antigravity-feature.*', + 'whatsnew/pr/incompat-switching-to-perl.*'] # If true, '()' will be appended to :func: etc. cross-reference text. diff --git a/docs/source/interactive/autoawait.rst b/docs/source/interactive/autoawait.rst new file mode 100644 index 0000000..3d84e49 --- /dev/null +++ b/docs/source/interactive/autoawait.rst @@ -0,0 +1,186 @@ + +.. autoawait: + +Asynchronous in REPL: Autoawait +=============================== + +Starting with IPython 6.0, and when user Python 3.6 and above, IPython offer the +ability to run asynchronous code from the REPL. constructs which are +:exc:`SyntaxError` s in the Python REPL can be used seamlessly in IPython. + +When a supported libray is used, IPython will automatically `await` Futures +and Coroutines in the REPL. This will happen if an :ref:`await ` (or `async`) is +use at top level scope, or if any structure valid only in `async def +`_ function +context are present. For example, the following being a syntax error in the +Python REPL:: + + Python 3.6.0 + [GCC 4.2.1] + Type "help", "copyright", "credits" or "license" for more information. + >>> import aiohttp + >>> result = aiohttp.get('https://api.github.com') + >>> response = await result + File "", line 1 + response = await result + ^ + SyntaxError: invalid syntax + +Should behave as expected in the IPython REPL:: + + Python 3.6.0 + Type 'copyright', 'credits' or 'license' for more information + IPython 6.0.0.dev -- An enhanced Interactive Python. Type '?' for help. + + In [1]: import aiohttp + ...: result = aiohttp.get('https://api.github.com') + + In [2]: response = await result + + + In [3]: await response.json() + Out[3]: + {'authorizations_url': 'https://api.github.com/authorizations', + 'code_search_url': 'https://api.github.com/search/code?q={query}...', + ... + } + + +You can use the ``c.InteractiveShell.autoawait`` configuration option and set it +to :any:`False` to deactivate automatic wrapping of asynchronous code. You can also +use the :magic:`%autoawait` magic to toggle the behavior at runtime:: + + In [1]: %autoawait False + + In [2]: %autoawait + IPython autoawait is `Off`, and set to use `IPython.core.interactiveshell._asyncio_runner` + + + +By default IPython will assume integration with Python's provided +:mod:`asyncio`, but integration with other libraries is provided. In particular +we provide experimental integration with the ``curio`` and ``trio`` library. + +You can switch current integration by using the +``c.InteractiveShell.loop_runner`` option or the ``autoawait `` magic. + +For example:: + + In [1]: %autoawait trio + + In [2]: import trio + + In [3]: async def child(i): + ...: print(" child %s goes to sleep"%i) + ...: await trio.sleep(2) + ...: print(" child %s wakes up"%i) + + In [4]: print('parent start') + ...: async with trio.open_nursery() as n: + ...: for i in range(5): + ...: n.spawn(child, i) + ...: print('parent end') + parent start + child 2 goes to sleep + child 0 goes to sleep + child 3 goes to sleep + child 1 goes to sleep + child 4 goes to sleep + + child 2 wakes up + child 1 wakes up + child 0 wakes up + child 3 wakes up + child 4 wakes up + parent end + + +In the above example, ``async with`` at top level scope is a syntax error in +Python. + +Using this mode can have unexpected consequences if used in interaction with +other features of IPython and various registered extensions. In particular if you +are a direct or indirect user of the AST transformers, these may not apply to +your code. + +The default loop, or runner does not run in the background, so top level +asynchronous code must finish for the REPL to allow you to enter more code. As +with usual Python semantic, the awaitables are started only when awaited for the +first time. That is to say, in first example, no network request is done between +``In[1]`` and ``In[2]``. + + +Internals +========= + +As running asynchronous code is not supported in interactive REPL as of Python +3.6 we have to rely to a number of complex workaround to allow this to happen. +It is interesting to understand how this works in order to understand potential +bugs, or provide a custom runner. + +Among the many approaches that are at our disposition, we find only one that +suited out need. Under the hood we :ct the code object from a async-def function +and run it in global namesapace after modifying the ``__code__`` object.:: + + async def inner_async(): + locals().update(**global_namespace) + # + # here is user code + # + return last_user_statement + codeobj = modify(inner_async.__code__) + coroutine = eval(codeobj, user_ns) + display(loop_runner(coroutine)) + + + +The first thing you'll notice is that unlike classical ``exec``, there is only +one name space. Second, user code runs in a function scope, and not a module +scope. + +On top of the above there are significant modification to the AST of +``function``, and ``loop_runner`` can be arbitrary complex. So there is a +significant overhead to this kind of code. + +By default the generated coroutine function will be consumed by Asyncio's +``loop_runner = asyncio.get_evenloop().run_until_complete()`` method. It is +though possible to provide your own. + +A loop runner is a *synchronous* function responsible from running a coroutine +object. + +The runner is responsible from ensuring that ``coroutine`` run to completion, +and should return the result of executing the coroutine. Let's write a +runner for ``trio`` that print a message when used as an exercise, ``trio`` is +special as it usually prefer to run a function object and make a coroutine by +itself, we can get around this limitation by wrapping it in an async-def without +parameters and passing this value to ``trio``:: + + + In [1]: import trio + ...: from types import CoroutineType + ...: + ...: def trio_runner(coro:CoroutineType): + ...: print('running asynchronous code') + ...: async def corowrap(coro): + ...: return await coro + ...: return trio.run(corowrap, coro) + +We can set it up by passing it to ``%autoawait``:: + + In [2]: %autoawait trio_runner + + In [3]: async def async_hello(name): + ...: await trio.sleep(1) + ...: print(f'Hello {name} world !') + ...: await trio.sleep(1) + + In [4]: await async_hello('async') + running asynchronous code + Hello async world ! + + +Asynchronous programming in python (and in particular in the REPL) is still a +relatively young subject. Feel free to contribute improvements to this codebase +and give us feedback. diff --git a/docs/source/interactive/index.rst b/docs/source/interactive/index.rst index 97332e1..7010c51 100644 --- a/docs/source/interactive/index.rst +++ b/docs/source/interactive/index.rst @@ -21,6 +21,7 @@ done some work in the classic Python REPL. plotting reference shell + autoawait tips python-ipython-diff magics diff --git a/docs/source/whatsnew/development.rst b/docs/source/whatsnew/development.rst index 13f02a4..4a62f47 100644 --- a/docs/source/whatsnew/development.rst +++ b/docs/source/whatsnew/development.rst @@ -11,6 +11,135 @@ This document describes in-flight development work. `docs/source/whatsnew/pr` folder +Released .... ...., 2017 + + +Need to be updated: + +.. toctree:: + :maxdepth: 2 + :glob: + + pr/* + +IPython 6 feature a major improvement in the completion machinery which is now +capable of completing non-executed code. It is also the first version of IPython +to stop compatibility with Python 2, which is still supported on the bugfix only +5.x branch. Read below to have a non-exhaustive list of new features. + +Make sure you have pip > 9.0 before upgrading. +You should be able to update by using: + +.. code:: + + pip install ipython --upgrade + +New completion API and Interface +-------------------------------- + +The completer Completion API has seen an overhaul, and the new completer have +plenty of improvement both from the end users of terminal IPython or for +consumers of the API. + +This new API is capable of pulling completions from :any:`jedi`, thus allowing +type inference on non-executed code. If :any:`jedi` is installed completion like +the following are now becoming possible without code evaluation: + + >>> data = ['Number of users', 123_456] + ... data[0]. + +That is to say, IPython is now capable of inferring that `data[0]` is a string, +and will suggest completions like `.capitalize`. The completion power of IPython +will increase with new Jedi releases, and a number of bugs and more completions +are already available on development version of :any:`jedi` if you are curious. + +With the help of prompt toolkit, types of completions can be shown in the +completer interface: + +.. image:: ../_images/jedi_type_inference_60.png + :alt: Jedi showing ability to do type inference + :align: center + :width: 400px + :target: ../_images/jedi_type_inference_60.png + +The appearance of the completer is controlled by the +``c.TerminalInteractiveShell.display_completions`` option that will show the +type differently depending on the value among ``'column'``, ``'multicolumn'`` +and ``'readlinelike'`` + +The use of Jedi also full fill a number of request and fix a number of bugs +like case insensitive completion, completion after division operator: See +:ghpull:`10182`. + +Extra patches and updates will be needed to the :mod:`ipykernel` package for +this feature to be available to other clients like jupyter Notebook, Lab, +Nteract, Hydrogen... + +The use of Jedi can is barely noticeable on recent enough machines, but can be +feel on older ones, in cases were Jedi behavior need to be adjusted, the amount +of time given to Jedi to compute type inference can be adjusted with +``c.IPCompleter.jedi_compute_type_timeout``, with object whose type were not +inferred will be shown as ````. Jedi can also be completely deactivated +by using the ``c.Completer.use_jedi=False`` option. + + +The old ``Completer.complete()`` API is waiting deprecation and should be +replaced replaced by ``Completer.completions()`` in a near future. Feedback on +the current state of the API and suggestions welcome. + +Python 3 only codebase +---------------------- + +One of the large challenges in IPython 6.0 has been the adoption of a pure +Python 3 code base, which lead us to great length to upstream patches in pip, +pypi and warehouse to make sure Python 2 system still upgrade to the latest +compatible Python version compatible. + +We remind our Python 2 users that IPython 5 is still compatible with Python 2.7, +still maintained and get regular releases. Using pip 9+, upgrading IPython will +automatically upgrade to the latest version compatible with your system. + +.. warning:: + + If you are on a system using an older verison of pip on Python 2, pip may + still install IPython 6.0 on your system, and IPython will refuse to start. + You can fix this by ugrading pip, and reinstalling ipython, or forcing pip to + install an earlier version: ``pip install 'ipython<6'`` + +The ability to use only Python 3 on the code base of IPython has bring a number +of advantage. Most of the newly written code make use of `optional function type +anotation `_ leading to clearer code +and better documentation. + +The total size of the repository has also for a first time between releases +(excluding the big split for 4.0) decreased by about 1500 lines, potentially +quite a bit more codewide as some documents like this one are append only and +are about 300 lines long. + +The removal as of Python2/Python3 shim layer has made the code quite clearer and +more idiomatic in a number of location, and much friendlier to work with and +understand. We hope to further embrace Python 3 capability in the next release +cycle and introduce more of the Python 3 only idioms (yield from, kwarg only, +general unpacking) in the code base of IPython, and see if we can take advantage +of these as well to improve user experience with better error messages and +hints. + + +Miscs improvements +------------------ + + +- The :cellmagic:`capture` magic can now capture the result of a cell (from an + expression on the last line), as well as printed and displayed output. + :ghpull:`9851`. + +- Pressing Ctrl-Z in the terminal debugger now suspends IPython, as it already + does in the main terminal prompt. + +- autoreload can now reload ``Enum``. See :ghissue:`10232` and :ghpull:`10316` + +- IPython.display has gained a :any:`GeoJSON ` object. + :ghpull:`10288` and :ghpull:`10253` .. DO NOT EDIT THIS LINE BEFORE RELEASE. FEATURE INSERTION POINT. diff --git a/docs/source/whatsnew/pr/await-repl.rst b/docs/source/whatsnew/pr/await-repl.rst new file mode 100644 index 0000000..614a00a --- /dev/null +++ b/docs/source/whatsnew/pr/await-repl.rst @@ -0,0 +1,55 @@ +Await REPL +---------- + +:ghpull:`10390` introduced the ability to ``await`` Futures and +Coroutines in the REPL. For example:: + + Python 3.6.0 + Type 'copyright', 'credits' or 'license' for more information + IPython 6.0.0.dev -- An enhanced Interactive Python. Type '?' for help. + + In [1]: import aiohttp + ...: result = aiohttp.get('https://api.github.com') + + In [2]: response = await result + + + In [3]: await response.json() + Out[3]: + {'authorizations_url': 'https://api.github.com/authorizations', + 'code_search_url': 'https://api.github.com/search/code?q={query}{&page,per_page,sort,order}', + ... + } + + +Integration is by default with `asyncio`, but other libraries can be configured, +like ``curio`` or ``trio``, to improve concurrency in the REPL:: + + In [1]: %autoawait trio + + In [2]: import trio + + In [3]: async def child(i): + ...: print(" child %s goes to sleep"%i) + ...: await trio.sleep(2) + ...: print(" child %s wakes up"%i) + + In [4]: print('parent start') + ...: async with trio.open_nursery() as n: + ...: for i in range(3): + ...: n.spawn(child, i) + ...: print('parent end') + parent start + child 2 goes to sleep + child 0 goes to sleep + child 1 goes to sleep + + child 2 wakes up + child 1 wakes up + child 0 wakes up + parent end + +See :ref:`autoawait` for more information. + + + diff --git a/setup.py b/setup.py index bd72088..ed18841 100755 --- a/setup.py +++ b/setup.py @@ -201,6 +201,7 @@ install_requires = [ extras_require.update({ ':python_version == "3.4"': ['typing'], + ':python_version >= "3.5"': ['trio', 'curio'], ':sys_platform != "win32"': ['pexpect'], ':sys_platform == "darwin"': ['appnope'], ':sys_platform == "win32"': ['colorama'],