From 6f33fcd449312e0df728e9ec6ed4185b103e34f2 2018-08-14 01:46:23
From: Matthias Bussonnier <bussonniermatthias@gmail.com>
Date: 2018-08-14 01:46:23
Subject: [PATCH] Prototype async REPL using IPython, take III
This is a squash and a rebase of a large number of commits from Min and
I. For simplicity of managing it, history has been reduced to a single
commit, but more historical versions can be found, in particular in
PR 11155, or commit aedb5d6d3a441dcdb7180ac9b5cc03f91329117b to be more
exact.
---
diff --git a/IPython/core/async_helpers.py b/IPython/core/async_helpers.py
new file mode 100644
index 0000000..f22d93e
--- /dev/null
+++ b/IPython/core/async_helpers.py
@@ -0,0 +1,88 @@
+"""
+Async helper function that are invalid syntax on Python 3.5 and below.
+
+Known limitation and possible improvement.
+
+Top level code that contain a return statement (instead of, or in addition to
+await) will be detected as requiring being wrapped in async calls. This should
+be prevented as early return will not work.
+"""
+
+
+
+import ast
+import sys
+import inspect
+from textwrap import dedent, indent
+from types import CodeType
+
+
+def _asyncio_runner(coro):
+ """
+ Handler for asyncio autoawait
+ """
+ import asyncio
+ return asyncio.get_event_loop().run_until_complete(coro)
+
+
+def _curio_runner(coroutine):
+ """
+ handler for curio autoawait
+ """
+ import curio
+ return curio.run(coroutine)
+
+
+if sys.version_info > (3, 5):
+ # nose refuses to avoid this file and async def is invalidsyntax
+ s = dedent('''
+ def _trio_runner(function):
+ import trio
+ async def loc(coro):
+ """
+ We need the dummy no-op async def to protect from
+ trio's internal. See https://github.com/python-trio/trio/issues/89
+ """
+ return await coro
+ return trio.run(loc, function)
+ ''')
+ exec(s, globals(), locals())
+
+
+def _asyncify(code: str) -> str:
+ """wrap code in async def definition.
+
+ And setup a bit of context to run it later.
+ """
+ res = dedent("""
+ async def __wrapper__():
+ try:
+ {usercode}
+ finally:
+ locals()
+ """).format(usercode=indent(code, ' ' * 8)[8:])
+ return res
+
+
+def _should_be_async(cell: str) -> bool:
+ """Detect if a block of code need to be wrapped in an `async def`
+
+ Attempt to parse the block of code, it it compile we're fine.
+ Otherwise we wrap if and try to compile.
+
+ If it works, assume it should be async. Otherwise Return False.
+
+ Not handled yet: If the block of code has a return statement as the top
+ level, it will be seen as async. This is a know limitation.
+ """
+
+ try:
+ ast.parse(cell)
+ return False
+ except SyntaxError:
+ try:
+ ast.parse(_asyncify(cell))
+ except SyntaxError:
+ return False
+ return True
+ return False
diff --git a/IPython/core/interactiveshell.py b/IPython/core/interactiveshell.py
index 9d17bdf..b754011 100644
--- a/IPython/core/interactiveshell.py
+++ b/IPython/core/interactiveshell.py
@@ -13,6 +13,7 @@
import abc
import ast
+import asyncio
import atexit
import builtins as builtin_mod
import functools
@@ -30,6 +31,7 @@ from io import open as io_open
from pickleshare import PickleShareDB
from traitlets.config.configurable import SingletonConfigurable
+from traitlets.utils.importstring import import_item
from IPython.core import oinspect
from IPython.core import magic
from IPython.core import page
@@ -73,7 +75,7 @@ from IPython.utils.text import format_screen, LSString, SList, DollarFormatter
from IPython.utils.tempdir import TemporaryDirectory
from traitlets import (
Integer, Bool, CaselessStrEnum, Enum, List, Dict, Unicode, Instance, Type,
- observe, default,
+ observe, default, validate, Any
)
from warnings import warn
from logging import error
@@ -113,6 +115,102 @@ else:
_single_targets_nodes = (ast.AugAssign, )
#-----------------------------------------------------------------------------
+# Await Helpers
+#-----------------------------------------------------------------------------
+
+def removed_co_newlocals(function:types.FunctionType) -> types.FunctionType:
+ """Return a function that do not create a new local scope.
+
+ Given a function, create a clone of this function where the co_newlocal flag
+ has been removed, making this function code actually run in the sourounding
+ scope.
+
+ We need this in order to run asynchronous code in user level namespace.
+ """
+ from types import CodeType, FunctionType
+ CO_NEWLOCALS = 0x0002
+ code = function.__code__
+ new_code = CodeType(
+ code.co_argcount,
+ code.co_kwonlyargcount,
+ code.co_nlocals,
+ code.co_stacksize,
+ code.co_flags & ~CO_NEWLOCALS,
+ code.co_code,
+ code.co_consts,
+ code.co_names,
+ code.co_varnames,
+ code.co_filename,
+ code.co_name,
+ code.co_firstlineno,
+ code.co_lnotab,
+ code.co_freevars,
+ code.co_cellvars
+ )
+ return FunctionType(new_code, globals(), function.__name__, function.__defaults__)
+
+
+if sys.version_info > (3,5):
+ from .async_helpers import (_asyncio_runner, _curio_runner, _trio_runner,
+ _should_be_async, _asyncify
+ )
+else :
+ _asyncio_runner = _curio_runner = _trio_runner = None
+
+ def _should_be_async(whatever:str)->bool:
+ return False
+
+
+def _ast_asyncify(cell:str, wrapper_name:str) -> ast.Module:
+ """
+ Parse a cell with top-level await and modify the AST to be able to run it later.
+
+ Parameter
+ ---------
+
+ cell: str
+ The code cell to asyncronify
+ wrapper_name: str
+ The name of the function to be used to wrap the passed `cell`. It is
+ advised to **not** use a python identifier in order to not pollute the
+ global namespace in which the function will be ran.
+
+ Return
+ ------
+
+ A module object AST containing **one** function named `wrapper_name`.
+
+ The given code is wrapped in a async-def function, parsed into an AST, and
+ the resulting function definition AST is modified to return the last
+ expression.
+
+ The last expression or await node is moved into a return statement at the
+ end of the function, and removed from its original location. If the last
+ node is not Expr or Await nothing is done.
+
+ The function `__code__` will need to be later modified (by
+ ``removed_co_newlocals``) in a subsequent step to not create new `locals()`
+ meaning that the local and global scope are the same, ie as if the body of
+ the function was at module level.
+
+ Lastly a call to `locals()` is made just before the last expression of the
+ function, or just after the last assignment or statement to make sure the
+ global dict is updated as python function work with a local fast cache which
+ is updated only on `local()` calls.
+ """
+
+ from ast import Expr, Await, Return
+ tree = ast.parse(_asyncify(cell))
+
+ function_def = tree.body[0]
+ function_def.name = wrapper_name
+ try_block = function_def.body[0]
+ lastexpr = try_block.body[-1]
+ if isinstance(lastexpr, (Expr, Await)):
+ try_block.body[-1] = Return(lastexpr.value)
+ ast.fix_missing_locations(tree)
+ return tree
+#-----------------------------------------------------------------------------
# Globals
#-----------------------------------------------------------------------------
@@ -258,6 +356,40 @@ class InteractiveShell(SingletonConfigurable):
"""
).tag(config=True)
+ autoawait = Bool(True, help=
+ """
+ Automatically run await statement in the top level repl.
+ """
+ ).tag(config=True)
+
+ loop_runner_map ={
+ 'asyncio':_asyncio_runner,
+ 'curio':_curio_runner,
+ 'trio':_trio_runner,
+ }
+
+ loop_runner = Any(default_value="IPython.core.interactiveshell._asyncio_runner",
+ allow_none=True,
+ help="""Select the loop runner that will be used to execute top-level asynchronous code"""
+ ).tag(config=True)
+
+ @default('loop_runner')
+ def _default_loop_runner(self):
+ return import_item("IPython.core.interactiveshell._asyncio_runner")
+
+ @validate('loop_runner')
+ def _import_runner(self, proposal):
+ if isinstance(proposal.value, str):
+ if proposal.value in self.loop_runner_map:
+ return self.loop_runner_map[proposal.value]
+ runner = import_item(proposal.value)
+ if not callable(runner):
+ raise ValueError('loop_runner must be callable')
+ return runner
+ if not callable(proposal.value):
+ raise ValueError('loop_runner must be callable')
+ return proposal.value
+
automagic = Bool(True, help=
"""
Enable magic commands to be called without the leading %.
@@ -1449,6 +1581,7 @@ class InteractiveShell(SingletonConfigurable):
parent = None
obj = None
+
# Look for the given name by splitting it in parts. If the head is
# found, then we look for all the remaining parts as members, and only
# declare success if we can find them all.
@@ -1984,7 +2117,6 @@ class InteractiveShell(SingletonConfigurable):
self.set_hook('complete_command', cd_completer, str_key = '%cd')
self.set_hook('complete_command', reset_completer, str_key = '%reset')
-
@skip_doctest
def complete(self, text, line=None, cursor_pos=None):
"""Return the completed text and a list of completions.
@@ -2667,14 +2799,36 @@ class InteractiveShell(SingletonConfigurable):
return result
def _run_cell(self, raw_cell, store_history, silent, shell_futures):
- """Internal method to run a complete IPython cell.
+ """Internal method to run a complete IPython cell."""
+ return self.loop_runner(
+ self.run_cell_async(
+ raw_cell,
+ store_history=store_history,
+ silent=silent,
+ shell_futures=shell_futures,
+ )
+ )
+
+ @asyncio.coroutine
+ def run_cell_async(self, raw_cell, store_history=False, silent=False, shell_futures=True):
+ """Run a complete IPython cell asynchronously.
Parameters
----------
raw_cell : str
+ The code (including IPython code such as %magic functions) to run.
store_history : bool
+ If True, the raw and translated cell will be stored in IPython's
+ history. For user code calling back into IPython's machinery, this
+ should be set to False.
silent : bool
+ If True, avoid side-effects, such as implicit displayhooks and
+ and logging. silent=True forces store_history=False.
shell_futures : bool
+ If True, the code will share future statements with the interactive
+ shell. It will both be affected by previous __future__ imports, and
+ any __future__ imports in the code will affect the shell. If False,
+ __future__ imports are not shared in either direction.
Returns
-------
@@ -2749,13 +2903,33 @@ class InteractiveShell(SingletonConfigurable):
# compiler
compiler = self.compile if shell_futures else CachingCompiler()
+ _run_async = False
+
with self.builtin_trap:
cell_name = self.compile.cache(cell, self.execution_count)
with self.display_trap:
# Compile to bytecode
try:
- code_ast = compiler.ast_parse(cell, filename=cell_name)
+ if self.autoawait and _should_be_async(cell):
+ # the code AST below will not be user code: we wrap it
+ # in an `async def`. This will likely make some AST
+ # transformer below miss some transform opportunity and
+ # introduce a small coupling to run_code (in which we
+ # bake some assumptions of what _ast_asyncify returns.
+ # they are ways around (like grafting part of the ast
+ # later:
+ # - Here, return code_ast.body[0].body[1:-1], as well
+ # as last expression in return statement which is
+ # the user code part.
+ # - Let it go through the AST transformers, and graft
+ # - it back after the AST transform
+ # But that seem unreasonable, at least while we
+ # do not need it.
+ code_ast = _ast_asyncify(cell, 'async-def-wrapper')
+ _run_async = True
+ else:
+ code_ast = compiler.ast_parse(cell, filename=cell_name)
except self.custom_exceptions as e:
etype, value, tb = sys.exc_info()
self.CustomTB(etype, value, tb)
@@ -2780,9 +2954,11 @@ class InteractiveShell(SingletonConfigurable):
self.displayhook.exec_result = result
# Execute the user code
- interactivity = 'none' if silent else self.ast_node_interactivity
- has_raised = self.run_ast_nodes(code_ast.body, cell_name,
- interactivity=interactivity, compiler=compiler, result=result)
+ interactivity = "none" if silent else self.ast_node_interactivity
+ if _run_async:
+ interactivity = 'async'
+ has_raised = yield from self.run_ast_nodes(code_ast.body, cell_name,
+ interactivity=interactivity, compiler=compiler, result=result)
self.last_execution_succeeded = not has_raised
self.last_execution_result = result
@@ -2826,12 +3002,12 @@ class InteractiveShell(SingletonConfigurable):
except Exception:
warn("AST transformer %r threw an error. It will be unregistered." % transformer)
self.ast_transformers.remove(transformer)
-
+
if self.ast_transformers:
ast.fix_missing_locations(node)
return node
-
+ @asyncio.coroutine
def run_ast_nodes(self, nodelist:ListType[AST], cell_name:str, interactivity='last_expr',
compiler=compile, result=None):
"""Run a sequence of AST nodes. The execution mode depends on the
@@ -2852,6 +3028,12 @@ class InteractiveShell(SingletonConfigurable):
are not displayed) 'last_expr_or_assign' will run the last expression
or the last assignment. Other values for this parameter will raise a
ValueError.
+
+ Experimental value: 'async' Will try to run top level interactive
+ async/await code in default runner, this will not respect the
+ interactivty setting and will only run the last node if it is an
+ expression.
+
compiler : callable
A function with the same interface as the built-in compile(), to turn
the AST nodes into code objects. Default is the built-in compile().
@@ -2880,6 +3062,7 @@ class InteractiveShell(SingletonConfigurable):
nodelist.append(nnode)
interactivity = 'last_expr'
+ _async = False
if interactivity == 'last_expr':
if isinstance(nodelist[-1], ast.Expr):
interactivity = "last"
@@ -2892,20 +3075,32 @@ class InteractiveShell(SingletonConfigurable):
to_run_exec, to_run_interactive = nodelist[:-1], nodelist[-1:]
elif interactivity == 'all':
to_run_exec, to_run_interactive = [], nodelist
+ elif interactivity == 'async':
+ _async = True
else:
raise ValueError("Interactivity was %r" % interactivity)
try:
- for i, node in enumerate(to_run_exec):
- mod = ast.Module([node])
- code = compiler(mod, cell_name, "exec")
- if self.run_code(code, result):
- return True
-
- for i, node in enumerate(to_run_interactive):
- mod = ast.Interactive([node])
- code = compiler(mod, cell_name, "single")
- if self.run_code(code, result):
+ if _async:
+ # If interactivity is async the semantics of run_code are
+ # completely different Skip usual machinery.
+ mod = ast.Module(nodelist)
+ async_wrapper_code = compiler(mod, 'cell_name', 'exec')
+ exec(async_wrapper_code, self.user_global_ns, self.user_ns)
+ async_code = removed_co_newlocals(self.user_ns.pop('async-def-wrapper')).__code__
+ if (yield from self.run_code(async_code, result, async_=True)):
return True
+ else:
+ for i, node in enumerate(to_run_exec):
+ mod = ast.Module([node])
+ code = compiler(mod, cell_name, "exec")
+ if (yield from self.run_code(code, result)):
+ return True
+
+ for i, node in enumerate(to_run_interactive):
+ mod = ast.Interactive([node])
+ code = compiler(mod, cell_name, "single")
+ if (yield from self.run_code(code, result)):
+ return True
# Flush softspace
if softspace(sys.stdout, 0):
@@ -2928,7 +3123,23 @@ class InteractiveShell(SingletonConfigurable):
return False
- def run_code(self, code_obj, result=None):
+ def _async_exec(self, code_obj: types.CodeType, user_ns: dict):
+ """
+ Evaluate an asynchronous code object using a code runner
+
+ Fake asynchronous execution of code_object in a namespace via a proxy namespace.
+
+ Returns coroutine object, which can be executed via async loop runner
+
+ WARNING: The semantics of `async_exec` are quite different from `exec`,
+ in particular you can only pass a single namespace. It also return a
+ handle to the value of the last things returned by code_object.
+ """
+
+ return eval(code_obj, user_ns)
+
+ @asyncio.coroutine
+ def run_code(self, code_obj, result=None, *, async_=False):
"""Execute a code object.
When an exception occurs, self.showtraceback() is called to display a
@@ -2940,6 +3151,8 @@ class InteractiveShell(SingletonConfigurable):
A compiled code object, to be executed
result : ExecutionResult, optional
An object to store exceptions that occur during execution.
+ async_ : Bool (Experimental)
+ Attempt to run top-level asynchronous code in a default loop.
Returns
-------
@@ -2957,8 +3170,12 @@ class InteractiveShell(SingletonConfigurable):
try:
try:
self.hooks.pre_run_code_hook()
- #rprint('Running code', repr(code_obj)) # dbg
- exec(code_obj, self.user_global_ns, self.user_ns)
+ if async_:
+ last_expr = (yield from self._async_exec(code_obj, self.user_ns))
+ code = compile('last_expr', 'fake', "single")
+ exec(code, {'last_expr': last_expr})
+ else:
+ exec(code_obj, self.user_global_ns, self.user_ns)
finally:
# Reset our crash handler in place
sys.excepthook = old_excepthook
diff --git a/IPython/core/magics/basic.py b/IPython/core/magics/basic.py
index 87532e1..29434e9 100644
--- a/IPython/core/magics/basic.py
+++ b/IPython/core/magics/basic.py
@@ -2,19 +2,20 @@
import argparse
-import textwrap
+from logging import error
import io
-import sys
from pprint import pformat
+import textwrap
+import sys
+from warnings import warn
+from traitlets.utils.importstring import import_item
from IPython.core import magic_arguments, page
from IPython.core.error import UsageError
from IPython.core.magic import Magics, magics_class, line_magic, magic_escapes
from IPython.utils.text import format_screen, dedent, indent
from IPython.testing.skipdoctest import skip_doctest
from IPython.utils.ipstruct import Struct
-from warnings import warn
-from logging import error
class MagicsDisplay(object):
@@ -379,6 +380,64 @@ Currently the magic system has the following functions:""",
xmode_switch_err('user')
@line_magic
+ def autoawait(self, parameter_s):
+ """
+ Allow to change the status of the autoawait option.
+
+ This allow you to set a specific asynchronous code runner.
+
+ If no value is passed, print the currently used asynchronous integration
+ and whether it is activated.
+
+ It can take a number of value evaluated in the following order:
+
+ - False/false/off deactivate autoawait integration
+ - True/true/on activate autoawait integration using configured default
+ loop
+ - asyncio/curio/trio activate autoawait integration and use integration
+ with said library.
+
+ If the passed parameter does not match any of the above and is a python
+ identifier, get said object from user namespace and set it as the
+ runner, and activate autoawait.
+
+ If the object is a fully qualified object name, attempt to import it and
+ set it as the runner, and activate autoawait."""
+
+ param = parameter_s.strip()
+ d = {True: "on", False: "off"}
+
+ if not param:
+ print("IPython autoawait is `{}`, and set to use `{}`".format(
+ d[self.shell.autoawait],
+ self.shell.loop_runner
+ ))
+ return None
+
+ if param.lower() in ('false', 'off'):
+ self.shell.autoawait = False
+ return None
+ if param.lower() in ('true', 'on'):
+ self.shell.autoawait = True
+ return None
+
+ if param in self.shell.loop_runner_map:
+ self.shell.loop_runner = param
+ self.shell.autoawait = True
+ return None
+
+ if param in self.shell.user_ns :
+ self.shell.loop_runner = self.shell.user_ns[param]
+ self.shell.autoawait = True
+ return None
+
+ runner = import_item(param)
+
+ self.shell.loop_runner = runner
+ self.shell.autoawait = True
+
+
+ @line_magic
def pip(self, args=''):
"""
Intercept usage of ``pip`` in IPython and direct user to run command outside of IPython.
diff --git a/IPython/core/tests/test_async_helpers.py b/IPython/core/tests/test_async_helpers.py
new file mode 100644
index 0000000..6c54208
--- /dev/null
+++ b/IPython/core/tests/test_async_helpers.py
@@ -0,0 +1,52 @@
+"""
+Test for async helpers.
+
+Should only trigger on python 3.5+ or will have syntax errors.
+"""
+
+import sys
+import nose.tools as nt
+from textwrap import dedent
+from unittest import TestCase
+
+ip = get_ipython()
+iprc = lambda x: ip.run_cell(dedent(x))
+
+if sys.version_info > (3,5):
+ from IPython.core.async_helpers import _should_be_async
+
+ class AsyncTest(TestCase):
+
+ def test_should_be_async(self):
+ nt.assert_false(_should_be_async("False"))
+ nt.assert_true(_should_be_async("await bar()"))
+ nt.assert_true(_should_be_async("x = await bar()"))
+ nt.assert_false(_should_be_async(dedent("""
+ async def awaitable():
+ pass
+ """)))
+
+ def test_execute(self):
+ iprc("""
+ import asyncio
+ await asyncio.sleep(0.001)
+ """)
+
+ def test_autoawait(self):
+ ip.run_cell('%autoawait False')
+ ip.run_cell('%autoawait True')
+ iprc('''
+ from asyncio import sleep
+ await.sleep(0.1)
+ ''')
+
+ def test_autoawait_curio(self):
+ ip.run_cell('%autoawait curio')
+
+ def test_autoawait_trio(self):
+ ip.run_cell('%autoawait trio')
+
+ def tearDown(self):
+ ip.loop_runner = 'asyncio'
+
+
diff --git a/IPython/terminal/embed.py b/IPython/terminal/embed.py
index ea5c4f1..fa0345c 100644
--- a/IPython/terminal/embed.py
+++ b/IPython/terminal/embed.py
@@ -19,6 +19,23 @@ from IPython.terminal.ipapp import load_default_config
from traitlets import Bool, CBool, Unicode
from IPython.utils.io import ask_yes_no
+from contextlib import contextmanager
+
+_sentinel = object()
+@contextmanager
+def new_context():
+ import trio._core._run as tcr
+ old_runner = getattr(tcr.GLOBAL_RUN_CONTEXT, 'runner', _sentinel)
+ old_task = getattr(tcr.GLOBAL_RUN_CONTEXT, 'task', None)
+ if old_runner is not _sentinel:
+ del tcr.GLOBAL_RUN_CONTEXT.runner
+ tcr.GLOBAL_RUN_CONTEXT.task = None
+ yield
+ if old_runner is not _sentinel:
+ tcr.GLOBAL_RUN_CONTEXT.runner = old_runner
+ tcr.GLOBAL_RUN_CONTEXT.task = old_task
+
+
class KillEmbedded(Exception):pass
# kept for backward compatibility as IPython 6 was released with
@@ -366,6 +383,9 @@ def embed(**kwargs):
config = load_default_config()
config.InteractiveShellEmbed = config.TerminalInteractiveShell
kwargs['config'] = config
+ using = kwargs.get('using', 'trio')
+ if using :
+ kwargs['config'].update({'TerminalInteractiveShell':{'loop_runner':using, 'colors':'NoColor'}})
#save ps1/ps2 if defined
ps1 = None
ps2 = None
@@ -380,11 +400,12 @@ def embed(**kwargs):
cls = type(saved_shell_instance)
cls.clear_instance()
frame = sys._getframe(1)
- shell = InteractiveShellEmbed.instance(_init_location_id='%s:%s' % (
- frame.f_code.co_filename, frame.f_lineno), **kwargs)
- shell(header=header, stack_depth=2, compile_flags=compile_flags,
- _call_location_id='%s:%s' % (frame.f_code.co_filename, frame.f_lineno))
- InteractiveShellEmbed.clear_instance()
+ with new_context():
+ shell = InteractiveShellEmbed.instance(_init_location_id='%s:%s' % (
+ frame.f_code.co_filename, frame.f_lineno), **kwargs)
+ shell(header=header, stack_depth=2, compile_flags=compile_flags,
+ _call_location_id='%s:%s' % (frame.f_code.co_filename, frame.f_lineno))
+ InteractiveShellEmbed.clear_instance()
#restore previous instance
if saved_shell_instance is not None:
cls = type(saved_shell_instance)
diff --git a/IPython/terminal/tests/test_embed.py b/IPython/terminal/tests/test_embed.py
index 5d75ad0..de5b1e3 100644
--- a/IPython/terminal/tests/test_embed.py
+++ b/IPython/terminal/tests/test_embed.py
@@ -72,6 +72,7 @@ def test_nest_embed():
child = pexpect.spawn(sys.executable, ['-m', 'IPython', '--colors=nocolor'],
env=env)
+ child.timeout = 5
child.expect(ipy_prompt)
child.sendline("import IPython")
child.expect(ipy_prompt)
@@ -86,7 +87,8 @@ def test_nest_embed():
except pexpect.TIMEOUT as e:
print(e)
#child.interact()
- child.sendline("embed1 = get_ipython()"); child.expect(ipy_prompt)
+ child.sendline("embed1 = get_ipython()")
+ child.expect(ipy_prompt)
child.sendline("print('true' if embed1 is not ip0 else 'false')")
assert(child.expect(['true\r\n', 'false\r\n']) == 0)
child.expect(ipy_prompt)
@@ -103,7 +105,8 @@ def test_nest_embed():
except pexpect.TIMEOUT as e:
print(e)
#child.interact()
- child.sendline("embed2 = get_ipython()"); child.expect(ipy_prompt)
+ child.sendline("embed2 = get_ipython()")
+ child.expect(ipy_prompt)
child.sendline("print('true' if embed2 is not embed1 else 'false')")
assert(child.expect(['true\r\n', 'false\r\n']) == 0)
child.expect(ipy_prompt)
diff --git a/docs/source/conf.py b/docs/source/conf.py
index 20ef8f9..7d99ebd 100755
--- a/docs/source/conf.py
+++ b/docs/source/conf.py
@@ -143,7 +143,8 @@ today_fmt = '%B %d, %Y'
# Exclude these glob-style patterns when looking for source files. They are
# relative to the source/ directory.
-exclude_patterns = ['whatsnew/pr']
+exclude_patterns = ['whatsnew/pr/antigravity-feature.*',
+ 'whatsnew/pr/incompat-switching-to-perl.*']
# If true, '()' will be appended to :func: etc. cross-reference text.
diff --git a/docs/source/interactive/autoawait.rst b/docs/source/interactive/autoawait.rst
new file mode 100644
index 0000000..3d84e49
--- /dev/null
+++ b/docs/source/interactive/autoawait.rst
@@ -0,0 +1,186 @@
+
+.. autoawait:
+
+Asynchronous in REPL: Autoawait
+===============================
+
+Starting with IPython 6.0, and when user Python 3.6 and above, IPython offer the
+ability to run asynchronous code from the REPL. constructs which are
+:exc:`SyntaxError` s in the Python REPL can be used seamlessly in IPython.
+
+When a supported libray is used, IPython will automatically `await` Futures
+and Coroutines in the REPL. This will happen if an :ref:`await <await>` (or `async`) is
+use at top level scope, or if any structure valid only in `async def
+<https://docs.python.org/3/reference/compound_stmts.html#async-def>`_ function
+context are present. For example, the following being a syntax error in the
+Python REPL::
+
+ Python 3.6.0
+ [GCC 4.2.1]
+ Type "help", "copyright", "credits" or "license" for more information.
+ >>> import aiohttp
+ >>> result = aiohttp.get('https://api.github.com')
+ >>> response = await result
+ File "<stdin>", line 1
+ response = await result
+ ^
+ SyntaxError: invalid syntax
+
+Should behave as expected in the IPython REPL::
+
+ Python 3.6.0
+ Type 'copyright', 'credits' or 'license' for more information
+ IPython 6.0.0.dev -- An enhanced Interactive Python. Type '?' for help.
+
+ In [1]: import aiohttp
+ ...: result = aiohttp.get('https://api.github.com')
+
+ In [2]: response = await result
+ <pause for a few 100s ms>
+
+ In [3]: await response.json()
+ Out[3]:
+ {'authorizations_url': 'https://api.github.com/authorizations',
+ 'code_search_url': 'https://api.github.com/search/code?q={query}...',
+ ...
+ }
+
+
+You can use the ``c.InteractiveShell.autoawait`` configuration option and set it
+to :any:`False` to deactivate automatic wrapping of asynchronous code. You can also
+use the :magic:`%autoawait` magic to toggle the behavior at runtime::
+
+ In [1]: %autoawait False
+
+ In [2]: %autoawait
+ IPython autoawait is `Off`, and set to use `IPython.core.interactiveshell._asyncio_runner`
+
+
+
+By default IPython will assume integration with Python's provided
+:mod:`asyncio`, but integration with other libraries is provided. In particular
+we provide experimental integration with the ``curio`` and ``trio`` library.
+
+You can switch current integration by using the
+``c.InteractiveShell.loop_runner`` option or the ``autoawait <name
+integration>`` magic.
+
+For example::
+
+ In [1]: %autoawait trio
+
+ In [2]: import trio
+
+ In [3]: async def child(i):
+ ...: print(" child %s goes to sleep"%i)
+ ...: await trio.sleep(2)
+ ...: print(" child %s wakes up"%i)
+
+ In [4]: print('parent start')
+ ...: async with trio.open_nursery() as n:
+ ...: for i in range(5):
+ ...: n.spawn(child, i)
+ ...: print('parent end')
+ parent start
+ child 2 goes to sleep
+ child 0 goes to sleep
+ child 3 goes to sleep
+ child 1 goes to sleep
+ child 4 goes to sleep
+ <about 2 seconds pause>
+ child 2 wakes up
+ child 1 wakes up
+ child 0 wakes up
+ child 3 wakes up
+ child 4 wakes up
+ parent end
+
+
+In the above example, ``async with`` at top level scope is a syntax error in
+Python.
+
+Using this mode can have unexpected consequences if used in interaction with
+other features of IPython and various registered extensions. In particular if you
+are a direct or indirect user of the AST transformers, these may not apply to
+your code.
+
+The default loop, or runner does not run in the background, so top level
+asynchronous code must finish for the REPL to allow you to enter more code. As
+with usual Python semantic, the awaitables are started only when awaited for the
+first time. That is to say, in first example, no network request is done between
+``In[1]`` and ``In[2]``.
+
+
+Internals
+=========
+
+As running asynchronous code is not supported in interactive REPL as of Python
+3.6 we have to rely to a number of complex workaround to allow this to happen.
+It is interesting to understand how this works in order to understand potential
+bugs, or provide a custom runner.
+
+Among the many approaches that are at our disposition, we find only one that
+suited out need. Under the hood we :ct the code object from a async-def function
+and run it in global namesapace after modifying the ``__code__`` object.::
+
+ async def inner_async():
+ locals().update(**global_namespace)
+ #
+ # here is user code
+ #
+ return last_user_statement
+ codeobj = modify(inner_async.__code__)
+ coroutine = eval(codeobj, user_ns)
+ display(loop_runner(coroutine))
+
+
+
+The first thing you'll notice is that unlike classical ``exec``, there is only
+one name space. Second, user code runs in a function scope, and not a module
+scope.
+
+On top of the above there are significant modification to the AST of
+``function``, and ``loop_runner`` can be arbitrary complex. So there is a
+significant overhead to this kind of code.
+
+By default the generated coroutine function will be consumed by Asyncio's
+``loop_runner = asyncio.get_evenloop().run_until_complete()`` method. It is
+though possible to provide your own.
+
+A loop runner is a *synchronous* function responsible from running a coroutine
+object.
+
+The runner is responsible from ensuring that ``coroutine`` run to completion,
+and should return the result of executing the coroutine. Let's write a
+runner for ``trio`` that print a message when used as an exercise, ``trio`` is
+special as it usually prefer to run a function object and make a coroutine by
+itself, we can get around this limitation by wrapping it in an async-def without
+parameters and passing this value to ``trio``::
+
+
+ In [1]: import trio
+ ...: from types import CoroutineType
+ ...:
+ ...: def trio_runner(coro:CoroutineType):
+ ...: print('running asynchronous code')
+ ...: async def corowrap(coro):
+ ...: return await coro
+ ...: return trio.run(corowrap, coro)
+
+We can set it up by passing it to ``%autoawait``::
+
+ In [2]: %autoawait trio_runner
+
+ In [3]: async def async_hello(name):
+ ...: await trio.sleep(1)
+ ...: print(f'Hello {name} world !')
+ ...: await trio.sleep(1)
+
+ In [4]: await async_hello('async')
+ running asynchronous code
+ Hello async world !
+
+
+Asynchronous programming in python (and in particular in the REPL) is still a
+relatively young subject. Feel free to contribute improvements to this codebase
+and give us feedback.
diff --git a/docs/source/interactive/index.rst b/docs/source/interactive/index.rst
index 97332e1..7010c51 100644
--- a/docs/source/interactive/index.rst
+++ b/docs/source/interactive/index.rst
@@ -21,6 +21,7 @@ done some work in the classic Python REPL.
plotting
reference
shell
+ autoawait
tips
python-ipython-diff
magics
diff --git a/docs/source/whatsnew/development.rst b/docs/source/whatsnew/development.rst
index 13f02a4..4a62f47 100644
--- a/docs/source/whatsnew/development.rst
+++ b/docs/source/whatsnew/development.rst
@@ -11,6 +11,135 @@ This document describes in-flight development work.
`docs/source/whatsnew/pr` folder
+Released .... ...., 2017
+
+
+Need to be updated:
+
+.. toctree::
+ :maxdepth: 2
+ :glob:
+
+ pr/*
+
+IPython 6 feature a major improvement in the completion machinery which is now
+capable of completing non-executed code. It is also the first version of IPython
+to stop compatibility with Python 2, which is still supported on the bugfix only
+5.x branch. Read below to have a non-exhaustive list of new features.
+
+Make sure you have pip > 9.0 before upgrading.
+You should be able to update by using:
+
+.. code::
+
+ pip install ipython --upgrade
+
+New completion API and Interface
+--------------------------------
+
+The completer Completion API has seen an overhaul, and the new completer have
+plenty of improvement both from the end users of terminal IPython or for
+consumers of the API.
+
+This new API is capable of pulling completions from :any:`jedi`, thus allowing
+type inference on non-executed code. If :any:`jedi` is installed completion like
+the following are now becoming possible without code evaluation:
+
+ >>> data = ['Number of users', 123_456]
+ ... data[0].<tab>
+
+That is to say, IPython is now capable of inferring that `data[0]` is a string,
+and will suggest completions like `.capitalize`. The completion power of IPython
+will increase with new Jedi releases, and a number of bugs and more completions
+are already available on development version of :any:`jedi` if you are curious.
+
+With the help of prompt toolkit, types of completions can be shown in the
+completer interface:
+
+.. image:: ../_images/jedi_type_inference_60.png
+ :alt: Jedi showing ability to do type inference
+ :align: center
+ :width: 400px
+ :target: ../_images/jedi_type_inference_60.png
+
+The appearance of the completer is controlled by the
+``c.TerminalInteractiveShell.display_completions`` option that will show the
+type differently depending on the value among ``'column'``, ``'multicolumn'``
+and ``'readlinelike'``
+
+The use of Jedi also full fill a number of request and fix a number of bugs
+like case insensitive completion, completion after division operator: See
+:ghpull:`10182`.
+
+Extra patches and updates will be needed to the :mod:`ipykernel` package for
+this feature to be available to other clients like jupyter Notebook, Lab,
+Nteract, Hydrogen...
+
+The use of Jedi can is barely noticeable on recent enough machines, but can be
+feel on older ones, in cases were Jedi behavior need to be adjusted, the amount
+of time given to Jedi to compute type inference can be adjusted with
+``c.IPCompleter.jedi_compute_type_timeout``, with object whose type were not
+inferred will be shown as ``<unknown>``. Jedi can also be completely deactivated
+by using the ``c.Completer.use_jedi=False`` option.
+
+
+The old ``Completer.complete()`` API is waiting deprecation and should be
+replaced replaced by ``Completer.completions()`` in a near future. Feedback on
+the current state of the API and suggestions welcome.
+
+Python 3 only codebase
+----------------------
+
+One of the large challenges in IPython 6.0 has been the adoption of a pure
+Python 3 code base, which lead us to great length to upstream patches in pip,
+pypi and warehouse to make sure Python 2 system still upgrade to the latest
+compatible Python version compatible.
+
+We remind our Python 2 users that IPython 5 is still compatible with Python 2.7,
+still maintained and get regular releases. Using pip 9+, upgrading IPython will
+automatically upgrade to the latest version compatible with your system.
+
+.. warning::
+
+ If you are on a system using an older verison of pip on Python 2, pip may
+ still install IPython 6.0 on your system, and IPython will refuse to start.
+ You can fix this by ugrading pip, and reinstalling ipython, or forcing pip to
+ install an earlier version: ``pip install 'ipython<6'``
+
+The ability to use only Python 3 on the code base of IPython has bring a number
+of advantage. Most of the newly written code make use of `optional function type
+anotation <https://www.python.org/dev/peps/pep-0484/>`_ leading to clearer code
+and better documentation.
+
+The total size of the repository has also for a first time between releases
+(excluding the big split for 4.0) decreased by about 1500 lines, potentially
+quite a bit more codewide as some documents like this one are append only and
+are about 300 lines long.
+
+The removal as of Python2/Python3 shim layer has made the code quite clearer and
+more idiomatic in a number of location, and much friendlier to work with and
+understand. We hope to further embrace Python 3 capability in the next release
+cycle and introduce more of the Python 3 only idioms (yield from, kwarg only,
+general unpacking) in the code base of IPython, and see if we can take advantage
+of these as well to improve user experience with better error messages and
+hints.
+
+
+Miscs improvements
+------------------
+
+
+- The :cellmagic:`capture` magic can now capture the result of a cell (from an
+ expression on the last line), as well as printed and displayed output.
+ :ghpull:`9851`.
+
+- Pressing Ctrl-Z in the terminal debugger now suspends IPython, as it already
+ does in the main terminal prompt.
+
+- autoreload can now reload ``Enum``. See :ghissue:`10232` and :ghpull:`10316`
+
+- IPython.display has gained a :any:`GeoJSON <IPython.display.GeoJSON>` object.
+ :ghpull:`10288` and :ghpull:`10253`
.. DO NOT EDIT THIS LINE BEFORE RELEASE. FEATURE INSERTION POINT.
diff --git a/docs/source/whatsnew/pr/await-repl.rst b/docs/source/whatsnew/pr/await-repl.rst
new file mode 100644
index 0000000..614a00a
--- /dev/null
+++ b/docs/source/whatsnew/pr/await-repl.rst
@@ -0,0 +1,55 @@
+Await REPL
+----------
+
+:ghpull:`10390` introduced the ability to ``await`` Futures and
+Coroutines in the REPL. For example::
+
+ Python 3.6.0
+ Type 'copyright', 'credits' or 'license' for more information
+ IPython 6.0.0.dev -- An enhanced Interactive Python. Type '?' for help.
+
+ In [1]: import aiohttp
+ ...: result = aiohttp.get('https://api.github.com')
+
+ In [2]: response = await result
+ <pause for a few 100s ms>
+
+ In [3]: await response.json()
+ Out[3]:
+ {'authorizations_url': 'https://api.github.com/authorizations',
+ 'code_search_url': 'https://api.github.com/search/code?q={query}{&page,per_page,sort,order}',
+ ...
+ }
+
+
+Integration is by default with `asyncio`, but other libraries can be configured,
+like ``curio`` or ``trio``, to improve concurrency in the REPL::
+
+ In [1]: %autoawait trio
+
+ In [2]: import trio
+
+ In [3]: async def child(i):
+ ...: print(" child %s goes to sleep"%i)
+ ...: await trio.sleep(2)
+ ...: print(" child %s wakes up"%i)
+
+ In [4]: print('parent start')
+ ...: async with trio.open_nursery() as n:
+ ...: for i in range(3):
+ ...: n.spawn(child, i)
+ ...: print('parent end')
+ parent start
+ child 2 goes to sleep
+ child 0 goes to sleep
+ child 1 goes to sleep
+ <about 2 seconds pause>
+ child 2 wakes up
+ child 1 wakes up
+ child 0 wakes up
+ parent end
+
+See :ref:`autoawait` for more information.
+
+
+
diff --git a/setup.py b/setup.py
index bd72088..ed18841 100755
--- a/setup.py
+++ b/setup.py
@@ -201,6 +201,7 @@ install_requires = [
extras_require.update({
':python_version == "3.4"': ['typing'],
+ ':python_version >= "3.5"': ['trio', 'curio'],
':sys_platform != "win32"': ['pexpect'],
':sys_platform == "darwin"': ['appnope'],
':sys_platform == "win32"': ['colorama'],