some more fixes for 3.8

IPython/core/async_helpers.py

@@ -1,171 +1,173 b''
 """
 Async helper function that are invalid syntax on Python 3.5 and below.
 
 This code is best effort, and may have edge cases not behaving as expected. In
 particular it contain a number of heuristics to detect whether code is
 effectively async and need to run in an event loop or not.
 
 Some constructs (like top-level `return`, or `yield`) are taken care of
 explicitly to actually raise a SyntaxError and stay as close as possible to
 Python semantics.
 """
 
 
 import ast
 import sys
 import inspect
 from textwrap import dedent, indent
 
 
 class _AsyncIORunner:
 
     def __call__(self, coro):
         """
         Handler for asyncio autoawait
         """
         import asyncio
 
         return asyncio.get_event_loop().run_until_complete(coro)
 
     def __str__(self):
         return 'asyncio'
 
 _asyncio_runner = _AsyncIORunner()
 
 
 def _curio_runner(coroutine):
     """
     handler for curio autoawait
     """
     import curio
 
     return curio.run(coroutine)
 
 
 def _trio_runner(async_fn):
     import trio
 
     async def loc(coro):
         """
         We need the dummy no-op async def to protect from
         trio's internal. See https://github.com/python-trio/trio/issues/89
         """
         return await coro
 
     return trio.run(loc, async_fn)
 
 
 def _pseudo_sync_runner(coro):
     """
     A runner that does not really allow async execution, and just advance the coroutine.
 
     See discussion in https://github.com/python-trio/trio/issues/608,
 
     Credit to Nathaniel Smith
 
     """
     try:
         coro.send(None)
     except StopIteration as exc:
         return exc.value
     else:
         # TODO: do not raise but return an execution result with the right info.
         raise RuntimeError(
             "{coro_name!r} needs a real async loop".format(coro_name=coro.__name__)
         )
 
 
 def _asyncify(code: str) -> str:
     """wrap code in async def definition.
 
     And setup a bit of context to run it later.
     """
     res = dedent(
         """
     async def __wrapper__():
         try:
     {usercode}
         finally:
             locals()
     """
     ).format(usercode=indent(code, " " * 8))
     return res
 
 
 class _AsyncSyntaxErrorVisitor(ast.NodeVisitor):
     """
     Find syntax errors that would be an error in an async repl, but because
     the implementation involves wrapping the repl in an async function, it
     is erroneously allowed (e.g. yield or return at the top level)
     """
     def __init__(self):
         if sys.version_info >= (3,8):
             raise ValueError('DEPRECATED in Python 3.8+')
         self.depth = 0
         super().__init__()
 
     def generic_visit(self, node):
         func_types = (ast.FunctionDef, ast.AsyncFunctionDef)
         invalid_types_by_depth = {
             0: (ast.Return, ast.Yield, ast.YieldFrom),
             1: (ast.Nonlocal,)
         }
 
         should_traverse = self.depth < max(invalid_types_by_depth.keys())
         if isinstance(node, func_types) and should_traverse:
             self.depth += 1
             super().generic_visit(node)
             self.depth -= 1
         elif isinstance(node, invalid_types_by_depth[self.depth]):
             raise SyntaxError()
         else:
             super().generic_visit(node)
 
 
 def _async_parse_cell(cell: str) -> ast.AST:
     """
     This is a compatibility shim for pre-3.7 when async outside of a function
     is a syntax error at the parse stage.
 
     It will return an abstract syntax tree parsed as if async and await outside
     of a function were not a syntax error.
     """
     if sys.version_info < (3, 7):
         # Prior to 3.7 you need to asyncify before parse
         wrapped_parse_tree = ast.parse(_asyncify(cell))
         return wrapped_parse_tree.body[0].body[0]
     else:
         return ast.parse(cell)
 
 
 def _should_be_async(cell: str) -> bool:
     """Detect if a block of code need to be wrapped in an `async def`
 
     Attempt to parse the block of code, it it compile we're fine.
     Otherwise we  wrap if and try to compile.
 
     If it works, assume it should be async. Otherwise Return False.
 
     Not handled yet: If the block of code has a return statement as  the top
     level, it will be seen as async. This is a know limitation.
     """
     if sys.version_info > (3, 8):
+        try:
         code = compile(cell, "<>", "exec", flags=getattr(ast,'PyCF_ALLOW_TOP_LEVEL_AWAIT', 0x0))
         return inspect.CO_COROUTINE & code.co_flags == inspect.CO_COROUTINE
-
+        except SyntaxError:
+            return False
     try:
         # we can't limit ourself to ast.parse, as it __accepts__ to parse on
         # 3.7+, but just does not _compile_
         code = compile(cell, "<>", "exec")
     except SyntaxError:
         try:
             parse_tree = _async_parse_cell(cell)
 
             # Raise a SyntaxError if there are top-level return or yields
             v = _AsyncSyntaxErrorVisitor()
             v.visit(parse_tree)
 
         except SyntaxError:
             return False
         return True
     return False

IPython/core/compilerop.py

@@ -1,155 +1,160 b''
 """Compiler tools with improved interactive support.
 
 Provides compilation machinery similar to codeop, but with caching support so
 we can provide interactive tracebacks.
 
 Authors
 -------
 * Robert Kern
 * Fernando Perez
 * Thomas Kluyver
 """
 
 # Note: though it might be more natural to name this module 'compiler', that
 # name is in the stdlib and name collisions with the stdlib tend to produce
 # weird problems (often with third-party tools).
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2010-2011 The IPython Development Team.
 #
 #  Distributed under the terms of the BSD License.
 #
 #  The full license is in the file COPYING.txt, distributed with this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 # Stdlib imports
 import __future__
 from ast import PyCF_ONLY_AST
 import codeop
 import functools
 import hashlib
 import linecache
 import operator
 import time
 from contextlib import contextmanager
 
 #-----------------------------------------------------------------------------
 # Constants
 #-----------------------------------------------------------------------------
 
 # Roughly equal to PyCF_MASK | PyCF_MASK_OBSOLETE as defined in pythonrun.h,
 # this is used as a bitmask to extract future-related code flags.
 PyCF_MASK = functools.reduce(operator.or_,
                              (getattr(__future__, fname).compiler_flag
                               for fname in __future__.all_feature_names))
 
 #-----------------------------------------------------------------------------
 # Local utilities
 #-----------------------------------------------------------------------------
 
 def code_name(code, number=0):
     """ Compute a (probably) unique name for code for caching.
 
     This now expects code to be unicode.
     """
     hash_digest = hashlib.sha1(code.encode("utf-8")).hexdigest()
     # Include the number and 12 characters of the hash in the name.  It's
     # pretty much impossible that in a single session we'll have collisions
     # even with truncated hashes, and the full one makes tracebacks too long
     return '<ipython-input-{0}-{1}>'.format(number, hash_digest[:12])
 
 #-----------------------------------------------------------------------------
 # Classes and functions
 #-----------------------------------------------------------------------------
 
 class CachingCompiler(codeop.Compile):
     """A compiler that caches code compiled from interactive statements.
     """
 
     def __init__(self):
         codeop.Compile.__init__(self)
 
         # This is ugly, but it must be done this way to allow multiple
         # simultaneous ipython instances to coexist.  Since Python itself
         # directly accesses the data structures in the linecache module, and
         # the cache therein is global, we must work with that data structure.
         # We must hold a reference to the original checkcache routine and call
         # that in our own check_cache() below, but the special IPython cache
         # must also be shared by all IPython instances.  If we were to hold
         # separate caches (one in each CachingCompiler instance), any call made
         # by Python itself to linecache.checkcache() would obliterate the
         # cached data from the other IPython instances.
         if not hasattr(linecache, '_ipython_cache'):
             linecache._ipython_cache = {}
         if not hasattr(linecache, '_checkcache_ori'):
             linecache._checkcache_ori = linecache.checkcache
         # Now, we must monkeypatch the linecache directly so that parts of the
         # stdlib that call it outside our control go through our codepath
         # (otherwise we'd lose our tracebacks).
         linecache.checkcache = check_linecache_ipython
 
 
     def ast_parse(self, source, filename='<unknown>', symbol='exec'):
         """Parse code to an AST with the current compiler flags active.
 
         Arguments are exactly the same as ast.parse (in the standard library),
         and are passed to the built-in compile function."""
         return compile(source, filename, symbol, self.flags | PyCF_ONLY_AST, 1)
     
     def reset_compiler_flags(self):
         """Reset compiler flags to default state."""
         # This value is copied from codeop.Compile.__init__, so if that ever
         # changes, it will need to be updated.
         self.flags = codeop.PyCF_DONT_IMPLY_DEDENT
 
     @property
     def compiler_flags(self):
         """Flags currently active in the compilation process.
         """
         return self.flags
 
     def cache(self, code, number=0):
         """Make a name for a block of code, and cache the code.
 
         Parameters
         ----------
         code : str
           The Python source code to cache.
         number : int
           A number which forms part of the code's name. Used for the execution
           counter.
 
         Returns
         -------
         The name of the cached code (as a string). Pass this as the filename
         argument to compilation, so that tracebacks are correctly hooked up.
         """
         name = code_name(code, number)
         entry = (len(code), time.time(),
                  [line+'\n' for line in code.splitlines()], name)
         linecache.cache[name] = entry
         linecache._ipython_cache[name] = entry
         return name
 
     @contextmanager
     def extra_flags(self, flags):
-        old_flags = self.flags
+        ## bits that we'll set to 1
+        turn_on_bits = ~self.flags & flags
+
+
         self.flags = self.flags | flags
         try:
             yield
         finally:
-            self.flags = old_flags
+            # turn off only the bits we turned on so that something like
+            # __future__ that set flags stays. 
+            self.flags &= ~turn_on_bits
 
 
 def check_linecache_ipython(*args):
     """Call linecache.checkcache() safely protecting our cached values.
     """
     # First call the original checkcache as intended
     linecache._checkcache_ori(*args)
     # Then, update back the cache with our data, so that tracebacks related
     # to our compiled codes can be produced.
     linecache.cache.update(linecache._ipython_cache)