avoid deprecated get_event_loop

use our own `async_helpers.get_asyncio_loop` to track the global event loop

script magics use dedicated background asyncio loop
instead of trying to work on the main loop, which may or may not exist

_AsyncIOProxy wraps background script objects to transfer awaitables across loops

only works for coroutine methods, which might be good enough? Works for read, etc.

IPython/core/async_helpers.py

@@ -1,103 +1,156 b''
 """
 Async helper function that are invalid syntax on Python 3.5 and below.
 
 This code is best effort, and may have edge cases not behaving as expected. In
 particular it contain a number of heuristics to detect whether code is
 effectively async and need to run in an event loop or not.
 
 Some constructs (like top-level `return`, or `yield`) are taken care of
 explicitly to actually raise a SyntaxError and stay as close as possible to
 Python semantics.
 """
 
 
 import ast
 import asyncio
 import inspect
+from functools import wraps
 
+_asyncio_event_loop = None
 
-class _AsyncIORunner:
-    def __init__(self):
-        self._loop = None
-
-    @property
-    def loop(self):
-        """Always returns a non-closed event loop"""
-        if self._loop is None or self._loop.is_closed():
-            policy = asyncio.get_event_loop_policy()
-            self._loop = policy.new_event_loop()
-            policy.set_event_loop(self._loop)
-        return self._loop
 
+def get_asyncio_loop():
+    """asyncio has deprecated get_event_loop
+
+    Replicate it here, with our desired semantics:
+
+    - always returns a valid, not-closed loop
+    - not thread-local like asyncio's,
+      because we only want one loop for IPython
+    - if called from inside a coroutine (e.g. in ipykernel),
+      return the running loop
+
+    .. versionadded:: 8.0
+    """
+    try:
+        return asyncio.get_running_loop()
+    except RuntimeError:
+        # not inside a coroutine,
+        # track our own global
+        pass
+
+    # not thread-local like asyncio's,
+    # because we only track one event loop to run for IPython itself,
+    # always in the main thread.
+    global _asyncio_event_loop
+    if _asyncio_event_loop is None or _asyncio_event_loop.is_closed():
+        _asyncio_event_loop = asyncio.new_event_loop()
+    return _asyncio_event_loop
+
+
+class _AsyncIORunner:
     def __call__(self, coro):
         """
         Handler for asyncio autoawait
         """
-        return self.loop.run_until_complete(coro)
+        return get_asyncio_loop().run_until_complete(coro)
 
     def __str__(self):
         return "asyncio"
 
 
 _asyncio_runner = _AsyncIORunner()
 
 
+class _AsyncIOProxy:
+    """Proxy-object for an asyncio
+
+    Any coroutine methods will be wrapped in event_loop.run_
+    """
+
+    def __init__(self, obj, event_loop):
+        self._obj = obj
+        self._event_loop = event_loop
+
+    def __repr__(self):
+        return f"<_AsyncIOProxy({self._obj!r})>"
+
+    def __getattr__(self, key):
+        attr = getattr(self._obj, key)
+        if inspect.iscoroutinefunction(attr):
+            # if it's a coroutine method,
+            # return a threadsafe wrapper onto the _current_ asyncio loop
+            @wraps(attr)
+            def _wrapped(*args, **kwargs):
+                concurrent_future = asyncio.run_coroutine_threadsafe(
+                    attr(*args, **kwargs), self._event_loop
+                )
+                return asyncio.wrap_future(concurrent_future)
+
+            return _wrapped
+        else:
+            return attr
+
+    def __dir__(self):
+        return dir(self._obj)
+
+
 def _curio_runner(coroutine):
     """
     handler for curio autoawait
     """
     import curio
 
     return curio.run(coroutine)
 
 
 def _trio_runner(async_fn):
     import trio
 
     async def loc(coro):
         """
         We need the dummy no-op async def to protect from
         trio's internal. See https://github.com/python-trio/trio/issues/89
         """
         return await coro
 
     return trio.run(loc, async_fn)
 
 
 def _pseudo_sync_runner(coro):
     """
     A runner that does not really allow async execution, and just advance the coroutine.
 
     See discussion in https://github.com/python-trio/trio/issues/608,
 
     Credit to Nathaniel Smith
     """
     try:
         coro.send(None)
     except StopIteration as exc:
         return exc.value
     else:
         # TODO: do not raise but return an execution result with the right info.
         raise RuntimeError(
             "{coro_name!r} needs a real async loop".format(coro_name=coro.__name__)
         )
 
 
 def _should_be_async(cell: str) -> bool:
     """Detect if a block of code need to be wrapped in an `async def`
 
     Attempt to parse the block of code, it it compile we're fine.
     Otherwise we  wrap if and try to compile.
 
     If it works, assume it should be async. Otherwise Return False.
 
     Not handled yet: If the block of code has a return statement as the top
     level, it will be seen as async. This is a know limitation.
     """
     try:
         code = compile(
             cell, "<>", "exec", flags=getattr(ast, "PyCF_ALLOW_TOP_LEVEL_AWAIT", 0x0)
         )
         return inspect.CO_COROUTINE & code.co_flags == inspect.CO_COROUTINE
     except (SyntaxError, MemoryError):
         return False

IPython/core/magics/script.py

@@ -1,375 +1,362 b''
 """Magic functions for running cells in various scripts."""
 
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
 import asyncio
 import atexit
 import errno
-import functools
 import os
 import signal
 import sys
 import time
-from contextlib import contextmanager
 from subprocess import CalledProcessError
+from threading import Thread
 
-from traitlets import Dict, List, default
+from traitlets import Any, Dict, List, default
 
 from IPython.core import magic_arguments
+from IPython.core.async_helpers import _AsyncIOProxy
 from IPython.core.magic import Magics, cell_magic, line_magic, magics_class
-from IPython.lib.backgroundjobs import BackgroundJobManager
 from IPython.utils.process import arg_split
 
 #-----------------------------------------------------------------------------
 # Magic implementation classes
 #-----------------------------------------------------------------------------
 
 def script_args(f):
     """single decorator for adding script args"""
     args = [
         magic_arguments.argument(
             '--out', type=str,
             help="""The variable in which to store stdout from the script.
             If the script is backgrounded, this will be the stdout *pipe*,
             instead of the stderr text itself and will not be auto closed.
             """
         ),
         magic_arguments.argument(
             '--err', type=str,
             help="""The variable in which to store stderr from the script.
             If the script is backgrounded, this will be the stderr *pipe*,
             instead of the stderr text itself and will not be autoclosed.
             """
         ),
         magic_arguments.argument(
             '--bg', action="store_true",
             help="""Whether to run the script in the background.
             If given, the only way to see the output of the command is
             with --out/err.
             """
         ),
         magic_arguments.argument(
             '--proc', type=str,
             help="""The variable in which to store Popen instance.
             This is used only when --bg option is given.
             """
         ),
         magic_arguments.argument(
             '--no-raise-error', action="store_false", dest='raise_error',
-            help="""Whether you should raise an error message in addition to 
+            help="""Whether you should raise an error message in addition to
             a stream on stderr if you get a nonzero exit code.
             """
         )
     ]
     for arg in args:
         f = arg(f)
     return f
 
 
-@contextmanager
-def safe_watcher():
-    if sys.platform == "win32":
-        yield
-        return
-
-    from asyncio import SafeChildWatcher
-
-    policy = asyncio.get_event_loop_policy()
-    old_watcher = policy.get_child_watcher()
-    if isinstance(old_watcher, SafeChildWatcher):
-        yield
-        return
-
-    try:
-        loop = policy.get_event_loop()
-        if loop.is_closed():
-            raise RuntimeError("open a new one")
-    except RuntimeError:
-        # closed loop, make a new one
-        loop = policy.new_event_loop()
-        policy.set_event_loop(loop)
-
-    try:
-        watcher = asyncio.SafeChildWatcher()
-        watcher.attach_loop(loop)
-        policy.set_child_watcher(watcher)
-        yield
-    finally:
-        watcher.close()
-        policy.set_child_watcher(old_watcher)
-
-
-def dec_safe_watcher(fun):
-    @functools.wraps(fun)
-    def _inner(*args, **kwargs):
-        with safe_watcher():
-            return fun(*args, **kwargs)
-
-    return _inner
-
-
 @magics_class
 class ScriptMagics(Magics):
     """Magics for talking to scripts
     
     This defines a base `%%script` cell magic for running a cell
     with a program in a subprocess, and registers a few top-level
     magics that call %%script with common interpreters.
     """
+
+    event_loop = Any(
+        help="""
+        The event loop on which to run subprocesses
+
+        Not the main event loop,
+        because we want to be able to make blocking calls
+        and have certain requirements we don't want to impose on the main loop.
+        """
+    )
+
     script_magics = List(
         help="""Extra script cell magics to define
         
         This generates simple wrappers of `%%script foo` as `%%foo`.
         
         If you want to add script magics that aren't on your path,
         specify them in script_paths
         """,
     ).tag(config=True)
     @default('script_magics')
     def _script_magics_default(self):
         """default to a common list of programs"""
         
         defaults = [
             'sh',
             'bash',
             'perl',
             'ruby',
             'python',
             'python2',
             'python3',
             'pypy',
         ]
         if os.name == 'nt':
             defaults.extend([
                 'cmd',
             ])
         
         return defaults
     
     script_paths = Dict(
         help="""Dict mapping short 'ruby' names to full paths, such as '/opt/secret/bin/ruby'
         
         Only necessary for items in script_magics where the default path will not
         find the right interpreter.
         """
     ).tag(config=True)
     
     def __init__(self, shell=None):
         super(ScriptMagics, self).__init__(shell=shell)
         self._generate_script_magics()
-        self.job_manager = BackgroundJobManager()
         self.bg_processes = []
         atexit.register(self.kill_bg_processes)
 
     def __del__(self):
         self.kill_bg_processes()
     
     def _generate_script_magics(self):
         cell_magics = self.magics['cell']
         for name in self.script_magics:
             cell_magics[name] = self._make_script_magic(name)
     
     def _make_script_magic(self, name):
         """make a named magic, that calls %%script with a particular program"""
         # expand to explicit path if necessary:
         script = self.script_paths.get(name, name)
         
         @magic_arguments.magic_arguments()
         @script_args
         def named_script_magic(line, cell):
             # if line, add it as cl-flags
             if line:
                 line = "%s %s" % (script, line)
             else:
                 line = script
             return self.shebang(line, cell)
         
         # write a basic docstring:
         named_script_magic.__doc__ = \
         """%%{name} script magic
         
         Run cells with {script} in a subprocess.
         
         This is a shortcut for `%%script {script}`
         """.format(**locals())
         
         return named_script_magic
     
     @magic_arguments.magic_arguments()
     @script_args
     @cell_magic("script")
-    @dec_safe_watcher
     def shebang(self, line, cell):
         """Run a cell via a shell command
 
         The `%%script` line is like the #! line of script,
         specifying a program (bash, perl, ruby, etc.) with which to run.
 
         The rest of the cell is run by that program.
 
         Examples
         --------
         ::
 
             In [1]: %%script bash
                ...: for i in 1 2 3; do
                ...:   echo $i
                ...: done
             1
             2
             3
         """
 
+        # Create the event loop in which to run script magics
+        # this operates on a background thread
+        if self.event_loop is None:
+            if sys.platform == "win32":
+                # don't override the current policy,
+                # just create an event loop
+                event_loop = asyncio.WindowsProactorEventLoopPolicy().new_event_loop()
+            else:
+                event_loop = asyncio.new_event_loop()
+            self.event_loop = event_loop
+
+            # start the loop in a background thread
+            asyncio_thread = Thread(target=event_loop.run_forever, daemon=True)
+            asyncio_thread.start()
+        else:
+            event_loop = self.event_loop
+
+        def in_thread(coro):
+            """Call a coroutine on the asyncio thread"""
+            return asyncio.run_coroutine_threadsafe(coro, event_loop).result()
+
         async def _handle_stream(stream, stream_arg, file_object):
             while True:
                 line = (await stream.readline()).decode("utf8")
                 if not line:
                     break
                 if stream_arg:
                     self.shell.user_ns[stream_arg] = line
                 else:
                     file_object.write(line)
                     file_object.flush()
 
         async def _stream_communicate(process, cell):
             process.stdin.write(cell)
             process.stdin.close()
             stdout_task = asyncio.create_task(
                 _handle_stream(process.stdout, args.out, sys.stdout)
             )
             stderr_task = asyncio.create_task(
                 _handle_stream(process.stderr, args.err, sys.stderr)
             )
             await asyncio.wait([stdout_task, stderr_task])
             await process.wait()
 
-        policy = asyncio.get_event_loop_policy()
-        if sys.platform.startswith("win") and not isinstance(
-            policy, asyncio.WindowsProactorEventLoopPolicy
-        ):
-            # _do not_ overwrite the current policy
-            policy = asyncio.WindowsProactorEventLoopPolicy()
-
-        try:
-            loop = policy.get_event_loop()
-        except RuntimeError:
-            # closed loop, make a new one
-            loop = policy.new_event_loop()
-            policy.set_event_loop(loop)
         argv = arg_split(line, posix=not sys.platform.startswith("win"))
         args, cmd = self.shebang.parser.parse_known_args(argv)
+
         try:
-            p = loop.run_until_complete(
+            p = in_thread(
                 asyncio.create_subprocess_exec(
                     *cmd,
                     stdout=asyncio.subprocess.PIPE,
                     stderr=asyncio.subprocess.PIPE,
                     stdin=asyncio.subprocess.PIPE,
                 )
             )
         except OSError as e:
             if e.errno == errno.ENOENT:
                 print("Couldn't find program: %r" % cmd[0])
                 return
             else:
                 raise
-        
+
         if not cell.endswith('\n'):
             cell += '\n'
         cell = cell.encode('utf8', 'replace')
         if args.bg:
             self.bg_processes.append(p)
             self._gc_bg_processes()
             to_close = []
             if args.out:
-                self.shell.user_ns[args.out] = p.stdout
+                self.shell.user_ns[args.out] = _AsyncIOProxy(p.stdout, event_loop)
             else:
                 to_close.append(p.stdout)
             if args.err:
-                self.shell.user_ns[args.err] = p.stderr
+                self.shell.user_ns[args.err] = _AsyncIOProxy(p.stderr, event_loop)
             else:
                 to_close.append(p.stderr)
-            self.job_manager.new(self._run_script, p, cell, to_close, daemon=True)
+            event_loop.call_soon_threadsafe(
+                lambda: asyncio.Task(self._run_script(p, cell, to_close))
+            )
             if args.proc:
-                self.shell.user_ns[args.proc] = p
+                proc_proxy = _AsyncIOProxy(p, event_loop)
+                proc_proxy.stdout = _AsyncIOProxy(p.stdout, event_loop)
+                proc_proxy.stderr = _AsyncIOProxy(p.stderr, event_loop)
+                self.shell.user_ns[args.proc] = proc_proxy
             return
-        
+
         try:
-            loop.run_until_complete(_stream_communicate(p, cell))
+            in_thread(_stream_communicate(p, cell))
         except KeyboardInterrupt:
             try:
                 p.send_signal(signal.SIGINT)
-                time.sleep(0.1)
+                in_thread(asyncio.wait_for(p.wait(), timeout=0.1))
                 if p.returncode is not None:
                     print("Process is interrupted.")
                     return
                 p.terminate()
-                time.sleep(0.1)
+                in_thread(asyncio.wait_for(p.wait(), timeout=0.1))
                 if p.returncode is not None:
                     print("Process is terminated.")
                     return
                 p.kill()
                 print("Process is killed.")
             except OSError:
                 pass
             except Exception as e:
                 print("Error while terminating subprocess (pid=%i): %s" % (p.pid, e))
             return
-        if args.raise_error and p.returncode!=0:
+
+        if args.raise_error and p.returncode != 0:
             # If we get here and p.returncode is still None, we must have
             # killed it but not yet seen its return code. We don't wait for it,
             # in case it's stuck in uninterruptible sleep. -9 = SIGKILL
             rc = p.returncode or -9
             raise CalledProcessError(rc, cell)
 
     shebang.__skip_doctest__ = os.name != "posix"
-    
-    def _run_script(self, p, cell, to_close):
+
+    async def _run_script(self, p, cell, to_close):
         """callback for running the script in the background"""
+
         p.stdin.write(cell)
+        await p.stdin.drain()
         p.stdin.close()
+        await p.stdin.wait_closed()
+        await p.wait()
+        # asyncio read pipes have no close
+        # but we should drain the data anyway
         for s in to_close:
-            s.close()
-        p.wait()
+            await s.read()
+        self._gc_bg_processes()
 
     @line_magic("killbgscripts")
     def killbgscripts(self, _nouse_=''):
         """Kill all BG processes started by %%script and its family."""
         self.kill_bg_processes()
         print("All background processes were killed.")
 
     def kill_bg_processes(self):
         """Kill all BG processes which are still running."""
         if not self.bg_processes:
             return
         for p in self.bg_processes:
             if p.returncode is None:
                 try:
                     p.send_signal(signal.SIGINT)
                 except:
                     pass
         time.sleep(0.1)
         self._gc_bg_processes()
         if not self.bg_processes:
             return
         for p in self.bg_processes:
             if p.returncode is None:
                 try:
                     p.terminate()
                 except:
                     pass
         time.sleep(0.1)
         self._gc_bg_processes()
         if not self.bg_processes:
             return
         for p in self.bg_processes:
             if p.returncode is None:
                 try:
                     p.kill()
                 except:
                     pass
         self._gc_bg_processes()
 
     def _gc_bg_processes(self):
         self.bg_processes = [p for p in self.bg_processes if p.returncode is None]

IPython/core/tests/test_interactiveshell.py

@@ -1,1098 +1,1098 b''
 # -*- coding: utf-8 -*-
 """Tests for the key interactiveshell module.
 
 Historically the main classes in interactiveshell have been under-tested.  This
 module should grow as many single-method tests as possible to trap many of the
 recurring bugs we seem to encounter with high-level interaction.
 """
 
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
 import asyncio
 import ast
 import os
 import signal
 import shutil
 import sys
 import tempfile
 import unittest
 from unittest import mock
 
 from os.path import join
 
 from IPython.core.error import InputRejected
 from IPython.core.inputtransformer import InputTransformer
 from IPython.core import interactiveshell
 from IPython.testing.decorators import (
     skipif, skip_win32, onlyif_unicode_paths, onlyif_cmds_exist,
 )
 from IPython.testing import tools as tt
 from IPython.utils.process import find_cmd
 
 #-----------------------------------------------------------------------------
 # Globals
 #-----------------------------------------------------------------------------
 # This is used by every single test, no point repeating it ad nauseam
 
 #-----------------------------------------------------------------------------
 # Tests
 #-----------------------------------------------------------------------------
 
 class DerivedInterrupt(KeyboardInterrupt):
     pass
 
 class InteractiveShellTestCase(unittest.TestCase):
     def test_naked_string_cells(self):
         """Test that cells with only naked strings are fully executed"""
         # First, single-line inputs
         ip.run_cell('"a"\n')
         self.assertEqual(ip.user_ns['_'], 'a')
         # And also multi-line cells
         ip.run_cell('"""a\nb"""\n')
         self.assertEqual(ip.user_ns['_'], 'a\nb')
 
     def test_run_empty_cell(self):
         """Just make sure we don't get a horrible error with a blank
         cell of input. Yes, I did overlook that."""
         old_xc = ip.execution_count
         res = ip.run_cell('')
         self.assertEqual(ip.execution_count, old_xc)
         self.assertEqual(res.execution_count, None)
 
     def test_run_cell_multiline(self):
         """Multi-block, multi-line cells must execute correctly.
         """
         src = '\n'.join(["x=1",
                          "y=2",
                          "if 1:",
                          "    x += 1",
                          "    y += 1",])
         res = ip.run_cell(src)
         self.assertEqual(ip.user_ns['x'], 2)
         self.assertEqual(ip.user_ns['y'], 3)
         self.assertEqual(res.success, True)
         self.assertEqual(res.result, None)
 
     def test_multiline_string_cells(self):
         "Code sprinkled with multiline strings should execute (GH-306)"
         ip.run_cell('tmp=0')
         self.assertEqual(ip.user_ns['tmp'], 0)
         res = ip.run_cell('tmp=1;"""a\nb"""\n')
         self.assertEqual(ip.user_ns['tmp'], 1)
         self.assertEqual(res.success, True)
         self.assertEqual(res.result, "a\nb")
 
     def test_dont_cache_with_semicolon(self):
         "Ending a line with semicolon should not cache the returned object (GH-307)"
         oldlen = len(ip.user_ns['Out'])
         for cell in ['1;', '1;1;']:
             res = ip.run_cell(cell, store_history=True)
             newlen = len(ip.user_ns['Out'])
             self.assertEqual(oldlen, newlen)
             self.assertIsNone(res.result)
         i = 0
         #also test the default caching behavior
         for cell in ['1', '1;1']:
             ip.run_cell(cell, store_history=True)
             newlen = len(ip.user_ns['Out'])
             i += 1
             self.assertEqual(oldlen+i, newlen)
 
     def test_syntax_error(self):
         res = ip.run_cell("raise = 3")
         self.assertIsInstance(res.error_before_exec, SyntaxError)
 
     def test_In_variable(self):
         "Verify that In variable grows with user input (GH-284)"
         oldlen = len(ip.user_ns['In'])
         ip.run_cell('1;', store_history=True)
         newlen = len(ip.user_ns['In'])
         self.assertEqual(oldlen+1, newlen)
         self.assertEqual(ip.user_ns['In'][-1],'1;')
         
     def test_magic_names_in_string(self):
         ip.run_cell('a = """\n%exit\n"""')
         self.assertEqual(ip.user_ns['a'], '\n%exit\n')
     
     def test_trailing_newline(self):
         """test that running !(command) does not raise a SyntaxError"""
         ip.run_cell('!(true)\n', False)
         ip.run_cell('!(true)\n\n\n', False)
     
     def test_gh_597(self):
         """Pretty-printing lists of objects with non-ascii reprs may cause
         problems."""
         class Spam(object):
             def __repr__(self):
                 return "\xe9"*50
         import IPython.core.formatters
         f = IPython.core.formatters.PlainTextFormatter()
         f([Spam(),Spam()])
     
 
     def test_future_flags(self):
         """Check that future flags are used for parsing code (gh-777)"""
         ip.run_cell('from __future__ import barry_as_FLUFL')
         try:
             ip.run_cell('prfunc_return_val = 1 <> 2')
             assert 'prfunc_return_val' in ip.user_ns
         finally:
             # Reset compiler flags so we don't mess up other tests.
             ip.compile.reset_compiler_flags()
 
     def test_can_pickle(self):
         "Can we pickle objects defined interactively (GH-29)"
         ip = get_ipython()
         ip.reset()
         ip.run_cell(("class Mylist(list):\n"
                      "    def __init__(self,x=[]):\n"
                      "        list.__init__(self,x)"))
         ip.run_cell("w=Mylist([1,2,3])")
         
         from pickle import dumps
         
         # We need to swap in our main module - this is only necessary
         # inside the test framework, because IPython puts the interactive module
         # in place (but the test framework undoes this).
         _main = sys.modules['__main__']
         sys.modules['__main__'] = ip.user_module
         try:
             res = dumps(ip.user_ns["w"])
         finally:
             sys.modules['__main__'] = _main
         self.assertTrue(isinstance(res, bytes))
         
     def test_global_ns(self):
         "Code in functions must be able to access variables outside them."
         ip = get_ipython()
         ip.run_cell("a = 10")
         ip.run_cell(("def f(x):\n"
                      "    return x + a"))
         ip.run_cell("b = f(12)")
         self.assertEqual(ip.user_ns["b"], 22)
 
     def test_bad_custom_tb(self):
         """Check that InteractiveShell is protected from bad custom exception handlers"""
         ip.set_custom_exc((IOError,), lambda etype,value,tb: 1/0)
         self.assertEqual(ip.custom_exceptions, (IOError,))
         with tt.AssertPrints("Custom TB Handler failed", channel='stderr'):
             ip.run_cell(u'raise IOError("foo")')
         self.assertEqual(ip.custom_exceptions, ())
 
     def test_bad_custom_tb_return(self):
         """Check that InteractiveShell is protected from bad return types in custom exception handlers"""
         ip.set_custom_exc((NameError,),lambda etype,value,tb, tb_offset=None: 1)
         self.assertEqual(ip.custom_exceptions, (NameError,))
         with tt.AssertPrints("Custom TB Handler failed", channel='stderr'):
             ip.run_cell(u'a=abracadabra')
         self.assertEqual(ip.custom_exceptions, ())
 
     def test_drop_by_id(self):
         myvars = {"a":object(), "b":object(), "c": object()}
         ip.push(myvars, interactive=False)
         for name in myvars:
             assert name in ip.user_ns, name
             assert name in ip.user_ns_hidden, name
         ip.user_ns['b'] = 12
         ip.drop_by_id(myvars)
         for name in ["a", "c"]:
             assert name not in ip.user_ns, name
             assert name not in ip.user_ns_hidden, name
         assert ip.user_ns['b'] == 12
         ip.reset()
 
     def test_var_expand(self):
         ip.user_ns['f'] = u'Ca\xf1o'
         self.assertEqual(ip.var_expand(u'echo $f'), u'echo Ca\xf1o')
         self.assertEqual(ip.var_expand(u'echo {f}'), u'echo Ca\xf1o')
         self.assertEqual(ip.var_expand(u'echo {f[:-1]}'), u'echo Ca\xf1')
         self.assertEqual(ip.var_expand(u'echo {1*2}'), u'echo 2')
         
         self.assertEqual(ip.var_expand(u"grep x | awk '{print $1}'"), u"grep x | awk '{print $1}'")
 
         ip.user_ns['f'] = b'Ca\xc3\xb1o'
         # This should not raise any exception:
         ip.var_expand(u'echo $f')
    
     def test_var_expand_local(self):
         """Test local variable expansion in !system and %magic calls"""
         # !system
         ip.run_cell(
             "def test():\n"
             '    lvar = "ttt"\n'
             "    ret = !echo {lvar}\n"
             "    return ret[0]\n"
         )
         res = ip.user_ns["test"]()
         self.assertIn("ttt", res)
 
         # %magic
         ip.run_cell(
             "def makemacro():\n"
             '    macroname = "macro_var_expand_locals"\n'
             "    %macro {macroname} codestr\n"
         )
         ip.user_ns["codestr"] = "str(12)"
         ip.run_cell("makemacro()")
         self.assertIn("macro_var_expand_locals", ip.user_ns)
 
     def test_var_expand_self(self):
         """Test variable expansion with the name 'self', which was failing.
         
         See https://github.com/ipython/ipython/issues/1878#issuecomment-7698218
         """
         ip.run_cell(
             "class cTest:\n"
             '  classvar="see me"\n'
             "  def test(self):\n"
             "    res = !echo Variable: {self.classvar}\n"
             "    return res[0]\n"
         )
         self.assertIn("see me", ip.user_ns["cTest"]().test())
 
     def test_bad_var_expand(self):
         """var_expand on invalid formats shouldn't raise"""
         # SyntaxError
         self.assertEqual(ip.var_expand(u"{'a':5}"), u"{'a':5}")
         # NameError
         self.assertEqual(ip.var_expand(u"{asdf}"), u"{asdf}")
         # ZeroDivisionError
         self.assertEqual(ip.var_expand(u"{1/0}"), u"{1/0}")
     
     def test_silent_postexec(self):
         """run_cell(silent=True) doesn't invoke pre/post_run_cell callbacks"""
         pre_explicit = mock.Mock()
         pre_always = mock.Mock()
         post_explicit = mock.Mock()
         post_always = mock.Mock()
         all_mocks = [pre_explicit, pre_always, post_explicit, post_always]
         
         ip.events.register('pre_run_cell', pre_explicit)
         ip.events.register('pre_execute', pre_always)
         ip.events.register('post_run_cell', post_explicit)
         ip.events.register('post_execute', post_always)
         
         try:
             ip.run_cell("1", silent=True)
             assert pre_always.called
             assert not pre_explicit.called
             assert post_always.called
             assert not post_explicit.called
             # double-check that non-silent exec did what we expected
             # silent to avoid
             ip.run_cell("1")
             assert pre_explicit.called
             assert post_explicit.called
             info, = pre_explicit.call_args[0]
             result, = post_explicit.call_args[0]
             self.assertEqual(info, result.info)
             # check that post hooks are always called
             [m.reset_mock() for m in all_mocks]
             ip.run_cell("syntax error")
             assert pre_always.called
             assert pre_explicit.called
             assert post_always.called
             assert post_explicit.called
             info, = pre_explicit.call_args[0]
             result, = post_explicit.call_args[0]
             self.assertEqual(info, result.info)
         finally:
             # remove post-exec
             ip.events.unregister('pre_run_cell', pre_explicit)
             ip.events.unregister('pre_execute', pre_always)
             ip.events.unregister('post_run_cell', post_explicit)
             ip.events.unregister('post_execute', post_always)
     
     def test_silent_noadvance(self):
         """run_cell(silent=True) doesn't advance execution_count"""
         ec = ip.execution_count
         # silent should force store_history=False
         ip.run_cell("1", store_history=True, silent=True)
         
         self.assertEqual(ec, ip.execution_count)
         # double-check that non-silent exec did what we expected
         # silent to avoid
         ip.run_cell("1", store_history=True)
         self.assertEqual(ec+1, ip.execution_count)
     
     def test_silent_nodisplayhook(self):
         """run_cell(silent=True) doesn't trigger displayhook"""
         d = dict(called=False)
         
         trap = ip.display_trap
         save_hook = trap.hook
         
         def failing_hook(*args, **kwargs):
             d['called'] = True
         
         try:
             trap.hook = failing_hook
             res = ip.run_cell("1", silent=True)
             self.assertFalse(d['called'])
             self.assertIsNone(res.result)
             # double-check that non-silent exec did what we expected
             # silent to avoid
             ip.run_cell("1")
             self.assertTrue(d['called'])
         finally:
             trap.hook = save_hook
 
     def test_ofind_line_magic(self):
         from IPython.core.magic import register_line_magic
         
         @register_line_magic
         def lmagic(line):
             "A line magic"
 
         # Get info on line magic
         lfind = ip._ofind("lmagic")
         info = dict(
             found=True,
             isalias=False,
             ismagic=True,
             namespace="IPython internal",
             obj=lmagic,
             parent=None,
         )
         self.assertEqual(lfind, info)
         
     def test_ofind_cell_magic(self):
         from IPython.core.magic import register_cell_magic
         
         @register_cell_magic
         def cmagic(line, cell):
             "A cell magic"
 
         # Get info on cell magic
         find = ip._ofind("cmagic")
         info = dict(
             found=True,
             isalias=False,
             ismagic=True,
             namespace="IPython internal",
             obj=cmagic,
             parent=None,
         )
         self.assertEqual(find, info)
 
     def test_ofind_property_with_error(self):
         class A(object):
             @property
             def foo(self):
                 raise NotImplementedError()
         a = A()
 
         found = ip._ofind('a.foo', [('locals', locals())])
         info = dict(found=True, isalias=False, ismagic=False,
                     namespace='locals', obj=A.foo, parent=a)
         self.assertEqual(found, info)
 
     def test_ofind_multiple_attribute_lookups(self):
         class A(object):
             @property
             def foo(self):
                 raise NotImplementedError()
 
         a = A()
         a.a = A()
         a.a.a = A()
 
         found = ip._ofind('a.a.a.foo', [('locals', locals())])
         info = dict(found=True, isalias=False, ismagic=False,
                     namespace='locals', obj=A.foo, parent=a.a.a)
         self.assertEqual(found, info)
 
     def test_ofind_slotted_attributes(self):
         class A(object):
             __slots__ = ['foo']
             def __init__(self):
                 self.foo = 'bar'
 
         a = A()
         found = ip._ofind('a.foo', [('locals', locals())])
         info = dict(found=True, isalias=False, ismagic=False,
                     namespace='locals', obj=a.foo, parent=a)
         self.assertEqual(found, info)
 
         found = ip._ofind('a.bar', [('locals', locals())])
         info = dict(found=False, isalias=False, ismagic=False,
                     namespace=None, obj=None, parent=a)
         self.assertEqual(found, info)
 
     def test_ofind_prefers_property_to_instance_level_attribute(self):
         class A(object):
             @property
             def foo(self):
                 return 'bar'
         a = A()
         a.__dict__["foo"] = "baz"
         self.assertEqual(a.foo, "bar")
         found = ip._ofind("a.foo", [("locals", locals())])
         self.assertIs(found["obj"], A.foo)
 
     def test_custom_syntaxerror_exception(self):
         called = []
         def my_handler(shell, etype, value, tb, tb_offset=None):
             called.append(etype)
             shell.showtraceback((etype, value, tb), tb_offset=tb_offset)
 
         ip.set_custom_exc((SyntaxError,), my_handler)
         try:
             ip.run_cell("1f")
             # Check that this was called, and only once.
             self.assertEqual(called, [SyntaxError])
         finally:
             # Reset the custom exception hook
             ip.set_custom_exc((), None)
 
     def test_custom_exception(self):
         called = []
         def my_handler(shell, etype, value, tb, tb_offset=None):
             called.append(etype)
             shell.showtraceback((etype, value, tb), tb_offset=tb_offset)
         
         ip.set_custom_exc((ValueError,), my_handler)
         try:
             res = ip.run_cell("raise ValueError('test')")
             # Check that this was called, and only once.
             self.assertEqual(called, [ValueError])
             # Check that the error is on the result object
             self.assertIsInstance(res.error_in_exec, ValueError)
         finally:
             # Reset the custom exception hook
             ip.set_custom_exc((), None)
     
     @mock.patch("builtins.print")
     def test_showtraceback_with_surrogates(self, mocked_print):
         values = []
 
         def mock_print_func(value, sep=" ", end="\n", file=sys.stdout, flush=False):
             values.append(value)
             if value == chr(0xD8FF):
                 raise UnicodeEncodeError("utf-8", chr(0xD8FF), 0, 1, "")
 
         # mock builtins.print
         mocked_print.side_effect = mock_print_func
 
         # ip._showtraceback() is replaced in globalipapp.py.
         # Call original method to test.
         interactiveshell.InteractiveShell._showtraceback(ip, None, None, chr(0xD8FF))
 
         self.assertEqual(mocked_print.call_count, 2)
         self.assertEqual(values, [chr(0xD8FF), "\\ud8ff"])
 
     def test_mktempfile(self):
         filename = ip.mktempfile()
         # Check that we can open the file again on Windows
         with open(filename, 'w') as f:
             f.write('abc')
 
         filename = ip.mktempfile(data='blah')
         with open(filename, 'r') as f:
             self.assertEqual(f.read(), 'blah')
 
     def test_new_main_mod(self):
         # Smoketest to check that this accepts a unicode module name
         name = u'jiefmw'
         mod = ip.new_main_mod(u'%s.py' % name, name)
         self.assertEqual(mod.__name__, name)
 
     def test_get_exception_only(self):
         try:
             raise KeyboardInterrupt
         except KeyboardInterrupt:
             msg = ip.get_exception_only()
         self.assertEqual(msg, 'KeyboardInterrupt\n')
 
         try:
             raise DerivedInterrupt("foo")
         except KeyboardInterrupt:
             msg = ip.get_exception_only()
         self.assertEqual(msg, 'IPython.core.tests.test_interactiveshell.DerivedInterrupt: foo\n')
 
     def test_inspect_text(self):
         ip.run_cell('a = 5')
         text = ip.object_inspect_text('a')
         self.assertIsInstance(text, str)
 
     def test_last_execution_result(self):
         """ Check that last execution result gets set correctly (GH-10702) """
         result = ip.run_cell('a = 5; a')
         self.assertTrue(ip.last_execution_succeeded)
         self.assertEqual(ip.last_execution_result.result, 5)
 
         result = ip.run_cell('a = x_invalid_id_x')
         self.assertFalse(ip.last_execution_succeeded)
         self.assertFalse(ip.last_execution_result.success)
         self.assertIsInstance(ip.last_execution_result.error_in_exec, NameError)
 
     def test_reset_aliasing(self):
         """ Check that standard posix aliases work after %reset. """
         if os.name != 'posix':
             return
 
         ip.reset()
         for cmd in ('clear', 'more', 'less', 'man'):
             res = ip.run_cell('%' + cmd)
             self.assertEqual(res.success, True)
 
 
 class TestSafeExecfileNonAsciiPath(unittest.TestCase):
 
     @onlyif_unicode_paths
     def setUp(self):
         self.BASETESTDIR = tempfile.mkdtemp()
         self.TESTDIR = join(self.BASETESTDIR, u"åäö")
         os.mkdir(self.TESTDIR)
         with open(join(self.TESTDIR, u"åäötestscript.py"), "w") as sfile:
             sfile.write("pass\n")
         self.oldpath = os.getcwd()
         os.chdir(self.TESTDIR)
         self.fname = u"åäötestscript.py"
 
     def tearDown(self):
         os.chdir(self.oldpath)
         shutil.rmtree(self.BASETESTDIR)
 
     @onlyif_unicode_paths
     def test_1(self):
         """Test safe_execfile with non-ascii path
         """
         ip.safe_execfile(self.fname, {}, raise_exceptions=True)
 
 class ExitCodeChecks(tt.TempFileMixin):
 
     def setUp(self):
         self.system = ip.system_raw
 
     def test_exit_code_ok(self):
         self.system('exit 0')
         self.assertEqual(ip.user_ns['_exit_code'], 0)
 
     def test_exit_code_error(self):
         self.system('exit 1')
         self.assertEqual(ip.user_ns['_exit_code'], 1)
     
     @skipif(not hasattr(signal, 'SIGALRM'))
     def test_exit_code_signal(self):
         self.mktmp("import signal, time\n"
                    "signal.setitimer(signal.ITIMER_REAL, 0.1)\n"
                    "time.sleep(1)\n")
         self.system("%s %s" % (sys.executable, self.fname))
         self.assertEqual(ip.user_ns['_exit_code'], -signal.SIGALRM)
     
     @onlyif_cmds_exist("csh")
     def test_exit_code_signal_csh(self):
         SHELL = os.environ.get('SHELL', None)
         os.environ['SHELL'] = find_cmd("csh")
         try:
             self.test_exit_code_signal()
         finally:
             if SHELL is not None:
                 os.environ['SHELL'] = SHELL
             else:
                 del os.environ['SHELL']
 
 
 class TestSystemRaw(ExitCodeChecks):
 
     def setUp(self):
         super().setUp()
         self.system = ip.system_raw
 
     @onlyif_unicode_paths
     def test_1(self):
         """Test system_raw with non-ascii cmd
         """
         cmd = u'''python -c "'åäö'"   '''
         ip.system_raw(cmd)
 
     @mock.patch('subprocess.call', side_effect=KeyboardInterrupt)
     @mock.patch('os.system', side_effect=KeyboardInterrupt)
     def test_control_c(self, *mocks):
         try:
             self.system("sleep 1 # wont happen")
         except KeyboardInterrupt:
             self.fail(
                 "system call should intercept "
                 "keyboard interrupt from subprocess.call"
             )
         self.assertEqual(ip.user_ns["_exit_code"], -signal.SIGINT)
 
     def test_magic_warnings(self):
         for magic_cmd in ("ls", "pip", "conda", "cd"):
             with self.assertWarnsRegex(Warning, "You executed the system command"):
                 ip.system_raw(magic_cmd)
 
 # TODO: Exit codes are currently ignored on Windows.
 class TestSystemPipedExitCode(ExitCodeChecks):
 
     def setUp(self):
         super().setUp()
         self.system = ip.system_piped
 
     @skip_win32
     def test_exit_code_ok(self):
         ExitCodeChecks.test_exit_code_ok(self)
 
     @skip_win32
     def test_exit_code_error(self):
         ExitCodeChecks.test_exit_code_error(self)
 
     @skip_win32
     def test_exit_code_signal(self):
         ExitCodeChecks.test_exit_code_signal(self)
 
 class TestModules(tt.TempFileMixin):
     def test_extraneous_loads(self):
         """Test we're not loading modules on startup that we shouldn't.
         """
         self.mktmp("import sys\n"
                    "print('numpy' in sys.modules)\n"
                    "print('ipyparallel' in sys.modules)\n"
                    "print('ipykernel' in sys.modules)\n"
                    )
         out = "False\nFalse\nFalse\n"
         tt.ipexec_validate(self.fname, out)
 
 class Negator(ast.NodeTransformer):
     """Negates all number literals in an AST."""
 
     # for python 3.7 and earlier
     def visit_Num(self, node):
         node.n = -node.n
         return node
 
     # for python 3.8+
     def visit_Constant(self, node):
         if isinstance(node.value, int):
             return self.visit_Num(node)
         return node
 
 class TestAstTransform(unittest.TestCase):
     def setUp(self):
         self.negator = Negator()
         ip.ast_transformers.append(self.negator)
     
     def tearDown(self):
         ip.ast_transformers.remove(self.negator)
     
     def test_run_cell(self):
         with tt.AssertPrints('-34'):
             ip.run_cell('print (12 + 22)')
         
         # A named reference to a number shouldn't be transformed.
         ip.user_ns['n'] = 55
         with tt.AssertNotPrints('-55'):
             ip.run_cell('print (n)')
     
     def test_timeit(self):
         called = set()
         def f(x):
             called.add(x)
         ip.push({'f':f})
         
         with tt.AssertPrints("std. dev. of"):
             ip.run_line_magic("timeit", "-n1 f(1)")
         self.assertEqual(called, {-1})
         called.clear()
 
         with tt.AssertPrints("std. dev. of"):
             ip.run_cell_magic("timeit", "-n1 f(2)", "f(3)")
         self.assertEqual(called, {-2, -3})
     
     def test_time(self):
         called = []
         def f(x):
             called.append(x)
         ip.push({'f':f})
         
         # Test with an expression
         with tt.AssertPrints("Wall time: "):
             ip.run_line_magic("time", "f(5+9)")
         self.assertEqual(called, [-14])
         called[:] = []
         
         # Test with a statement (different code path)
         with tt.AssertPrints("Wall time: "):
             ip.run_line_magic("time", "a = f(-3 + -2)")
         self.assertEqual(called, [5])
     
     def test_macro(self):
         ip.push({'a':10})
         # The AST transformation makes this do a+=-1
         ip.define_macro("amacro", "a+=1\nprint(a)")
         
         with tt.AssertPrints("9"):
             ip.run_cell("amacro")
         with tt.AssertPrints("8"):
             ip.run_cell("amacro")
 
 class TestMiscTransform(unittest.TestCase):
 
 
     def test_transform_only_once(self):
         cleanup = 0
         line_t = 0
         def count_cleanup(lines):
             nonlocal cleanup
             cleanup += 1
             return lines
 
         def count_line_t(lines):
             nonlocal line_t
             line_t += 1
             return lines
 
         ip.input_transformer_manager.cleanup_transforms.append(count_cleanup)
         ip.input_transformer_manager.line_transforms.append(count_line_t)
 
         ip.run_cell('1')
 
         assert cleanup == 1
         assert line_t == 1
 
 class IntegerWrapper(ast.NodeTransformer):
     """Wraps all integers in a call to Integer()"""
 
     # for Python 3.7 and earlier
 
     # for Python 3.7 and earlier
     def visit_Num(self, node):
         if isinstance(node.n, int):
             return ast.Call(func=ast.Name(id='Integer', ctx=ast.Load()),
                             args=[node], keywords=[])
         return node
 
     # For Python 3.8+
     def visit_Constant(self, node):
         if isinstance(node.value, int):
             return self.visit_Num(node)
         return node
 
 
 class TestAstTransform2(unittest.TestCase):
     def setUp(self):
         self.intwrapper = IntegerWrapper()
         ip.ast_transformers.append(self.intwrapper)
         
         self.calls = []
         def Integer(*args):
             self.calls.append(args)
             return args
         ip.push({"Integer": Integer})
     
     def tearDown(self):
         ip.ast_transformers.remove(self.intwrapper)
         del ip.user_ns['Integer']
     
     def test_run_cell(self):
         ip.run_cell("n = 2")
         self.assertEqual(self.calls, [(2,)])
         
         # This shouldn't throw an error
         ip.run_cell("o = 2.0")
         self.assertEqual(ip.user_ns['o'], 2.0)
     
     def test_timeit(self):
         called = set()
         def f(x):
             called.add(x)
         ip.push({'f':f})
 
         with tt.AssertPrints("std. dev. of"):
             ip.run_line_magic("timeit", "-n1 f(1)")
         self.assertEqual(called, {(1,)})
         called.clear()
 
         with tt.AssertPrints("std. dev. of"):
             ip.run_cell_magic("timeit", "-n1 f(2)", "f(3)")
         self.assertEqual(called, {(2,), (3,)})
 
 class ErrorTransformer(ast.NodeTransformer):
     """Throws an error when it sees a number."""
 
     # for Python 3.7 and earlier
     def visit_Num(self, node):
         raise ValueError("test")
 
     # for Python 3.8+
     def visit_Constant(self, node):
         if isinstance(node.value, int):
             return self.visit_Num(node)
         return node
 
 
 class TestAstTransformError(unittest.TestCase):
     def test_unregistering(self):
         err_transformer = ErrorTransformer()
         ip.ast_transformers.append(err_transformer)
         
         with self.assertWarnsRegex(UserWarning, "It will be unregistered"):
             ip.run_cell("1 + 2")
         
         # This should have been removed.
         self.assertNotIn(err_transformer, ip.ast_transformers)
 
 
 class StringRejector(ast.NodeTransformer):
     """Throws an InputRejected when it sees a string literal.
 
     Used to verify that NodeTransformers can signal that a piece of code should
     not be executed by throwing an InputRejected.
     """
     
     #for python 3.7 and earlier
     def visit_Str(self, node):
         raise InputRejected("test")
 
     # 3.8 only
     def visit_Constant(self, node):
         if isinstance(node.value, str):
             raise InputRejected("test")
         return node
 
 
 class TestAstTransformInputRejection(unittest.TestCase):
 
     def setUp(self):
         self.transformer = StringRejector()
         ip.ast_transformers.append(self.transformer)
 
     def tearDown(self):
         ip.ast_transformers.remove(self.transformer)
 
     def test_input_rejection(self):
         """Check that NodeTransformers can reject input."""
 
         expect_exception_tb = tt.AssertPrints("InputRejected: test")
         expect_no_cell_output = tt.AssertNotPrints("'unsafe'", suppress=False)
 
         # Run the same check twice to verify that the transformer is not
         # disabled after raising.
         with expect_exception_tb, expect_no_cell_output:
             ip.run_cell("'unsafe'")
 
         with expect_exception_tb, expect_no_cell_output:
             res = ip.run_cell("'unsafe'")
 
         self.assertIsInstance(res.error_before_exec, InputRejected)
 
 def test__IPYTHON__():
     # This shouldn't raise a NameError, that's all
     __IPYTHON__
 
 
 class DummyRepr(object):
     def __repr__(self):
         return "DummyRepr"
     
     def _repr_html_(self):
         return "<b>dummy</b>"
     
     def _repr_javascript_(self):
         return "console.log('hi');", {'key': 'value'}
     
 
 def test_user_variables():
     # enable all formatters
     ip.display_formatter.active_types = ip.display_formatter.format_types
     
     ip.user_ns['dummy'] = d = DummyRepr()
     keys = {'dummy', 'doesnotexist'}
     r = ip.user_expressions({ key:key for key in keys})
 
     assert keys == set(r.keys())
     dummy = r["dummy"]
     assert {"status", "data", "metadata"} == set(dummy.keys())
     assert dummy["status"] == "ok"
     data = dummy["data"]
     metadata = dummy["metadata"]
     assert data.get("text/html") == d._repr_html_()
     js, jsmd = d._repr_javascript_()
     assert data.get("application/javascript") == js
     assert metadata.get("application/javascript") == jsmd
 
     dne = r["doesnotexist"]
     assert dne["status"] == "error"
     assert dne["ename"] == "NameError"
 
     # back to text only
     ip.display_formatter.active_types = ['text/plain']
     
 def test_user_expression():
     # enable all formatters
     ip.display_formatter.active_types = ip.display_formatter.format_types
     query = {
         'a' : '1 + 2',
         'b' : '1/0',
     }
     r = ip.user_expressions(query)
     import pprint
     pprint.pprint(r)
     assert set(r.keys()) == set(query.keys())
     a = r["a"]
     assert {"status", "data", "metadata"} == set(a.keys())
     assert a["status"] == "ok"
     data = a["data"]
     metadata = a["metadata"]
     assert data.get("text/plain") == "3"
 
     b = r["b"]
     assert b["status"] == "error"
     assert b["ename"] == "ZeroDivisionError"
 
     # back to text only
     ip.display_formatter.active_types = ['text/plain']
 
 
 class TestSyntaxErrorTransformer(unittest.TestCase):
     """Check that SyntaxError raised by an input transformer is handled by run_cell()"""
 
     @staticmethod
     def transformer(lines):
         for line in lines:
             pos = line.find('syntaxerror')
             if pos >= 0:
                 e = SyntaxError('input contains "syntaxerror"')
                 e.text = line
                 e.offset = pos + 1
                 raise e
         return lines
 
     def setUp(self):
         ip.input_transformers_post.append(self.transformer)
 
     def tearDown(self):
         ip.input_transformers_post.remove(self.transformer)
 
     def test_syntaxerror_input_transformer(self):
         with tt.AssertPrints('1234'):
             ip.run_cell('1234')
         with tt.AssertPrints('SyntaxError: invalid syntax'):
             ip.run_cell('1 2 3')   # plain python syntax error
         with tt.AssertPrints('SyntaxError: input contains "syntaxerror"'):
             ip.run_cell('2345  # syntaxerror')  # input transformer syntax error
         with tt.AssertPrints('3456'):
             ip.run_cell('3456')
 
 
 class TestWarningSuppression(unittest.TestCase):
     def test_warning_suppression(self):
         ip.run_cell("import warnings")
         try:
             with self.assertWarnsRegex(UserWarning, "asdf"):
                 ip.run_cell("warnings.warn('asdf')")
             # Here's the real test -- if we run that again, we should get the
             # warning again. Traditionally, each warning was only issued once per
             # IPython session (approximately), even if the user typed in new and
             # different code that should have also triggered the warning, leading
             # to much confusion.
             with self.assertWarnsRegex(UserWarning, "asdf"):
                 ip.run_cell("warnings.warn('asdf')")
         finally:
             ip.run_cell("del warnings")
 
 
     def test_deprecation_warning(self):
         ip.run_cell("""
 import warnings
 def wrn():
     warnings.warn(
         "I AM  A WARNING",
         DeprecationWarning
     )
         """)
         try:
             with self.assertWarnsRegex(DeprecationWarning, "I AM  A WARNING"):
                 ip.run_cell("wrn()")
         finally:
             ip.run_cell("del warnings")
             ip.run_cell("del wrn")
 
 
 class TestImportNoDeprecate(tt.TempFileMixin):
 
     def setUp(self):
         """Make a valid python temp file."""
         self.mktmp("""
 import warnings
 def wrn():
     warnings.warn(
         "I AM  A WARNING",
         DeprecationWarning
     )
 """)
         super().setUp()
 
     def test_no_dep(self):
         """
         No deprecation warning should be raised from imported functions
         """
         ip.run_cell("from {} import wrn".format(self.fname))
 
         with tt.AssertNotPrints("I AM  A WARNING"):
             ip.run_cell("wrn()")
         ip.run_cell("del wrn")
 
 
 def test_custom_exc_count():
     hook = mock.Mock(return_value=None)
     ip.set_custom_exc((SyntaxError,), hook)
     before = ip.execution_count
     ip.run_cell("def foo()", store_history=True)
     # restore default excepthook
     ip.set_custom_exc((), None)
     assert hook.call_count == 1
     assert ip.execution_count == before + 1
 
 
 def test_run_cell_async():
-    loop = asyncio.get_event_loop_policy().get_event_loop()
     ip.run_cell("import asyncio")
     coro = ip.run_cell_async("await asyncio.sleep(0.01)\n5")
     assert asyncio.iscoroutine(coro)
+    loop = asyncio.new_event_loop()
     result = loop.run_until_complete(coro)
     assert isinstance(result, interactiveshell.ExecutionResult)
     assert result.result == 5
 
 
 def test_run_cell_await():
     ip.run_cell("import asyncio")
     result = ip.run_cell("await asyncio.sleep(0.01); 10")
     assert ip.user_ns["_"] == 10
 
 
 def test_run_cell_asyncio_run():
     ip.run_cell("import asyncio")
     result = ip.run_cell("await asyncio.sleep(0.01); 1")
     assert ip.user_ns["_"] == 1
     result = ip.run_cell("asyncio.run(asyncio.sleep(0.01)); 2")
     assert ip.user_ns["_"] == 2
     result = ip.run_cell("await asyncio.sleep(0.01); 3")
     assert ip.user_ns["_"] == 3
 
 
 def test_should_run_async():
     assert not ip.should_run_async("a = 5")
     assert ip.should_run_async("await x")
     assert ip.should_run_async("import asyncio; await asyncio.sleep(1)")
 
 
 def test_set_custom_completer():
     num_completers = len(ip.Completer.matchers)
 
     def foo(*args, **kwargs):
         return "I'm a completer!"
 
     ip.set_custom_completer(foo, 0)
 
     # check that we've really added a new completer
     assert len(ip.Completer.matchers) == num_completers + 1
 
     # check that the first completer is the function we defined
     assert ip.Completer.matchers[0]() == "I'm a completer!"
 
     # clean up
     ip.Completer.custom_matchers.pop()

IPython/core/tests/test_magic.py

@@ -1,1358 +1,1375 b''
 # -*- coding: utf-8 -*-
 """Tests for various magic functions."""
 
 import asyncio
 import gc
 import io
 import os
 import re
 import shlex
 import sys
 import warnings
 from importlib import invalidate_caches
 from io import StringIO
 from pathlib import Path
 from textwrap import dedent
 from unittest import TestCase, mock
 
 import pytest
 
 from IPython import get_ipython
 from IPython.core import magic
 from IPython.core.error import UsageError
 from IPython.core.magic import (
     Magics,
     cell_magic,
     line_magic,
     magics_class,
     register_cell_magic,
     register_line_magic,
 )
 from IPython.core.magics import code, execution, logging, osm, script
 from IPython.testing import decorators as dec
 from IPython.testing import tools as tt
 from IPython.utils.io import capture_output
 from IPython.utils.process import find_cmd
 from IPython.utils.tempdir import TemporaryDirectory, TemporaryWorkingDirectory
 
 from .test_debugger import PdbTestInput
 
 
 @magic.magics_class
 class DummyMagics(magic.Magics): pass
 
 def test_extract_code_ranges():
     instr = "1 3 5-6 7-9 10:15 17: :10 10- -13 :"
     expected = [
         (0, 1),
         (2, 3),
         (4, 6),
         (6, 9),
         (9, 14),
         (16, None),
         (None, 9),
         (9, None),
         (None, 13),
         (None, None),
     ]
     actual = list(code.extract_code_ranges(instr))
     assert actual == expected
 
 def test_extract_symbols():
     source = """import foo\na = 10\ndef b():\n    return 42\n\n\nclass A: pass\n\n\n"""
     symbols_args = ["a", "b", "A", "A,b", "A,a", "z"]
     expected = [([], ['a']),
                 (["def b():\n    return 42\n"], []),
                 (["class A: pass\n"], []),
                 (["class A: pass\n", "def b():\n    return 42\n"], []),
                 (["class A: pass\n"], ['a']),
                 ([], ['z'])]
     for symbols, exp in zip(symbols_args, expected):
         assert code.extract_symbols(source, symbols) == exp
 
 
 def test_extract_symbols_raises_exception_with_non_python_code():
     source = ("=begin A Ruby program :)=end\n"
               "def hello\n"
               "puts 'Hello world'\n"
               "end")
     with pytest.raises(SyntaxError):
         code.extract_symbols(source, "hello")
 
 
 def test_magic_not_found():
     # magic not found raises UsageError
     with pytest.raises(UsageError):
         _ip.magic('doesntexist')
 
     # ensure result isn't success when a magic isn't found
     result = _ip.run_cell('%doesntexist')
     assert isinstance(result.error_in_exec, UsageError)
 
 
 def test_cell_magic_not_found():
     # magic not found raises UsageError
     with pytest.raises(UsageError):
         _ip.run_cell_magic('doesntexist', 'line', 'cell')
 
     # ensure result isn't success when a magic isn't found
     result = _ip.run_cell('%%doesntexist')
     assert isinstance(result.error_in_exec, UsageError)
 
 
 def test_magic_error_status():
     def fail(shell):
         1/0
     _ip.register_magic_function(fail)
     result = _ip.run_cell('%fail')
     assert isinstance(result.error_in_exec, ZeroDivisionError)
 
 
 def test_config():
     """ test that config magic does not raise
     can happen if Configurable init is moved too early into
     Magics.__init__ as then a Config object will be registered as a
     magic.
     """
     ## should not raise.
     _ip.magic('config')
 
 def test_config_available_configs():
     """ test that config magic prints available configs in unique and
     sorted order. """
     with capture_output() as captured:
         _ip.magic('config')
 
     stdout = captured.stdout
     config_classes = stdout.strip().split('\n')[1:]
     assert config_classes == sorted(set(config_classes))
 
 def test_config_print_class():
     """ test that config with a classname prints the class's options. """
     with capture_output() as captured:
         _ip.magic('config TerminalInteractiveShell')
 
     stdout = captured.stdout
     assert re.match(
         "TerminalInteractiveShell.* options", stdout.splitlines()[0]
     ), f"{stdout}\n\n1st line of stdout not like 'TerminalInteractiveShell.* options'"
 
 
 def test_rehashx():
     # clear up everything
     _ip.alias_manager.clear_aliases()
     del _ip.db['syscmdlist']
 
     _ip.magic('rehashx')
     # Practically ALL ipython development systems will have more than 10 aliases
 
     assert len(_ip.alias_manager.aliases) > 10
     for name, cmd in _ip.alias_manager.aliases:
         # we must strip dots from alias names
         assert "." not in name
 
     # rehashx must fill up syscmdlist
     scoms = _ip.db['syscmdlist']
     assert len(scoms) > 10
 
 
 def test_magic_parse_options():
     """Test that we don't mangle paths when parsing magic options."""
     ip = get_ipython()
     path = 'c:\\x'
     m = DummyMagics(ip)
     opts = m.parse_options('-f %s' % path,'f:')[0]
     # argv splitting is os-dependent
     if os.name == 'posix':
         expected = 'c:x'
     else:
         expected = path
     assert opts["f"] == expected
 
 
 def test_magic_parse_long_options():
     """Magic.parse_options can handle --foo=bar long options"""
     ip = get_ipython()
     m = DummyMagics(ip)
     opts, _ = m.parse_options("--foo --bar=bubble", "a", "foo", "bar=")
     assert "foo" in opts
     assert "bar" in opts
     assert opts["bar"] == "bubble"
 
 
 def doctest_hist_f():
     """Test %hist -f with temporary filename.
 
     In [9]: import tempfile
 
     In [10]: tfile = tempfile.mktemp('.py','tmp-ipython-')
 
     In [11]: %hist -nl -f $tfile 3
 
     In [13]: import os; os.unlink(tfile)
     """
 
 
 def doctest_hist_op():
     """Test %hist -op
 
     In [1]: class b(float):
        ...:     pass
        ...:
 
     In [2]: class s(object):
        ...:     def __str__(self):
        ...:         return 's'
        ...:
 
     In [3]:
 
     In [4]: class r(b):
        ...:     def __repr__(self):
        ...:         return 'r'
        ...:
 
     In [5]: class sr(s,r): pass
        ...:
 
     In [6]:
 
     In [7]: bb=b()
 
     In [8]: ss=s()
 
     In [9]: rr=r()
 
     In [10]: ssrr=sr()
 
     In [11]: 4.5
     Out[11]: 4.5
 
     In [12]: str(ss)
     Out[12]: 's'
 
     In [13]:
 
     In [14]: %hist -op
     >>> class b:
     ...     pass
     ...
     >>> class s(b):
     ...     def __str__(self):
     ...         return 's'
     ...
     >>>
     >>> class r(b):
     ...     def __repr__(self):
     ...         return 'r'
     ...
     >>> class sr(s,r): pass
     >>>
     >>> bb=b()
     >>> ss=s()
     >>> rr=r()
     >>> ssrr=sr()
     >>> 4.5
     4.5
     >>> str(ss)
     's'
     >>>
     """
 
 def test_hist_pof():
     ip = get_ipython()
     ip.run_cell("1+2", store_history=True)
     #raise Exception(ip.history_manager.session_number)
     #raise Exception(list(ip.history_manager._get_range_session()))
     with TemporaryDirectory() as td:
         tf = os.path.join(td, 'hist.py')
         ip.run_line_magic('history', '-pof %s' % tf)
         assert os.path.isfile(tf)
 
 
 def test_macro():
     ip = get_ipython()
     ip.history_manager.reset()   # Clear any existing history.
     cmds = ["a=1", "def b():\n  return a**2", "print(a,b())"]
     for i, cmd in enumerate(cmds, start=1):
         ip.history_manager.store_inputs(i, cmd)
     ip.magic("macro test 1-3")
     assert ip.user_ns["test"].value == "\n".join(cmds) + "\n"
 
     # List macros
     assert "test" in ip.magic("macro")
 
 
 def test_macro_run():
     """Test that we can run a multi-line macro successfully."""
     ip = get_ipython()
     ip.history_manager.reset()
     cmds = ["a=10", "a+=1", "print(a)", "%macro test 2-3"]
     for cmd in cmds:
         ip.run_cell(cmd, store_history=True)
     assert ip.user_ns["test"].value == "a+=1\nprint(a)\n"
     with tt.AssertPrints("12"):
         ip.run_cell("test")
     with tt.AssertPrints("13"):
         ip.run_cell("test")
 
 
 def test_magic_magic():
     """Test %magic"""
     ip = get_ipython()
     with capture_output() as captured:
         ip.magic("magic")
 
     stdout = captured.stdout
     assert "%magic" in stdout
     assert "IPython" in stdout
     assert "Available" in stdout
 
 
 @dec.skipif_not_numpy
 def test_numpy_reset_array_undec():
     "Test '%reset array' functionality"
     _ip.ex("import numpy as np")
     _ip.ex("a = np.empty(2)")
     assert "a" in _ip.user_ns
     _ip.magic("reset -f array")
     assert "a" not in _ip.user_ns
 
 
 def test_reset_out():
     "Test '%reset out' magic"
     _ip.run_cell("parrot = 'dead'", store_history=True)
     # test '%reset -f out', make an Out prompt
     _ip.run_cell("parrot", store_history=True)
     assert "dead" in [_ip.user_ns[x] for x in ("_", "__", "___")]
     _ip.magic("reset -f out")
     assert "dead" not in [_ip.user_ns[x] for x in ("_", "__", "___")]
     assert len(_ip.user_ns["Out"]) == 0
 
 
 def test_reset_in():
     "Test '%reset in' magic"
     # test '%reset -f in'
     _ip.run_cell("parrot", store_history=True)
     assert "parrot" in [_ip.user_ns[x] for x in ("_i", "_ii", "_iii")]
     _ip.magic("%reset -f in")
     assert "parrot" not in [_ip.user_ns[x] for x in ("_i", "_ii", "_iii")]
     assert len(set(_ip.user_ns["In"])) == 1
 
 
 def test_reset_dhist():
     "Test '%reset dhist' magic"
     _ip.run_cell("tmp = [d for d in _dh]")  # copy before clearing
     _ip.magic("cd " + os.path.dirname(pytest.__file__))
     _ip.magic("cd -")
     assert len(_ip.user_ns["_dh"]) > 0
     _ip.magic("reset -f dhist")
     assert len(_ip.user_ns["_dh"]) == 0
     _ip.run_cell("_dh = [d for d in tmp]")  # restore
 
 
 def test_reset_in_length():
     "Test that '%reset in' preserves In[] length"
     _ip.run_cell("print 'foo'")
     _ip.run_cell("reset -f in")
     assert len(_ip.user_ns["In"]) == _ip.displayhook.prompt_count + 1
 
 
 class TestResetErrors(TestCase):
 
     def test_reset_redefine(self):
 
         @magics_class
         class KernelMagics(Magics):
               @line_magic
               def less(self, shell): pass
 
         _ip.register_magics(KernelMagics)
 
         with self.assertLogs() as cm:
             # hack, we want to just capture logs, but assertLogs fails if not
             # logs get produce.
             # so log one things we ignore.
             import logging as log_mod
             log = log_mod.getLogger()
             log.info('Nothing')
             # end hack.
             _ip.run_cell("reset -f")
 
         assert len(cm.output) == 1
         for out in cm.output:
             assert "Invalid alias" not in out
 
 def test_tb_syntaxerror():
     """test %tb after a SyntaxError"""
     ip = get_ipython()
     ip.run_cell("for")
 
     # trap and validate stdout
     save_stdout = sys.stdout
     try:
         sys.stdout = StringIO()
         ip.run_cell("%tb")
         out = sys.stdout.getvalue()
     finally:
         sys.stdout = save_stdout
     # trim output, and only check the last line
     last_line = out.rstrip().splitlines()[-1].strip()
     assert last_line == "SyntaxError: invalid syntax"
 
 
 def test_time():
     ip = get_ipython()
 
     with tt.AssertPrints("Wall time: "):
         ip.run_cell("%time None")
 
     ip.run_cell("def f(kmjy):\n"
                 "    %time print (2*kmjy)")
 
     with tt.AssertPrints("Wall time: "):
         with tt.AssertPrints("hihi", suppress=False):
             ip.run_cell("f('hi')")
 
 def test_time_last_not_expression():
     ip.run_cell("%%time\n"
                 "var_1 = 1\n"
                 "var_2 = 2\n")
     assert ip.user_ns['var_1'] == 1
     del ip.user_ns['var_1']
     assert ip.user_ns['var_2'] == 2
     del ip.user_ns['var_2']
 
 
 @dec.skip_win32
 def test_time2():
     ip = get_ipython()
 
     with tt.AssertPrints("CPU times: user "):
         ip.run_cell("%time None")
 
 def test_time3():
     """Erroneous magic function calls, issue gh-3334"""
     ip = get_ipython()
     ip.user_ns.pop('run', None)
 
     with tt.AssertNotPrints("not found", channel='stderr'):
         ip.run_cell("%%time\n"
                     "run = 0\n"
                     "run += 1")
 
 def test_multiline_time():
     """Make sure last statement from time return a value."""
     ip = get_ipython()
     ip.user_ns.pop('run', None)
 
     ip.run_cell(dedent("""\
         %%time
         a = "ho"
         b = "hey"
         a+b
         """
         )
     )
     assert ip.user_ns_hidden["_"] == "hohey"
 
 
 def test_time_local_ns():
     """
     Test that local_ns is actually global_ns when running a cell magic
     """
     ip = get_ipython()
     ip.run_cell("%%time\n" "myvar = 1")
     assert ip.user_ns["myvar"] == 1
     del ip.user_ns["myvar"]
 
 
 def test_doctest_mode():
     "Toggle doctest_mode twice, it should be a no-op and run without error"
     _ip.magic('doctest_mode')
     _ip.magic('doctest_mode')
 
 
 def test_parse_options():
     """Tests for basic options parsing in magics."""
     # These are only the most minimal of tests, more should be added later.  At
     # the very least we check that basic text/unicode calls work OK.
     m = DummyMagics(_ip)
     assert m.parse_options("foo", "")[1] == "foo"
     assert m.parse_options("foo", "")[1] == "foo"
 
 
 def test_parse_options_preserve_non_option_string():
     """Test to assert preservation of non-option part of magic-block, while parsing magic options."""
     m = DummyMagics(_ip)
     opts, stmt = m.parse_options(
         " -n1  -r 13 _ = 314 + foo", "n:r:", preserve_non_opts=True
     )
     assert opts == {"n": "1", "r": "13"}
     assert stmt == "_ = 314 + foo"
 
 
 def test_run_magic_preserve_code_block():
     """Test to assert preservation of non-option part of magic-block, while running magic."""
     _ip.user_ns["spaces"] = []
     _ip.magic("timeit -n1 -r1 spaces.append([s.count(' ') for s in ['document']])")
     assert _ip.user_ns["spaces"] == [[0]]
 
 
 def test_dirops():
     """Test various directory handling operations."""
     # curpath = lambda :os.path.splitdrive(os.getcwd())[1].replace('\\','/')
     curpath = os.getcwd
     startdir = os.getcwd()
     ipdir = os.path.realpath(_ip.ipython_dir)
     try:
         _ip.magic('cd "%s"' % ipdir)
         assert curpath() == ipdir
         _ip.magic('cd -')
         assert curpath() == startdir
         _ip.magic('pushd "%s"' % ipdir)
         assert curpath() == ipdir
         _ip.magic('popd')
         assert curpath() == startdir
     finally:
         os.chdir(startdir)
 
 
 def test_cd_force_quiet():
     """Test OSMagics.cd_force_quiet option"""
     _ip.config.OSMagics.cd_force_quiet = True
     osmagics = osm.OSMagics(shell=_ip)
 
     startdir = os.getcwd()
     ipdir = os.path.realpath(_ip.ipython_dir)
 
     try:
         with tt.AssertNotPrints(ipdir):
             osmagics.cd('"%s"' % ipdir)
         with tt.AssertNotPrints(startdir):
             osmagics.cd('-')
     finally:
         os.chdir(startdir)
 
 
 def test_xmode():
     # Calling xmode three times should be a no-op
     xmode = _ip.InteractiveTB.mode
     for i in range(4):
         _ip.magic("xmode")
     assert _ip.InteractiveTB.mode == xmode
 
 def test_reset_hard():
     monitor = []
     class A(object):
         def __del__(self):
             monitor.append(1)
         def __repr__(self):
             return "<A instance>"
 
     _ip.user_ns["a"] = A()
     _ip.run_cell("a")
 
     assert monitor == []
     _ip.magic("reset -f")
     assert monitor == [1]
 
 class TestXdel(tt.TempFileMixin):
     def test_xdel(self):
         """Test that references from %run are cleared by xdel."""
         src = ("class A(object):\n"
                "    monitor = []\n"
                "    def __del__(self):\n"
                "        self.monitor.append(1)\n"
                "a = A()\n")
         self.mktmp(src)
         # %run creates some hidden references...
         _ip.magic("run %s" % self.fname)
         # ... as does the displayhook.
         _ip.run_cell("a")
 
         monitor = _ip.user_ns["A"].monitor
         assert monitor == []
 
         _ip.magic("xdel a")
 
         # Check that a's __del__ method has been called.
         gc.collect(0)
         assert monitor == [1]
 
 def doctest_who():
     """doctest for %who
 
     In [1]: %reset -sf
 
     In [2]: alpha = 123
 
     In [3]: beta = 'beta'
 
     In [4]: %who int
     alpha
 
     In [5]: %who str
     beta
 
     In [6]: %whos
     Variable   Type    Data/Info
     ----------------------------
     alpha      int     123
     beta       str     beta
 
     In [7]: %who_ls
     Out[7]: ['alpha', 'beta']
     """
 
 def test_whos():
     """Check that whos is protected against objects where repr() fails."""
     class A(object):
         def __repr__(self):
             raise Exception()
     _ip.user_ns['a'] = A()
     _ip.magic("whos")
 
 def doctest_precision():
     """doctest for %precision
 
     In [1]: f = get_ipython().display_formatter.formatters['text/plain']
 
     In [2]: %precision 5
     Out[2]: '%.5f'
 
     In [3]: f.float_format
     Out[3]: '%.5f'
 
     In [4]: %precision %e
     Out[4]: '%e'
 
     In [5]: f(3.1415927)
     Out[5]: '3.141593e+00'
     """
 
 def test_debug_magic():
     """Test debugging a small code with %debug
 
     In [1]: with PdbTestInput(['c']):
        ...:     %debug print("a b") #doctest: +ELLIPSIS
        ...:
     ...
     ipdb> c
     a b
     In [2]:
     """
 
 def test_psearch():
     with tt.AssertPrints("dict.fromkeys"):
         _ip.run_cell("dict.fr*?")
     with tt.AssertPrints("π.is_integer"):
         _ip.run_cell("π = 3.14;\nπ.is_integ*?")
 
 def test_timeit_shlex():
     """test shlex issues with timeit (#1109)"""
     _ip.ex("def f(*a,**kw): pass")
     _ip.magic('timeit -n1 "this is a bug".count(" ")')
     _ip.magic('timeit -r1 -n1 f(" ", 1)')
     _ip.magic('timeit -r1 -n1 f(" ", 1, " ", 2, " ")')
     _ip.magic('timeit -r1 -n1 ("a " + "b")')
     _ip.magic('timeit -r1 -n1 f("a " + "b")')
     _ip.magic('timeit -r1 -n1 f("a " + "b ")')
 
 
 def test_timeit_special_syntax():
     "Test %%timeit with IPython special syntax"
     @register_line_magic
     def lmagic(line):
         ip = get_ipython()
         ip.user_ns['lmagic_out'] = line
 
     # line mode test
     _ip.run_line_magic("timeit", "-n1 -r1 %lmagic my line")
     assert _ip.user_ns["lmagic_out"] == "my line"
     # cell mode test
     _ip.run_cell_magic("timeit", "-n1 -r1", "%lmagic my line2")
     assert _ip.user_ns["lmagic_out"] == "my line2"
 
 
 def test_timeit_return():
     """
     test whether timeit -o return object
     """
 
     res = _ip.run_line_magic('timeit','-n10 -r10 -o 1')
     assert(res is not None)
 
 def test_timeit_quiet():
     """
     test quiet option of timeit magic
     """
     with tt.AssertNotPrints("loops"):
         _ip.run_cell("%timeit -n1 -r1 -q 1")
 
 def test_timeit_return_quiet():
     with tt.AssertNotPrints("loops"):
         res = _ip.run_line_magic('timeit', '-n1 -r1 -q -o 1')
     assert (res is not None)
 
 def test_timeit_invalid_return():
     with pytest.raises(SyntaxError):
         _ip.run_line_magic('timeit', 'return')
 
 @dec.skipif(execution.profile is None)
 def test_prun_special_syntax():
     "Test %%prun with IPython special syntax"
     @register_line_magic
     def lmagic(line):
         ip = get_ipython()
         ip.user_ns['lmagic_out'] = line
 
     # line mode test
     _ip.run_line_magic("prun", "-q %lmagic my line")
     assert _ip.user_ns["lmagic_out"] == "my line"
     # cell mode test
     _ip.run_cell_magic("prun", "-q", "%lmagic my line2")
     assert _ip.user_ns["lmagic_out"] == "my line2"
 
 
 @dec.skipif(execution.profile is None)
 def test_prun_quotes():
     "Test that prun does not clobber string escapes (GH #1302)"
     _ip.magic(r"prun -q x = '\t'")
     assert _ip.user_ns["x"] == "\t"
 
 
 def test_extension():
     # Debugging information for failures of this test
     print('sys.path:')
     for p in sys.path:
         print(' ', p)
     print('CWD', os.getcwd())
 
     pytest.raises(ImportError, _ip.magic, "load_ext daft_extension")
     daft_path = os.path.join(os.path.dirname(__file__), "daft_extension")
     sys.path.insert(0, daft_path)
     try:
         _ip.user_ns.pop('arq', None)
         invalidate_caches()   # Clear import caches
         _ip.magic("load_ext daft_extension")
         assert _ip.user_ns["arq"] == 185
         _ip.magic("unload_ext daft_extension")
         assert 'arq' not in _ip.user_ns
     finally:
         sys.path.remove(daft_path)
 
 
 def test_notebook_export_json():
     pytest.importorskip("nbformat")
     _ip = get_ipython()
     _ip.history_manager.reset()   # Clear any existing history.
     cmds = ["a=1", "def b():\n  return a**2", "print('noël, été', b())"]
     for i, cmd in enumerate(cmds, start=1):
         _ip.history_manager.store_inputs(i, cmd)
     with TemporaryDirectory() as td:
         outfile = os.path.join(td, "nb.ipynb")
         _ip.magic("notebook %s" % outfile)
 
 
 class TestEnv(TestCase):
 
     def test_env(self):
         env = _ip.magic("env")
         self.assertTrue(isinstance(env, dict))
 
     def test_env_secret(self):
         env = _ip.magic("env")
         hidden = "<hidden>"
         with mock.patch.dict(
             os.environ,
             {
                 "API_KEY": "abc123",
                 "SECRET_THING": "ssshhh",
                 "JUPYTER_TOKEN": "",
                 "VAR": "abc"
             }
         ):
             env = _ip.magic("env")
         assert env["API_KEY"] == hidden
         assert env["SECRET_THING"] == hidden
         assert env["JUPYTER_TOKEN"] == hidden
         assert env["VAR"] == "abc"
 
     def test_env_get_set_simple(self):
         env = _ip.magic("env var val1")
         self.assertEqual(env, None)
         self.assertEqual(os.environ['var'], 'val1')
         self.assertEqual(_ip.magic("env var"), 'val1')
         env = _ip.magic("env var=val2")
         self.assertEqual(env, None)
         self.assertEqual(os.environ['var'], 'val2')
 
     def test_env_get_set_complex(self):
         env = _ip.magic("env var 'val1 '' 'val2")
         self.assertEqual(env, None)
         self.assertEqual(os.environ['var'], "'val1 '' 'val2")
         self.assertEqual(_ip.magic("env var"), "'val1 '' 'val2")
         env = _ip.magic('env var=val2 val3="val4')
         self.assertEqual(env, None)
         self.assertEqual(os.environ['var'], 'val2 val3="val4')
 
     def test_env_set_bad_input(self):
         self.assertRaises(UsageError, lambda: _ip.magic("set_env var"))
 
     def test_env_set_whitespace(self):
         self.assertRaises(UsageError, lambda: _ip.magic("env var A=B"))
 
 
 class CellMagicTestCase(TestCase):
 
     def check_ident(self, magic):
         # Manually called, we get the result
         out = _ip.run_cell_magic(magic, "a", "b")
         assert out == ("a", "b")
         # Via run_cell, it goes into the user's namespace via displayhook
         _ip.run_cell("%%" + magic + " c\nd\n")
         assert _ip.user_ns["_"] == ("c", "d\n")
 
     def test_cell_magic_func_deco(self):
         "Cell magic using simple decorator"
         @register_cell_magic
         def cellm(line, cell):
             return line, cell
 
         self.check_ident('cellm')
 
     def test_cell_magic_reg(self):
         "Cell magic manually registered"
         def cellm(line, cell):
             return line, cell
 
         _ip.register_magic_function(cellm, 'cell', 'cellm2')
         self.check_ident('cellm2')
 
     def test_cell_magic_class(self):
         "Cell magics declared via a class"
         @magics_class
         class MyMagics(Magics):
 
             @cell_magic
             def cellm3(self, line, cell):
                 return line, cell
 
         _ip.register_magics(MyMagics)
         self.check_ident('cellm3')
 
     def test_cell_magic_class2(self):
         "Cell magics declared via a class, #2"
         @magics_class
         class MyMagics2(Magics):
 
             @cell_magic('cellm4')
             def cellm33(self, line, cell):
                 return line, cell
 
         _ip.register_magics(MyMagics2)
         self.check_ident('cellm4')
         # Check that nothing is registered as 'cellm33'
         c33 = _ip.find_cell_magic('cellm33')
         assert c33 == None
 
 def test_file():
     """Basic %%writefile"""
     ip = get_ipython()
     with TemporaryDirectory() as td:
         fname = os.path.join(td, 'file1')
         ip.run_cell_magic("writefile", fname, u'\n'.join([
             'line1',
             'line2',
         ]))
         s = Path(fname).read_text()
         assert "line1\n" in s
         assert "line2" in s
 
 
 @dec.skip_win32
 def test_file_single_quote():
     """Basic %%writefile with embedded single quotes"""
     ip = get_ipython()
     with TemporaryDirectory() as td:
         fname = os.path.join(td, '\'file1\'')
         ip.run_cell_magic("writefile", fname, u'\n'.join([
             'line1',
             'line2',
         ]))
         s = Path(fname).read_text()
         assert "line1\n" in s
         assert "line2" in s
 
 
 @dec.skip_win32
 def test_file_double_quote():
     """Basic %%writefile with embedded double quotes"""
     ip = get_ipython()
     with TemporaryDirectory() as td:
         fname = os.path.join(td, '"file1"')
         ip.run_cell_magic("writefile", fname, u'\n'.join([
             'line1',
             'line2',
         ]))
         s = Path(fname).read_text()
         assert "line1\n" in s
         assert "line2" in s
 
 
 def test_file_var_expand():
     """%%writefile $filename"""
     ip = get_ipython()
     with TemporaryDirectory() as td:
         fname = os.path.join(td, 'file1')
         ip.user_ns['filename'] = fname
         ip.run_cell_magic("writefile", '$filename', u'\n'.join([
             'line1',
             'line2',
         ]))
         s = Path(fname).read_text()
         assert "line1\n" in s
         assert "line2" in s
 
 
 def test_file_unicode():
     """%%writefile with unicode cell"""
     ip = get_ipython()
     with TemporaryDirectory() as td:
         fname = os.path.join(td, 'file1')
         ip.run_cell_magic("writefile", fname, u'\n'.join([
             u'liné1',
             u'liné2',
         ]))
         with io.open(fname, encoding='utf-8') as f:
             s = f.read()
         assert "liné1\n" in s
         assert "liné2" in s
 
 
 def test_file_amend():
     """%%writefile -a amends files"""
     ip = get_ipython()
     with TemporaryDirectory() as td:
         fname = os.path.join(td, 'file2')
         ip.run_cell_magic("writefile", fname, u'\n'.join([
             'line1',
             'line2',
         ]))
         ip.run_cell_magic("writefile", "-a %s" % fname, u'\n'.join([
             'line3',
             'line4',
         ]))
         s = Path(fname).read_text()
         assert "line1\n" in s
         assert "line3\n" in s
 
 
 def test_file_spaces():
     """%%file with spaces in filename"""
     ip = get_ipython()
     with TemporaryWorkingDirectory() as td:
         fname = "file name"
         ip.run_cell_magic("file", '"%s"'%fname, u'\n'.join([
             'line1',
             'line2',
         ]))
         s = Path(fname).read_text()
         assert "line1\n" in s
         assert "line2" in s
 
 
 def test_script_config():
     ip = get_ipython()
     ip.config.ScriptMagics.script_magics = ['whoda']
     sm = script.ScriptMagics(shell=ip)
     assert "whoda" in sm.magics["cell"]
 
 
 @pytest.fixture
 def event_loop():
     policy = asyncio.get_event_loop_policy()
     loop = policy.new_event_loop()
     policy.set_event_loop(loop)
     yield loop
     loop.close()
 
 
 @dec.skip_win32
 @pytest.mark.skipif(
     sys.platform == "win32", reason="This test does not run under Windows"
 )
 def test_script_out(event_loop):
     assert event_loop.is_running() is False
 
     ip = get_ipython()
     ip.run_cell_magic("script", "--out output sh", "echo 'hi'")
-    assert event_loop.is_running() is False
+    assert not event_loop.is_running()
     assert ip.user_ns["output"] == "hi\n"
 
 
 @dec.skip_win32
 @pytest.mark.skipif(
     sys.platform == "win32", reason="This test does not run under Windows"
 )
 def test_script_err(event_loop):
     ip = get_ipython()
-    assert event_loop.is_running() is False
+    assert not event_loop.is_running()
     ip.run_cell_magic("script", "--err error sh", "echo 'hello' >&2")
-    assert event_loop.is_running() is False
+    assert not event_loop.is_running()
     assert ip.user_ns["error"] == "hello\n"
 
 
 @dec.skip_win32
 @pytest.mark.skipif(
     sys.platform == "win32", reason="This test does not run under Windows"
 )
 def test_script_out_err():
 
     ip = get_ipython()
     ip.run_cell_magic(
         "script", "--out output --err error sh", "echo 'hi'\necho 'hello' >&2"
     )
     assert ip.user_ns["output"] == "hi\n"
     assert ip.user_ns["error"] == "hello\n"
 
 
 @dec.skip_win32
 @pytest.mark.skipif(
     sys.platform == "win32", reason="This test does not run under Windows"
 )
-async def test_script_bg_out(event_loop):
+async def test_script_bg_out():
     ip = get_ipython()
     ip.run_cell_magic("script", "--bg --out output sh", "echo 'hi'")
     assert (await ip.user_ns["output"].read()) == b"hi\n"
-    ip.user_ns["output"].close()
-    event_loop.stop()
-
+    assert ip.user_ns["output"].at_eof()
 
 @dec.skip_win32
 @pytest.mark.skipif(
     sys.platform == "win32", reason="This test does not run under Windows"
 )
 async def test_script_bg_err():
     ip = get_ipython()
     ip.run_cell_magic("script", "--bg --err error sh", "echo 'hello' >&2")
     assert (await ip.user_ns["error"].read()) == b"hello\n"
-    ip.user_ns["error"].close()
+    assert ip.user_ns["error"].at_eof()
 
 
 @dec.skip_win32
 @pytest.mark.skipif(
     sys.platform == "win32", reason="This test does not run under Windows"
 )
 async def test_script_bg_out_err():
     ip = get_ipython()
     ip.run_cell_magic(
         "script", "--bg --out output --err error sh", "echo 'hi'\necho 'hello' >&2"
     )
     assert (await ip.user_ns["output"].read()) == b"hi\n"
     assert (await ip.user_ns["error"].read()) == b"hello\n"
-    ip.user_ns["output"].close()
-    ip.user_ns["error"].close()
+    assert ip.user_ns["output"].at_eof()
+    assert ip.user_ns["error"].at_eof()
+
+
[e-mail]_win32
[e-mail](
+    sys.platform == "win32", reason="This test does not run under Windows"
+)
+async def test_script_bg_proc():
+    ip = get_ipython()
+    ip.run_cell_magic(
+        "script", "--bg --proc p sh --out out", "echo 'hi'\necho 'hello' >&2"
+    )
+    p = ip.user_ns["p"]
+    await p.wait()
+    assert p.returncode == 0
+    assert (await p.stdout.read()) == b"hi\n"
+    # not captured, so empty
+    assert (await p.stderr.read()) == b""
+    assert p.stdout.at_eof()
+    assert p.stderr.at_eof()
 
 
 def test_script_defaults():
     ip = get_ipython()
     for cmd in ['sh', 'bash', 'perl', 'ruby']:
         try:
             find_cmd(cmd)
         except Exception:
             pass
         else:
             assert cmd in ip.magics_manager.magics["cell"]
 
 
 @magics_class
 class FooFoo(Magics):
     """class with both %foo and %%foo magics"""
     @line_magic('foo')
     def line_foo(self, line):
         "I am line foo"
         pass
 
     @cell_magic("foo")
     def cell_foo(self, line, cell):
         "I am cell foo, not line foo"
         pass
 
 def test_line_cell_info():
     """%%foo and %foo magics are distinguishable to inspect"""
     ip = get_ipython()
     ip.magics_manager.register(FooFoo)
     oinfo = ip.object_inspect("foo")
     assert oinfo["found"] is True
     assert oinfo["ismagic"] is True
 
     oinfo = ip.object_inspect("%%foo")
     assert oinfo["found"] is True
     assert oinfo["ismagic"] is True
     assert oinfo["docstring"] == FooFoo.cell_foo.__doc__
 
     oinfo = ip.object_inspect("%foo")
     assert oinfo["found"] is True
     assert oinfo["ismagic"] is True
     assert oinfo["docstring"] == FooFoo.line_foo.__doc__
 
 
 def test_multiple_magics():
     ip = get_ipython()
     foo1 = FooFoo(ip)
     foo2 = FooFoo(ip)
     mm = ip.magics_manager
     mm.register(foo1)
     assert mm.magics["line"]["foo"].__self__ is foo1
     mm.register(foo2)
     assert mm.magics["line"]["foo"].__self__ is foo2
 
 
 def test_alias_magic():
     """Test %alias_magic."""
     ip = get_ipython()
     mm = ip.magics_manager
 
     # Basic operation: both cell and line magics are created, if possible.
     ip.run_line_magic("alias_magic", "timeit_alias timeit")
     assert "timeit_alias" in mm.magics["line"]
     assert "timeit_alias" in mm.magics["cell"]
 
     # --cell is specified, line magic not created.
     ip.run_line_magic("alias_magic", "--cell timeit_cell_alias timeit")
     assert "timeit_cell_alias" not in mm.magics["line"]
     assert "timeit_cell_alias" in mm.magics["cell"]
 
     # Test that line alias is created successfully.
     ip.run_line_magic("alias_magic", "--line env_alias env")
     assert ip.run_line_magic("env", "") == ip.run_line_magic("env_alias", "")
 
     # Test that line alias with parameters passed in is created successfully.
     ip.run_line_magic(
         "alias_magic", "--line history_alias history --params " + shlex.quote("3")
     )
     assert "history_alias" in mm.magics["line"]
 
 
 def test_save():
     """Test %save."""
     ip = get_ipython()
     ip.history_manager.reset()   # Clear any existing history.
     cmds = ["a=1", "def b():\n  return a**2", "print(a, b())"]
     for i, cmd in enumerate(cmds, start=1):
         ip.history_manager.store_inputs(i, cmd)
     with TemporaryDirectory() as tmpdir:
         file = os.path.join(tmpdir, "testsave.py")
         ip.run_line_magic("save", "%s 1-10" % file)
         content = Path(file).read_text()
         assert content.count(cmds[0]) == 1
         assert "coding: utf-8" in content
         ip.run_line_magic("save", "-a %s 1-10" % file)
         content = Path(file).read_text()
         assert content.count(cmds[0]) == 2
         assert "coding: utf-8" in content
 
 
 def test_save_with_no_args():
     ip = get_ipython()
     ip.history_manager.reset()  # Clear any existing history.
     cmds = ["a=1", "def b():\n    return a**2", "print(a, b())", "%save"]
     for i, cmd in enumerate(cmds, start=1):
         ip.history_manager.store_inputs(i, cmd)
 
     with TemporaryDirectory() as tmpdir:
         path = os.path.join(tmpdir, "testsave.py")
         ip.run_line_magic("save", path)
         content = Path(path).read_text()
         expected_content = dedent(
             """\
             # coding: utf-8
             a=1
             def b():
                 return a**2
             print(a, b())
             """
         )
         assert content == expected_content
 
 
 def test_store():
     """Test %store."""
     ip = get_ipython()
     ip.run_line_magic('load_ext', 'storemagic')
 
     # make sure the storage is empty
     ip.run_line_magic("store", "-z")
     ip.user_ns["var"] = 42
     ip.run_line_magic("store", "var")
     ip.user_ns["var"] = 39
     ip.run_line_magic("store", "-r")
     assert ip.user_ns["var"] == 42
 
     ip.run_line_magic("store", "-d var")
     ip.user_ns["var"] = 39
     ip.run_line_magic("store", "-r")
     assert ip.user_ns["var"] == 39
 
 
 def _run_edit_test(arg_s, exp_filename=None,
                         exp_lineno=-1,
                         exp_contents=None,
                         exp_is_temp=None):
     ip = get_ipython()
     M = code.CodeMagics(ip)
     last_call = ['','']
     opts,args = M.parse_options(arg_s,'prxn:')
     filename, lineno, is_temp = M._find_edit_target(ip, args, opts, last_call)
 
     if exp_filename is not None:
         assert exp_filename == filename
     if exp_contents is not None:
         with io.open(filename, 'r', encoding='utf-8') as f:
             contents = f.read()
         assert exp_contents == contents
     if exp_lineno != -1:
         assert exp_lineno == lineno
     if exp_is_temp is not None:
         assert exp_is_temp == is_temp
 
 
 def test_edit_interactive():
     """%edit on interactively defined objects"""
     ip = get_ipython()
     n = ip.execution_count
     ip.run_cell("def foo(): return 1", store_history=True)
 
     with pytest.raises(code.InteractivelyDefined) as e:
         _run_edit_test("foo")
     assert e.value.index == n
 
 
 def test_edit_cell():
     """%edit [cell id]"""
     ip = get_ipython()
 
     ip.run_cell("def foo(): return 1", store_history=True)
 
     # test
     _run_edit_test("1", exp_contents=ip.user_ns['In'][1], exp_is_temp=True)
 
 def test_edit_fname():
     """%edit file"""
     # test
     _run_edit_test("test file.py", exp_filename="test file.py")
 
 def test_bookmark():
     ip = get_ipython()
     ip.run_line_magic('bookmark', 'bmname')
     with tt.AssertPrints('bmname'):
         ip.run_line_magic('bookmark', '-l')
     ip.run_line_magic('bookmark', '-d bmname')
 
 def test_ls_magic():
     ip = get_ipython()
     json_formatter = ip.display_formatter.formatters['application/json']
     json_formatter.enabled = True
     lsmagic = ip.magic('lsmagic')
     with warnings.catch_warnings(record=True) as w:
         j = json_formatter(lsmagic)
     assert sorted(j) == ["cell", "line"]
     assert w == []  # no warnings
 
 
 def test_strip_initial_indent():
     def sii(s):
         lines = s.splitlines()
         return '\n'.join(code.strip_initial_indent(lines))
 
     assert sii("  a = 1\nb = 2") == "a = 1\nb = 2"
     assert sii("  a\n    b\nc") == "a\n  b\nc"
     assert sii("a\n  b") == "a\n  b"
 
 def test_logging_magic_quiet_from_arg():
     _ip.config.LoggingMagics.quiet = False
     lm = logging.LoggingMagics(shell=_ip)
     with TemporaryDirectory() as td:
         try:
             with tt.AssertNotPrints(re.compile("Activating.*")):
                 lm.logstart('-q {}'.format(
                         os.path.join(td, "quiet_from_arg.log")))
         finally:
             _ip.logger.logstop()
 
 def test_logging_magic_quiet_from_config():
     _ip.config.LoggingMagics.quiet = True
     lm = logging.LoggingMagics(shell=_ip)
     with TemporaryDirectory() as td:
         try:
             with tt.AssertNotPrints(re.compile("Activating.*")):
                 lm.logstart(os.path.join(td, "quiet_from_config.log"))
         finally:
             _ip.logger.logstop()
 
 
 def test_logging_magic_not_quiet():
     _ip.config.LoggingMagics.quiet = False
     lm = logging.LoggingMagics(shell=_ip)
     with TemporaryDirectory() as td:
         try:
             with tt.AssertPrints(re.compile("Activating.*")):
                 lm.logstart(os.path.join(td, "not_quiet.log"))
         finally:
             _ip.logger.logstop()
 
 
 def test_time_no_var_expand():
     _ip.user_ns['a'] = 5
     _ip.user_ns['b'] = []
     _ip.magic('time b.append("{a}")')
     assert _ip.user_ns['b'] == ['{a}']
 
 
 # this is slow, put at the end for local testing.
 def test_timeit_arguments():
     "Test valid timeit arguments, should not cause SyntaxError (GH #1269)"
     _ip.magic("timeit -n1 -r1 a=('#')")
 
 
 TEST_MODULE = """
 print('Loaded my_tmp')
 if __name__ == "__main__":
     print('I just ran a script')
 """
 
 
 def test_run_module_from_import_hook():
     "Test that a module can be loaded via an import hook"
     with TemporaryDirectory() as tmpdir:
         fullpath = os.path.join(tmpdir, 'my_tmp.py')
         Path(fullpath).write_text(TEST_MODULE)
 
         import importlib.abc
         import importlib.util
 
         class MyTempImporter(importlib.abc.MetaPathFinder, importlib.abc.SourceLoader):
             def find_spec(self, fullname, path, target=None):
                 if fullname == "my_tmp":
                     return importlib.util.spec_from_loader(fullname, self)
 
             def get_filename(self, fullname):
                 assert fullname == "my_tmp"
                 return fullpath
 
             def get_data(self, path):
                 assert Path(path).samefile(fullpath)
                 return Path(fullpath).read_text()
 
         sys.meta_path.insert(0, MyTempImporter())
 
         with capture_output() as captured:
             _ip.magic("run -m my_tmp")
             _ip.run_cell("import my_tmp")
 
         output = "Loaded my_tmp\nI just ran a script\nLoaded my_tmp\n"
         assert output == captured.stdout
 
         sys.meta_path.pop(0)

IPython/terminal/interactiveshell.py

@@ -1,699 +1,700 b''
 """IPython terminal interface using prompt_toolkit"""
 
 import asyncio
 import os
 import sys
 import warnings
 from warnings import warn
 
+from IPython.core.async_helpers import get_asyncio_loop
 from IPython.core.interactiveshell import InteractiveShell, InteractiveShellABC
 from IPython.utils import io
 from IPython.utils.py3compat import input
 from IPython.utils.terminal import toggle_set_term_title, set_term_title, restore_term_title
 from IPython.utils.process import abbrev_cwd
 from traitlets import (
     Bool,
     Unicode,
     Dict,
     Integer,
     observe,
     Instance,
     Type,
     default,
     Enum,
     Union,
     Any,
     validate,
     Float,
 )
 
 from prompt_toolkit.auto_suggest import AutoSuggestFromHistory
 from prompt_toolkit.enums import DEFAULT_BUFFER, EditingMode
 from prompt_toolkit.filters import (HasFocus, Condition, IsDone)
 from prompt_toolkit.formatted_text import PygmentsTokens
 from prompt_toolkit.history import InMemoryHistory
 from prompt_toolkit.layout.processors import ConditionalProcessor, HighlightMatchingBracketProcessor
 from prompt_toolkit.output import ColorDepth
 from prompt_toolkit.patch_stdout import patch_stdout
 from prompt_toolkit.shortcuts import PromptSession, CompleteStyle, print_formatted_text
 from prompt_toolkit.styles import DynamicStyle, merge_styles
 from prompt_toolkit.styles.pygments import style_from_pygments_cls, style_from_pygments_dict
 from prompt_toolkit import __version__ as ptk_version
 
 from pygments.styles import get_style_by_name
 from pygments.style import Style
 from pygments.token import Token
 
 from .debugger import TerminalPdb, Pdb
 from .magics import TerminalMagics
 from .pt_inputhooks import get_inputhook_name_and_func
 from .prompts import Prompts, ClassicPrompts, RichPromptDisplayHook
 from .ptutils import IPythonPTCompleter, IPythonPTLexer
 from .shortcuts import create_ipython_shortcuts
 
 PTK3 = ptk_version.startswith('3.')
 
 
 class _NoStyle(Style): pass
 
 
 
 _style_overrides_light_bg = {
             Token.Prompt: '#ansibrightblue',
             Token.PromptNum: '#ansiblue bold',
             Token.OutPrompt: '#ansibrightred',
             Token.OutPromptNum: '#ansired bold',
 }
 
 _style_overrides_linux = {
             Token.Prompt: '#ansibrightgreen',
             Token.PromptNum: '#ansigreen bold',
             Token.OutPrompt: '#ansibrightred',
             Token.OutPromptNum: '#ansired bold',
 }
 
 def get_default_editor():
     try:
         return os.environ['EDITOR']
     except KeyError:
         pass
     except UnicodeError:
         warn("$EDITOR environment variable is not pure ASCII. Using platform "
              "default editor.")
 
     if os.name == 'posix':
         return 'vi'  # the only one guaranteed to be there!
     else:
         return 'notepad' # same in Windows!
 
 # conservatively check for tty
 # overridden streams can result in things like:
 # - sys.stdin = None
 # - no isatty method
 for _name in ('stdin', 'stdout', 'stderr'):
     _stream = getattr(sys, _name)
     if not _stream or not hasattr(_stream, 'isatty') or not _stream.isatty():
         _is_tty = False
         break
 else:
     _is_tty = True
 
 
 _use_simple_prompt = ('IPY_TEST_SIMPLE_PROMPT' in os.environ) or (not _is_tty)
 
 def black_reformat_handler(text_before_cursor):
     """
     We do not need to protect against error,
     this is taken care at a higher level where any reformat error is ignored.
     Indeed we may call reformatting on incomplete code.
     """
     import black
 
     formatted_text = black.format_str(text_before_cursor, mode=black.FileMode())
     if not text_before_cursor.endswith("\n") and formatted_text.endswith("\n"):
         formatted_text = formatted_text[:-1]
     return formatted_text
 
 
 class TerminalInteractiveShell(InteractiveShell):
     mime_renderers = Dict().tag(config=True)
 
     space_for_menu = Integer(6, help='Number of line at the bottom of the screen '
                                      'to reserve for the tab completion menu, '
                                      'search history, ...etc, the height of '
                                      'these menus will at most this value. '
                                      'Increase it is you prefer long and skinny '
                                      'menus, decrease for short and wide.'
                             ).tag(config=True)
 
     pt_app = None
     debugger_history = None
 
     debugger_history_file = Unicode(
         "~/.pdbhistory", help="File in which to store and read history"
     ).tag(config=True)
 
     simple_prompt = Bool(_use_simple_prompt,
         help="""Use `raw_input` for the REPL, without completion and prompt colors.
 
             Useful when controlling IPython as a subprocess, and piping STDIN/OUT/ERR. Known usage are:
             IPython own testing machinery, and emacs inferior-shell integration through elpy.
 
             This mode default to `True` if the `IPY_TEST_SIMPLE_PROMPT`
             environment variable is set, or the current terminal is not a tty."""
             ).tag(config=True)
 
     @property
     def debugger_cls(self):
         return Pdb if self.simple_prompt else TerminalPdb
 
     confirm_exit = Bool(True,
         help="""
         Set to confirm when you try to exit IPython with an EOF (Control-D
         in Unix, Control-Z/Enter in Windows). By typing 'exit' or 'quit',
         you can force a direct exit without any confirmation.""",
     ).tag(config=True)
 
     editing_mode = Unicode('emacs',
         help="Shortcut style to use at the prompt. 'vi' or 'emacs'.",
     ).tag(config=True)
 
     emacs_bindings_in_vi_insert_mode = Bool(
         True,
         help="Add shortcuts from 'emacs' insert mode to 'vi' insert mode.",
     ).tag(config=True)
 
     modal_cursor = Bool(
         True,
         help="""
        Cursor shape changes depending on vi mode: beam in vi insert mode,
        block in nav mode, underscore in replace mode.""",
     ).tag(config=True)
 
     ttimeoutlen = Float(
         0.01,
         help="""The time in milliseconds that is waited for a key code
        to complete.""",
     ).tag(config=True)
 
     timeoutlen = Float(
         0.5,
         help="""The time in milliseconds that is waited for a mapped key
        sequence to complete.""",
     ).tag(config=True)
 
     autoformatter = Unicode(
         "black",
         help="Autoformatter to reformat Terminal code. Can be `'black'` or `None`",
         allow_none=True
     ).tag(config=True)
 
     auto_match = Bool(
         False,
         help="""
         Automatically add/delete closing bracket or quote when opening bracket or quote is entered/deleted.
         Brackets: (), [], {}
         Quotes: '', \"\"
         """,
     ).tag(config=True)
 
     mouse_support = Bool(False,
         help="Enable mouse support in the prompt\n(Note: prevents selecting text with the mouse)"
     ).tag(config=True)
 
     # We don't load the list of styles for the help string, because loading
     # Pygments plugins takes time and can cause unexpected errors.
     highlighting_style = Union([Unicode('legacy'), Type(klass=Style)],
         help="""The name or class of a Pygments style to use for syntax
         highlighting. To see available styles, run `pygmentize -L styles`."""
     ).tag(config=True)
 
     @validate('editing_mode')
     def _validate_editing_mode(self, proposal):
         if proposal['value'].lower() == 'vim':
             proposal['value']= 'vi'
         elif proposal['value'].lower() == 'default':
             proposal['value']= 'emacs'
 
         if hasattr(EditingMode, proposal['value'].upper()):
             return proposal['value'].lower()
 
         return self.editing_mode
 
 
     @observe('editing_mode')
     def _editing_mode(self, change):
         if self.pt_app:
             self.pt_app.editing_mode = getattr(EditingMode, change.new.upper())
 
     def _set_formatter(self, formatter):
         if formatter is None:
             self.reformat_handler = lambda x:x
         elif formatter == 'black':
             self.reformat_handler = black_reformat_handler
         else:
             raise ValueError
 
     @observe("autoformatter")
     def _autoformatter_changed(self, change):
         formatter = change.new
         self._set_formatter(formatter)
 
     @observe('highlighting_style')
     @observe('colors')
     def _highlighting_style_changed(self, change):
         self.refresh_style()
 
     def refresh_style(self):
         self._style = self._make_style_from_name_or_cls(self.highlighting_style)
 
 
     highlighting_style_overrides = Dict(
         help="Override highlighting format for specific tokens"
     ).tag(config=True)
 
     true_color = Bool(False,
         help="""Use 24bit colors instead of 256 colors in prompt highlighting.
         If your terminal supports true color, the following command should
         print ``TRUECOLOR`` in orange::
 
             printf \"\\x1b[38;2;255;100;0mTRUECOLOR\\x1b[0m\\n\"
         """,
     ).tag(config=True)
 
     editor = Unicode(get_default_editor(),
         help="Set the editor used by IPython (default to $EDITOR/vi/notepad)."
     ).tag(config=True)
 
     prompts_class = Type(Prompts, help='Class used to generate Prompt token for prompt_toolkit').tag(config=True)
 
     prompts = Instance(Prompts)
 
     @default('prompts')
     def _prompts_default(self):
         return self.prompts_class(self)
 
 #    @observe('prompts')
 #    def _(self, change):
 #        self._update_layout()
 
     @default('displayhook_class')
     def _displayhook_class_default(self):
         return RichPromptDisplayHook
 
     term_title = Bool(True,
         help="Automatically set the terminal title"
     ).tag(config=True)
 
     term_title_format = Unicode("IPython: {cwd}",
         help="Customize the terminal title format.  This is a python format string. " +
              "Available substitutions are: {cwd}."
     ).tag(config=True)
 
     display_completions = Enum(('column', 'multicolumn','readlinelike'),
         help= ( "Options for displaying tab completions, 'column', 'multicolumn', and "
                 "'readlinelike'. These options are for `prompt_toolkit`, see "
                 "`prompt_toolkit` documentation for more information."
                 ),
         default_value='multicolumn').tag(config=True)
 
     highlight_matching_brackets = Bool(True,
         help="Highlight matching brackets.",
     ).tag(config=True)
 
     extra_open_editor_shortcuts = Bool(False,
         help="Enable vi (v) or Emacs (C-X C-E) shortcuts to open an external editor. "
              "This is in addition to the F2 binding, which is always enabled."
     ).tag(config=True)
 
     handle_return = Any(None,
         help="Provide an alternative handler to be called when the user presses "
              "Return. This is an advanced option intended for debugging, which "
              "may be changed or removed in later releases."
     ).tag(config=True)
 
     enable_history_search = Bool(True,
         help="Allows to enable/disable the prompt toolkit history search"
     ).tag(config=True)
 
     prompt_includes_vi_mode = Bool(True,
         help="Display the current vi mode (when using vi editing mode)."
     ).tag(config=True)
 
     @observe('term_title')
     def init_term_title(self, change=None):
         # Enable or disable the terminal title.
         if self.term_title:
             toggle_set_term_title(True)
             set_term_title(self.term_title_format.format(cwd=abbrev_cwd()))
         else:
             toggle_set_term_title(False)
 
     def restore_term_title(self):
         if self.term_title:
             restore_term_title()
 
     def init_display_formatter(self):
         super(TerminalInteractiveShell, self).init_display_formatter()
         # terminal only supports plain text
         self.display_formatter.active_types = ["text/plain"]
 
     def init_prompt_toolkit_cli(self):
         if self.simple_prompt:
             # Fall back to plain non-interactive output for tests.
             # This is very limited.
             def prompt():
                 prompt_text = "".join(x[1] for x in self.prompts.in_prompt_tokens())
                 lines = [input(prompt_text)]
                 prompt_continuation = "".join(x[1] for x in self.prompts.continuation_prompt_tokens())
                 while self.check_complete('\n'.join(lines))[0] == 'incomplete':
                     lines.append( input(prompt_continuation) )
                 return '\n'.join(lines)
             self.prompt_for_code = prompt
             return
 
         # Set up keyboard shortcuts
         key_bindings = create_ipython_shortcuts(self)
 
         # Pre-populate history from IPython's history database
         history = InMemoryHistory()
         last_cell = u""
         for __, ___, cell in self.history_manager.get_tail(self.history_load_length,
                                                         include_latest=True):
             # Ignore blank lines and consecutive duplicates
             cell = cell.rstrip()
             if cell and (cell != last_cell):
                 history.append_string(cell)
                 last_cell = cell
 
         self._style = self._make_style_from_name_or_cls(self.highlighting_style)
         self.style = DynamicStyle(lambda: self._style)
 
         editing_mode = getattr(EditingMode, self.editing_mode.upper())
 
         self.pt_loop = asyncio.new_event_loop()
         self.pt_app = PromptSession(
             auto_suggest=AutoSuggestFromHistory(),
             editing_mode=editing_mode,
             key_bindings=key_bindings,
             history=history,
             completer=IPythonPTCompleter(shell=self),
             enable_history_search=self.enable_history_search,
             style=self.style,
             include_default_pygments_style=False,
             mouse_support=self.mouse_support,
             enable_open_in_editor=self.extra_open_editor_shortcuts,
             color_depth=self.color_depth,
             tempfile_suffix=".py",
             **self._extra_prompt_options()
         )
 
     def _make_style_from_name_or_cls(self, name_or_cls):
         """
         Small wrapper that make an IPython compatible style from a style name
 
         We need that to add style for prompt ... etc.
         """
         style_overrides = {}
         if name_or_cls == 'legacy':
             legacy = self.colors.lower()
             if legacy == 'linux':
                 style_cls = get_style_by_name('monokai')
                 style_overrides = _style_overrides_linux
             elif legacy == 'lightbg':
                 style_overrides = _style_overrides_light_bg
                 style_cls = get_style_by_name('pastie')
             elif legacy == 'neutral':
                 # The default theme needs to be visible on both a dark background
                 # and a light background, because we can't tell what the terminal
                 # looks like. These tweaks to the default theme help with that.
                 style_cls = get_style_by_name('default')
                 style_overrides.update({
                     Token.Number: '#ansigreen',
                     Token.Operator: 'noinherit',
                     Token.String: '#ansiyellow',
                     Token.Name.Function: '#ansiblue',
                     Token.Name.Class: 'bold #ansiblue',
                     Token.Name.Namespace: 'bold #ansiblue',
                     Token.Name.Variable.Magic: '#ansiblue',
                     Token.Prompt: '#ansigreen',
                     Token.PromptNum: '#ansibrightgreen bold',
                     Token.OutPrompt: '#ansired',
                     Token.OutPromptNum: '#ansibrightred bold',
                 })
 
                 # Hack: Due to limited color support on the Windows console
                 # the prompt colors will be wrong without this
                 if os.name == 'nt':
                     style_overrides.update({
                         Token.Prompt: '#ansidarkgreen',
                         Token.PromptNum: '#ansigreen bold',
                         Token.OutPrompt: '#ansidarkred',
                         Token.OutPromptNum: '#ansired bold',
                     })
             elif legacy =='nocolor':
                 style_cls=_NoStyle
                 style_overrides = {}
             else :
                 raise ValueError('Got unknown colors: ', legacy)
         else :
             if isinstance(name_or_cls, str):
                 style_cls = get_style_by_name(name_or_cls)
             else:
                 style_cls = name_or_cls
             style_overrides = {
                 Token.Prompt: '#ansigreen',
                 Token.PromptNum: '#ansibrightgreen bold',
                 Token.OutPrompt: '#ansired',
                 Token.OutPromptNum: '#ansibrightred bold',
             }
         style_overrides.update(self.highlighting_style_overrides)
         style = merge_styles([
             style_from_pygments_cls(style_cls),
             style_from_pygments_dict(style_overrides),
         ])
 
         return style
 
     @property
     def pt_complete_style(self):
         return {
             'multicolumn': CompleteStyle.MULTI_COLUMN,
             'column': CompleteStyle.COLUMN,
             'readlinelike': CompleteStyle.READLINE_LIKE,
         }[self.display_completions]
 
     @property
     def color_depth(self):
         return (ColorDepth.TRUE_COLOR if self.true_color else None)
 
     def _extra_prompt_options(self):
         """
         Return the current layout option for the current Terminal InteractiveShell
         """
         def get_message():
             return PygmentsTokens(self.prompts.in_prompt_tokens())
 
         if self.editing_mode == 'emacs':
             # with emacs mode the prompt is (usually) static, so we call only
             # the function once. With VI mode it can toggle between [ins] and
             # [nor] so we can't precompute.
             # here I'm going to favor the default keybinding which almost
             # everybody uses to decrease CPU usage.
             # if we have issues with users with custom Prompts we can see how to
             # work around this.
             get_message = get_message()
 
         options = {
                 'complete_in_thread': False,
                 'lexer':IPythonPTLexer(),
                 'reserve_space_for_menu':self.space_for_menu,
                 'message': get_message,
                 'prompt_continuation': (
                     lambda width, lineno, is_soft_wrap:
                         PygmentsTokens(self.prompts.continuation_prompt_tokens(width))),
                 'multiline': True,
                 'complete_style': self.pt_complete_style,
 
                 # Highlight matching brackets, but only when this setting is
                 # enabled, and only when the DEFAULT_BUFFER has the focus.
                 'input_processors': [ConditionalProcessor(
                         processor=HighlightMatchingBracketProcessor(chars='[](){}'),
                         filter=HasFocus(DEFAULT_BUFFER) & ~IsDone() &
                             Condition(lambda: self.highlight_matching_brackets))],
                 }
         if not PTK3:
             options['inputhook'] = self.inputhook
 
         return options
 
     def prompt_for_code(self):
         if self.rl_next_input:
             default = self.rl_next_input
             self.rl_next_input = None
         else:
             default = ''
 
         # In order to make sure that asyncio code written in the
         # interactive shell doesn't interfere with the prompt, we run the
         # prompt in a different event loop.
         # If we don't do this, people could spawn coroutine with a
         # while/true inside which will freeze the prompt.
 
         policy = asyncio.get_event_loop_policy()
-        try:
-            old_loop = policy.get_event_loop()
-        except RuntimeError:
-            # This happens when the the event loop is closed,
-            # e.g. by calling `asyncio.run()`.
-            old_loop = None
-
-        policy.set_event_loop(self.pt_loop)
+        old_loop = get_asyncio_loop()
+
+        # FIXME: prompt_toolkit is using the deprecated `asyncio.get_event_loop`
+        # to get the current event loop.
+        # This will probably be replaced by an attribute or input argument,
+        # at which point we can stop calling the soon-to-be-deprecated `set_event_loop` here.
+        if old_loop is not self.pt_loop:
+            policy.set_event_loop(self.pt_loop)
         try:
             with patch_stdout(raw=True):
                 text = self.pt_app.prompt(
                     default=default,
                     **self._extra_prompt_options())
         finally:
             # Restore the original event loop.
-            if old_loop is not None:
+            if old_loop is not None and old_loop is not self.pt_loop:
                 policy.set_event_loop(old_loop)
 
         return text
 
     def enable_win_unicode_console(self):
         # Since IPython 7.10 doesn't support python < 3.6 and PEP 528, Python uses the unicode APIs for the Windows
         # console by default, so WUC shouldn't be needed.
         from warnings import warn
         warn("`enable_win_unicode_console` is deprecated since IPython 7.10, does not do anything and will be removed in the future",
              DeprecationWarning,
              stacklevel=2)
 
     def init_io(self):
         if sys.platform not in {'win32', 'cli'}:
             return
 
         import colorama
         colorama.init()
 
     def init_magics(self):
         super(TerminalInteractiveShell, self).init_magics()
         self.register_magics(TerminalMagics)
 
     def init_alias(self):
         # The parent class defines aliases that can be safely used with any
         # frontend.
         super(TerminalInteractiveShell, self).init_alias()
 
         # Now define aliases that only make sense on the terminal, because they
         # need direct access to the console in a way that we can't emulate in
         # GUI or web frontend
         if os.name == 'posix':
             for cmd in ('clear', 'more', 'less', 'man'):
                 self.alias_manager.soft_define_alias(cmd, cmd)
 
 
     def __init__(self, *args, **kwargs):
         super(TerminalInteractiveShell, self).__init__(*args, **kwargs)
         self.init_prompt_toolkit_cli()
         self.init_term_title()
         self.keep_running = True
         self._set_formatter(self.autoformatter)
 
 
     def ask_exit(self):
         self.keep_running = False
 
     rl_next_input = None
 
     def interact(self):
         self.keep_running = True
         while self.keep_running:
             print(self.separate_in, end='')
 
             try:
                 code = self.prompt_for_code()
             except EOFError:
                 if (not self.confirm_exit) \
                         or self.ask_yes_no('Do you really want to exit ([y]/n)?','y','n'):
                     self.ask_exit()
 
             else:
                 if code:
                     self.run_cell(code, store_history=True)
 
     def mainloop(self):
         # An extra layer of protection in case someone mashing Ctrl-C breaks
         # out of our internal code.
         while True:
             try:
                 self.interact()
                 break
             except KeyboardInterrupt as e:
                 print("\n%s escaped interact()\n" % type(e).__name__)
             finally:
                 # An interrupt during the eventloop will mess up the
                 # internal state of the prompt_toolkit library.
                 # Stopping the eventloop fixes this, see
                 # https://github.com/ipython/ipython/pull/9867
                 if hasattr(self, '_eventloop'):
                     self._eventloop.stop()
 
                 self.restore_term_title()
 
         # try to call some at-exit operation optimistically as some things can't
         # be done during interpreter shutdown. this is technically inaccurate as
         # this make mainlool not re-callable, but that should be a rare if not
         # in existent use case.
 
         self._atexit_once()
 
 
     _inputhook = None
     def inputhook(self, context):
         if self._inputhook is not None:
             self._inputhook(context)
 
     active_eventloop = None
     def enable_gui(self, gui=None):
         if gui and (gui != 'inline') :
             self.active_eventloop, self._inputhook =\
                 get_inputhook_name_and_func(gui)
         else:
             self.active_eventloop = self._inputhook = None
 
         # For prompt_toolkit 3.0. We have to create an asyncio event loop with
         # this inputhook.
         if PTK3:
             import asyncio
             from prompt_toolkit.eventloop import new_eventloop_with_inputhook
 
             if gui == 'asyncio':
                 # When we integrate the asyncio event loop, run the UI in the
                 # same event loop as the rest of the code. don't use an actual
                 # input hook. (Asyncio is not made for nesting event loops.)
-                self.pt_loop = asyncio.get_event_loop_policy().get_event_loop()
+                self.pt_loop = get_asyncio_loop()
 
             elif self._inputhook:
                 # If an inputhook was set, create a new asyncio event loop with
                 # this inputhook for the prompt.
                 self.pt_loop = new_eventloop_with_inputhook(self._inputhook)
             else:
                 # When there's no inputhook, run the prompt in a separate
                 # asyncio event loop.
                 self.pt_loop = asyncio.new_event_loop()
 
     # Run !system commands directly, not through pipes, so terminal programs
     # work correctly.
     system = InteractiveShell.system_raw
 
     def auto_rewrite_input(self, cmd):
         """Overridden from the parent class to use fancy rewriting prompt"""
         if not self.show_rewritten_input:
             return
 
         tokens = self.prompts.rewrite_prompt_tokens()
         if self.pt_app:
             print_formatted_text(PygmentsTokens(tokens), end='',
                                  style=self.pt_app.app.style)
             print(cmd)
         else:
             prompt = ''.join(s for t, s in tokens)
             print(prompt, cmd, sep='')
 
     _prompts_before = None
     def switch_doctest_mode(self, mode):
         """Switch prompts to classic for %doctest_mode"""
         if mode:
             self._prompts_before = self.prompts
             self.prompts = ClassicPrompts(self)
         elif self._prompts_before:
             self.prompts = self._prompts_before
             self._prompts_before = None
 #        self._update_layout()
 
 
 InteractiveShellABC.register(TerminalInteractiveShell)
 
 if __name__ == '__main__':
     TerminalInteractiveShell.instance().interact()

IPython/terminal/pt_inputhooks/asyncio.py

@@ -1,64 +1,63 b''
 """
 Inputhook for running the original asyncio event loop while we're waiting for
 input.
 
 By default, in IPython, we run the prompt with a different asyncio event loop,
 because otherwise we risk that people are freezing the prompt by scheduling bad
 coroutines. E.g., a coroutine that does a while/true and never yield back
 control to the loop. We can't cancel that.
 
 However, sometimes we want the asyncio loop to keep running while waiting for
 a prompt.
 
 The following example will print the numbers from 1 to 10 above the prompt,
 while we are waiting for input. (This works also because we use
 prompt_toolkit`s `patch_stdout`)::
 
     In [1]: import asyncio
 
     In [2]: %gui asyncio
 
     In [3]: async def f():
        ...:     for i in range(10):
        ...:         await asyncio.sleep(1)
        ...:         print(i)
 
 
     In [4]: asyncio.ensure_future(f())
 
 """
-import asyncio
 from prompt_toolkit import __version__ as ptk_version
 
-PTK3 = ptk_version.startswith('3.')
+from IPython.core.async_helpers import get_asyncio_loop
 
+PTK3 = ptk_version.startswith('3.')
 
-# Keep reference to the original asyncio loop, because getting the event loop
-# within the input hook would return the other loop.
-loop = asyncio.get_event_loop_policy().get_event_loop()
 
 
 def inputhook(context):
     """
     Inputhook for asyncio event loop integration.
     """
     # For prompt_toolkit 3.0, this input hook literally doesn't do anything.
     # The event loop integration here is implemented in `interactiveshell.py`
     # by running the prompt itself in the current asyncio loop. The main reason
     # for this is that nesting asyncio event loops is unreliable.
     if PTK3:
         return
 
     # For prompt_toolkit 2.0, we can run the current asyncio event loop,
     # because prompt_toolkit 2.0 uses a different event loop internally.
 
+    # get the persistent asyncio event loop
+    loop = get_asyncio_loop()
+
     def stop():
         loop.stop()
 
     fileno = context.fileno()
     loop.add_reader(fileno, stop)
     try:
         loop.run_forever()
     finally:
         loop.remove_reader(fileno)
-