From 25f92b9988c81b70b444e88b4afc5aab8a875d8e 2021-10-17 19:47:25 From: Matthias Bussonnier Date: 2021-10-17 19:47:25 Subject: [PATCH] Try to fix some of current failures. See error found in CI below. Hopefully this fixes it. There are manycomplexity, we need to make sure the loop has no task after each tests, so we need to close and restore the subprocess watcher. We make it a context manager and provide a decorator. ====================================================================== ERROR: IPython.core.tests.test_magic.test_script_out ---------------------------------------------------------------------- Traceback (most recent call last): File "/opt/hostedtoolcache/Python/3.7.12/x64/lib/python3.7/site-packages/nose/case.py", line 198, in runTest self.test(*self.arg) File "/home/runner/work/ipython/ipython/IPython/testing/decorators.py", line 223, in skipper_func return f(*args, **kwargs) File "/home/runner/work/ipython/ipython/IPython/core/tests/test_magic.py", line 953, in test_script_out ip.run_cell_magic("script", "--out output sh", "echo 'hi'") File "/home/runner/work/ipython/ipython/IPython/core/interactiveshell.py", line 2417, in run_cell_magic result = fn(*args, **kwargs) File "/opt/hostedtoolcache/Python/3.7.12/x64/lib/python3.7/site-packages/decorator.py", line 232, in fun return caller(func, *(extras + args), **kw) File "/home/runner/work/ipython/ipython/IPython/core/magic.py", line 187, in call = lambda f, *a, **k: f(*a, **k) File "/home/runner/work/ipython/ipython/IPython/core/magics/script.py", line 216, in shebang stdin=asyncio.subprocess.PIPE, File "/opt/hostedtoolcache/Python/3.7.12/x64/lib/python3.7/asyncio/base_events.py", line 587, in run_until_complete return future.result() File "/opt/hostedtoolcache/Python/3.7.12/x64/lib/python3.7/asyncio/subprocess.py", line 217, in create_subprocess_exec stderr=stderr, **kwds) File "/opt/hostedtoolcache/Python/3.7.12/x64/lib/python3.7/asyncio/base_events.py", line 1544, in subprocess_exec bufsize, **kwargs) File "/opt/hostedtoolcache/Python/3.7.12/x64/lib/python3.7/asyncio/unix_events.py", line 193, in _make_subprocess_transport self._child_watcher_callback, transp) File "/opt/hostedtoolcache/Python/3.7.12/x64/lib/python3.7/asyncio/unix_events.py", line 941, in add_child_handler "Cannot add child handler, " RuntimeError: Cannot add child handler, the child watcher does not have a loop attached --- diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index ef24d62..36ae818 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -13,6 +13,7 @@ jobs: build: runs-on: ubuntu-latest + timeout-minutes: 10 strategy: matrix: python-version: [3.8] diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2788e86..19a0ba0 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -45,6 +45,6 @@ jobs: cp /tmp/.coverage ./ - name: pytest run: | - pytest + pytest -v - name: Upload coverage to Codecov uses: codecov/codecov-action@v1 diff --git a/IPython/core/magics/script.py b/IPython/core/magics/script.py index d4e7b08..99258a8 100644 --- a/IPython/core/magics/script.py +++ b/IPython/core/magics/script.py @@ -3,24 +3,24 @@ # Copyright (c) IPython Development Team. # Distributed under the terms of the Modified BSD License. +import asyncio +import atexit import errno +import functools import os -import sys import signal +import sys import time -import asyncio -import atexit - +from asyncio import SafeChildWatcher +from contextlib import contextmanager from subprocess import CalledProcessError +from traitlets import Dict, List, default + from IPython.core import magic_arguments -from IPython.core.magic import ( - Magics, magics_class, line_magic, cell_magic -) +from IPython.core.magic import Magics, cell_magic, line_magic, magics_class from IPython.lib.backgroundjobs import BackgroundJobManager from IPython.utils.process import arg_split -from traitlets import List, Dict, default - #----------------------------------------------------------------------------- # Magic implementation classes @@ -67,6 +67,39 @@ def script_args(f): f = arg(f) return f + +@contextmanager +def safe_watcher(): + if sys.platform == "win32": + yield + return + + policy = asyncio.get_event_loop_policy() + old_watcher = policy.get_child_watcher() + if isinstance(old_watcher, SafeChildWatcher): + yield + return + + loop = asyncio.get_event_loop() + try: + watcher = asyncio.SafeChildWatcher() + watcher.attach_loop(loop) + policy.set_child_watcher(watcher) + yield + finally: + watcher.close() + policy.set_child_watcher(old_watcher) + + +def dec_safe_watcher(fun): + @functools.wraps(fun) + def _inner(*args, **kwargs): + with safe_watcher(): + return fun(*args, **kwargs) + + return _inner + + @magics_class class ScriptMagics(Magics): """Magics for talking to scripts @@ -157,6 +190,7 @@ class ScriptMagics(Magics): @magic_arguments.magic_arguments() @script_args @cell_magic("script") + @dec_safe_watcher def shebang(self, line, cell): """Run a cell via a shell command @@ -204,7 +238,6 @@ class ScriptMagics(Magics): if sys.platform.startswith("win"): asyncio.set_event_loop_policy(asyncio.WindowsProactorEventLoopPolicy()) loop = asyncio.get_event_loop() - argv = arg_split(line, posix=not sys.platform.startswith("win")) args, cmd = self.shebang.parser.parse_known_args(argv) try: diff --git a/IPython/core/tests/test_magic.py b/IPython/core/tests/test_magic.py index fd5696d..6ecbe61 100644 --- a/IPython/core/tests/test_magic.py +++ b/IPython/core/tests/test_magic.py @@ -4,38 +4,40 @@ Needs to be run by nose (to make ipython session available). """ +import asyncio import io import os import re +import shlex import sys import warnings -from textwrap import dedent -from unittest import TestCase -from unittest import mock from importlib import invalidate_caches from io import StringIO from pathlib import Path +from textwrap import dedent +from unittest import TestCase, mock import nose.tools as nt -import shlex - from IPython import get_ipython from IPython.core import magic from IPython.core.error import UsageError -from IPython.core.magic import (Magics, magics_class, line_magic, - cell_magic, - register_line_magic, register_cell_magic) -from IPython.core.magics import execution, script, code, logging, osm +from IPython.core.magic import ( + Magics, + cell_magic, + line_magic, + magics_class, + register_cell_magic, + register_line_magic, +) +from IPython.core.magics import code, execution, logging, osm, script from IPython.testing import decorators as dec from IPython.testing import tools as tt from IPython.utils.io import capture_output -from IPython.utils.tempdir import (TemporaryDirectory, - TemporaryWorkingDirectory) from IPython.utils.process import find_cmd -from .test_debugger import PdbTestInput +from IPython.utils.tempdir import TemporaryDirectory, TemporaryWorkingDirectory -import pytest +from .test_debugger import PdbTestInput @magic.magics_class @@ -947,32 +949,48 @@ def test_script_config(): sm = script.ScriptMagics(shell=ip) nt.assert_in('whoda', sm.magics['cell']) +@dec.skip_iptest_but_not_pytest @dec.skip_win32 def test_script_out(): + assert asyncio.get_event_loop().is_running() is False + ip = get_ipython() ip.run_cell_magic("script", "--out output sh", "echo 'hi'") + assert asyncio.get_event_loop().is_running() is False nt.assert_equal(ip.user_ns['output'], 'hi\n') +@dec.skip_iptest_but_not_pytest @dec.skip_win32 def test_script_err(): ip = get_ipython() + assert asyncio.get_event_loop().is_running() is False ip.run_cell_magic("script", "--err error sh", "echo 'hello' >&2") + assert asyncio.get_event_loop().is_running() is False nt.assert_equal(ip.user_ns['error'], 'hello\n') + +@dec.skip_iptest_but_not_pytest @dec.skip_win32 def test_script_out_err(): + ip = get_ipython() - ip.run_cell_magic("script", "--out output --err error sh", "echo 'hi'\necho 'hello' >&2") - nt.assert_equal(ip.user_ns['output'], 'hi\n') - nt.assert_equal(ip.user_ns['error'], 'hello\n') + ip.run_cell_magic( + "script", "--out output --err error sh", "echo 'hi'\necho 'hello' >&2" + ) + nt.assert_equal(ip.user_ns["output"], "hi\n") + nt.assert_equal(ip.user_ns["error"], "hello\n") + +@dec.skip_iptest_but_not_pytest @dec.skip_win32 async def test_script_bg_out(): ip = get_ipython() ip.run_cell_magic("script", "--bg --out output sh", "echo 'hi'") nt.assert_equal((await ip.user_ns["output"].read()), b"hi\n") - ip.user_ns['output'].close() + ip.user_ns["output"].close() + asyncio.get_event_loop().stop() +@dec.skip_iptest_but_not_pytest @dec.skip_win32 async def test_script_bg_err(): ip = get_ipython() @@ -981,6 +999,7 @@ async def test_script_bg_err(): ip.user_ns["error"].close() +@dec.skip_iptest_but_not_pytest @dec.skip_win32 async def test_script_bg_out_err(): ip = get_ipython()