iptest.py
632 lines
| 23.1 KiB
| text/x-python
|
PythonLexer
Fernando Perez
|
r1574 | # -*- coding: utf-8 -*- | ||
"""IPython Test Suite Runner. | ||||
Fernando Perez
|
r1851 | |||
Brian Granger
|
r1972 | This module provides a main entry point to a user script to test IPython | ||
itself from the command line. There are two ways of running this script: | ||||
1. With the syntax `iptest all`. This runs our entire test suite by | ||||
Brian Granger
|
r3486 | calling this script (with different arguments) recursively. This | ||
Brian Granger
|
r1972 | causes modules and package to be tested in different processes, using nose | ||
or trial where appropriate. | ||||
2. With the regular nose syntax, like `iptest -vvs IPython`. In this form | ||||
the script simply calls nose, but with special command line flags and | ||||
plugins loaded. | ||||
Fernando Perez
|
r1574 | """ | ||
Fernando Perez
|
r1851 | #----------------------------------------------------------------------------- | ||
Matthias BUSSONNIER
|
r5390 | # Copyright (C) 2009-2011 The IPython Development Team | ||
Brian Granger
|
r2498 | # | ||
# Distributed under the terms of the BSD License. The full license is in | ||||
# the file COPYING, distributed as part of this software. | ||||
#----------------------------------------------------------------------------- | ||||
#----------------------------------------------------------------------------- | ||||
# Imports | ||||
Fernando Perez
|
r1851 | #----------------------------------------------------------------------------- | ||
Matthias BUSSONNIER
|
r7817 | from __future__ import print_function | ||
Fernando Perez
|
r1851 | |||
Fernando Perez
|
r2442 | # Stdlib | ||
MinRK
|
r7412 | import glob | ||
Brian Granger
|
r1972 | import os | ||
import os.path as path | ||||
Fernando Perez
|
r2399 | import signal | ||
Fernando Perez
|
r1574 | import sys | ||
Brian Granger
|
r1972 | import subprocess | ||
Fernando Perez
|
r2111 | import tempfile | ||
Brian Granger
|
r1972 | import time | ||
Fernando Perez
|
r1574 | import warnings | ||
Fernando Perez
|
r2442 | # Note: monkeypatch! | ||
# We need to monkeypatch a small problem in nose itself first, before importing | ||||
# it for actual use. This should get into nose upstream, but its release cycle | ||||
# is slow and we need it for our parametric tests to work correctly. | ||||
Fernando Perez
|
r2480 | from IPython.testing import nosepatch | ||
Thomas Kluyver
|
r7213 | |||
# Monkeypatch extra assert methods into nose.tools if they're not already there. | ||||
# This can be dropped once we no longer test on Python 2.6 | ||||
from IPython.testing import nose_assert_methods | ||||
Fernando Perez
|
r2442 | # Now, proceed to import nose itself | ||
Fernando Perez
|
r1574 | import nose.plugins.builtin | ||
Thomas Kluyver
|
r6101 | from nose.plugins.xunit import Xunit | ||
from nose import SkipTest | ||||
Fernando Perez
|
r1851 | from nose.core import TestProgram | ||
Fernando Perez
|
r1574 | |||
Fernando Perez
|
r2442 | # Our own imports | ||
Jörgen Stenarson
|
r7732 | from IPython.utils import py3compat | ||
MinRK
|
r4857 | from IPython.utils.importstring import import_item | ||
MinRK
|
r7126 | from IPython.utils.path import get_ipython_module_path, get_ipython_package_dir | ||
Thomas Kluyver
|
r11133 | from IPython.utils.process import pycmd2argv | ||
Brian Granger
|
r2498 | from IPython.utils.sysinfo import sys_info | ||
Bradley M. Froehle
|
r8038 | from IPython.utils.tempdir import TemporaryDirectory | ||
MinRK
|
r7121 | from IPython.utils.warn import warn | ||
Brian Granger
|
r2498 | |||
Fernando Perez
|
r2480 | from IPython.testing import globalipapp | ||
from IPython.testing.plugin.ipdoctest import IPythonDoctest | ||||
Thomas Kluyver
|
r6101 | from IPython.external.decorators import KnownFailure, knownfailureif | ||
Fernando Perez
|
r1574 | |||
Brian Granger
|
r1979 | pjoin = path.join | ||
Fernando Perez
|
r2494 | |||
#----------------------------------------------------------------------------- | ||||
# Globals | ||||
#----------------------------------------------------------------------------- | ||||
Fernando Perez
|
r1851 | #----------------------------------------------------------------------------- | ||
Fernando Perez
|
r2398 | # Warnings control | ||
#----------------------------------------------------------------------------- | ||||
Brian Granger
|
r2512 | |||
Fernando Perez
|
r2398 | # Twisted generates annoying warnings with Python 2.6, as will do other code | ||
# that imports 'sets' as of today | ||||
warnings.filterwarnings('ignore', 'the sets module is deprecated', | ||||
DeprecationWarning ) | ||||
Fernando Perez
|
r2461 | # This one also comes from Twisted | ||
warnings.filterwarnings('ignore', 'the sha module is deprecated', | ||||
DeprecationWarning) | ||||
Fernando Perez
|
r2483 | # Wx on Fedora11 spits these out | ||
warnings.filterwarnings('ignore', 'wxPython/wxWidgets release number mismatch', | ||||
UserWarning) | ||||
Thomas Kluyver
|
r6101 | # ------------------------------------------------------------------------------ | ||
# Monkeypatch Xunit to count known failures as skipped. | ||||
# ------------------------------------------------------------------------------ | ||||
Thomas Kluyver
|
r6102 | def monkeypatch_xunit(): | ||
Thomas Kluyver
|
r6101 | try: | ||
knownfailureif(True)(lambda: None)() | ||||
except Exception as e: | ||||
KnownFailureTest = type(e) | ||||
def addError(self, test, err, capt=None): | ||||
if issubclass(err[0], KnownFailureTest): | ||||
err = (SkipTest,) + err[1:] | ||||
return self.orig_addError(test, err, capt) | ||||
Xunit.orig_addError = Xunit.addError | ||||
Xunit.addError = addError | ||||
Fernando Perez
|
r2398 | #----------------------------------------------------------------------------- | ||
Administrator
|
r1981 | # Logic for skipping doctests | ||
Fernando Perez
|
r1851 | #----------------------------------------------------------------------------- | ||
MinRK
|
r4857 | def extract_version(mod): | ||
return mod.__version__ | ||||
Fernando Perez
|
r1851 | |||
MinRK
|
r4857 | def test_for(item, min_version=None, callback=extract_version): | ||
"""Test to see if item is importable, and optionally check against a minimum | ||||
version. | ||||
Bernardo B. Marques
|
r4872 | |||
MinRK
|
r4857 | If min_version is given, the default behavior is to check against the | ||
`__version__` attribute of the item, but specifying `callback` allows you to | ||||
extract the value you are interested in. e.g:: | ||||
Bernardo B. Marques
|
r4872 | |||
MinRK
|
r4857 | In [1]: import sys | ||
Bernardo B. Marques
|
r4872 | |||
MinRK
|
r4857 | In [2]: from IPython.testing.iptest import test_for | ||
Bernardo B. Marques
|
r4872 | |||
MinRK
|
r4857 | In [3]: test_for('sys', (2,6), callback=lambda sys: sys.version_info) | ||
Out[3]: True | ||||
Bernardo B. Marques
|
r4872 | |||
MinRK
|
r4857 | """ | ||
Administrator
|
r1981 | try: | ||
MinRK
|
r4857 | check = import_item(item) | ||
Fernando Perez
|
r2488 | except (ImportError, RuntimeError): | ||
MinRK
|
r4857 | # GTK reports Runtime error if it can't be initialized even if it's | ||
Fernando Perez
|
r2488 | # importable. | ||
Administrator
|
r1981 | return False | ||
else: | ||||
MinRK
|
r3640 | if min_version: | ||
MinRK
|
r4857 | if callback: | ||
# extra processing step to get version to compare | ||||
check = callback(check) | ||||
Bernardo B. Marques
|
r4872 | |||
MinRK
|
r4857 | return check >= min_version | ||
MinRK
|
r3640 | else: | ||
return True | ||||
Administrator
|
r1981 | |||
Fernando Perez
|
r2496 | # Global dict where we can store information on what we have and what we don't | ||
# have available at test run time | ||||
have = {} | ||||
have['curses'] = test_for('_curses') | ||||
Fernando Perez
|
r3731 | have['matplotlib'] = test_for('matplotlib') | ||
Fernando Perez
|
r7224 | have['numpy'] = test_for('numpy') | ||
MinRK
|
r4857 | have['pexpect'] = test_for('IPython.external.pexpect') | ||
Fernando Perez
|
r3731 | have['pymongo'] = test_for('pymongo') | ||
MinRK
|
r6874 | have['pygments'] = test_for('pygments') | ||
MinRK
|
r3761 | have['qt'] = test_for('IPython.external.qt') | ||
Fernando Perez
|
r7224 | have['rpy2'] = test_for('rpy2') | ||
MinRK
|
r5147 | have['sqlite3'] = test_for('sqlite3') | ||
Brian Granger
|
r7102 | have['cython'] = test_for('Cython') | ||
Stefan van der Walt
|
r7385 | have['oct2py'] = test_for('oct2py') | ||
MinRK
|
r4857 | have['tornado'] = test_for('tornado.version_info', (2,1,0), callback=None) | ||
Bradley M. Froehle
|
r8908 | have['jinja2'] = test_for('jinja2') | ||
Fernando Perez
|
r7224 | have['wx'] = test_for('wx') | ||
have['wx.aui'] = test_for('wx.aui') | ||||
Brian Granger
|
r8184 | have['azure'] = test_for('azure') | ||
Brian E. Granger
|
r11095 | have['sphinx'] = test_for('sphinx') | ||
have['markdown'] = test_for('markdown') | ||||
MinRK
|
r4857 | |||
MinRK
|
r9336 | min_zmq = (2,1,11) | ||
MinRK
|
r9339 | have['zmq'] = test_for('zmq.pyzmq_version_info', min_zmq, callback=lambda x: x()) | ||
Administrator
|
r1981 | |||
Fernando Perez
|
r2494 | #----------------------------------------------------------------------------- | ||
# Functions and classes | ||||
#----------------------------------------------------------------------------- | ||||
Administrator
|
r1980 | |||
Fernando Perez
|
r2496 | def report(): | ||
"""Return a string with a summary report of test-related variables.""" | ||||
Fernando Perez
|
r3204 | out = [ sys_info(), '\n'] | ||
Fernando Perez
|
r2496 | |||
avail = [] | ||||
not_avail = [] | ||||
Bernardo B. Marques
|
r4872 | |||
Fernando Perez
|
r2496 | for k, is_avail in have.items(): | ||
if is_avail: | ||||
avail.append(k) | ||||
else: | ||||
not_avail.append(k) | ||||
if avail: | ||||
out.append('\nTools and libraries available at test time:\n') | ||||
avail.sort() | ||||
out.append(' ' + ' '.join(avail)+'\n') | ||||
if not_avail: | ||||
out.append('\nTools and libraries NOT available at test time:\n') | ||||
not_avail.sort() | ||||
out.append(' ' + ' '.join(not_avail)+'\n') | ||||
Bernardo B. Marques
|
r4872 | |||
Fernando Perez
|
r2496 | return ''.join(out) | ||
Brian Granger
|
r2079 | def make_exclude(): | ||
Fernando Perez
|
r2454 | """Make patterns of modules and packages to exclude from testing. | ||
Brian Granger
|
r2512 | |||
Fernando Perez
|
r2454 | For the IPythonDoctest plugin, we need to exclude certain patterns that | ||
cause testing problems. We should strive to minimize the number of | ||||
Brian Granger
|
r2512 | skipped modules, since this means untested code. | ||
Fernando Perez
|
r2454 | These modules and packages will NOT get scanned by nose at all for tests. | ||
""" | ||||
# Simple utility to make IPython paths more readably, we need a lot of | ||||
# these below | ||||
ipjoin = lambda *paths: pjoin('IPython', *paths) | ||||
Bernardo B. Marques
|
r4872 | |||
Fernando Perez
|
r2454 | exclusions = [ipjoin('external'), | ||
ipjoin('quarantine'), | ||||
ipjoin('deathrow'), | ||||
Fernando Perez
|
r2461 | # This guy is probably attic material | ||
Fernando Perez
|
r2454 | ipjoin('testing', 'mkdoctests'), | ||
Fernando Perez
|
r2461 | # Testing inputhook will need a lot of thought, to figure out | ||
# how to have tests that don't lock up with the gui event | ||||
# loops in the picture | ||||
Fernando Perez
|
r2454 | ipjoin('lib', 'inputhook'), | ||
Fernando Perez
|
r2417 | # Config files aren't really importable stand-alone | ||
Fernando Perez
|
r2454 | ipjoin('config', 'profile'), | ||
Fernando Perez
|
r7801 | # The notebook 'static' directory contains JS, css and other | ||
# files for web serving. Occasionally projects may put a .py | ||||
# file in there (MathJax ships a conf.py), so we might as | ||||
# well play it safe and skip the whole thing. | ||||
MinRK
|
r11035 | ipjoin('html', 'static'), | ||
ipjoin('html', 'fabfile'), | ||||
Fernando Perez
|
r2417 | ] | ||
MinRK
|
r5147 | if not have['sqlite3']: | ||
exclusions.append(ipjoin('core', 'tests', 'test_history')) | ||||
exclusions.append(ipjoin('core', 'history')) | ||||
Fernando Perez
|
r2496 | if not have['wx']: | ||
Fernando Perez
|
r2454 | exclusions.append(ipjoin('lib', 'inputhookwx')) | ||
MinRK
|
r7455 | |||
MinRK
|
r9375 | if 'IPython.kernel.inprocess' not in sys.argv: | ||
exclusions.append(ipjoin('kernel', 'inprocess')) | ||||
MinRK
|
r7455 | # FIXME: temporarily disable autoreload tests, as they can produce | ||
# spurious failures in subsequent tests (cythonmagic). | ||||
exclusions.append(ipjoin('extensions', 'autoreload')) | ||||
exclusions.append(ipjoin('extensions', 'tests', 'test_autoreload')) | ||||
Bernardo B. Marques
|
r4872 | |||
Thomas Kluyver
|
r3458 | # We do this unconditionally, so that the test suite doesn't import | ||
# gtk, changing the default encoding and masking some unicode bugs. | ||||
exclusions.append(ipjoin('lib', 'inputhookgtk')) | ||||
MinRK
|
r9375 | exclusions.append(ipjoin('kernel', 'zmq', 'gui', 'gtkembed')) | ||
Brian Granger
|
r2083 | |||
Brian Granger
|
r2079 | # These have to be skipped on win32 because the use echo, rm, cd, etc. | ||
Thomas Kluyver
|
r3917 | # See ticket https://github.com/ipython/ipython/issues/87 | ||
Brian Granger
|
r2079 | if sys.platform == 'win32': | ||
Fernando Perez
|
r2454 | exclusions.append(ipjoin('testing', 'plugin', 'test_exampleip')) | ||
exclusions.append(ipjoin('testing', 'plugin', 'dtexample')) | ||||
Brian Granger
|
r2079 | |||
Fernando Perez
|
r2496 | if not have['pexpect']: | ||
MinRK
|
r7775 | exclusions.extend([ipjoin('lib', 'irunner'), | ||
MinRK
|
r5694 | ipjoin('lib', 'tests', 'test_irunner'), | ||
Fernando Perez
|
r11026 | ipjoin('terminal', 'console'), | ||
MinRK
|
r5694 | ]) | ||
Brian Granger
|
r2079 | |||
MinRK
|
r3640 | if not have['zmq']: | ||
MinRK
|
r9375 | exclusions.append(ipjoin('kernel')) | ||
Fernando Perez
|
r11026 | exclusions.append(ipjoin('qt')) | ||
exclusions.append(ipjoin('html')) | ||||
exclusions.append(ipjoin('consoleapp.py')) | ||||
exclusions.append(ipjoin('terminal', 'console')) | ||||
MinRK
|
r3672 | exclusions.append(ipjoin('parallel')) | ||
MinRK
|
r6874 | elif not have['qt'] or not have['pygments']: | ||
Fernando Perez
|
r11026 | exclusions.append(ipjoin('qt')) | ||
Bernardo B. Marques
|
r4872 | |||
MinRK
|
r3674 | if not have['pymongo']: | ||
exclusions.append(ipjoin('parallel', 'controller', 'mongodb')) | ||||
MinRK
|
r3875 | exclusions.append(ipjoin('parallel', 'tests', 'test_mongodb')) | ||
MinRK
|
r3640 | |||
Fernando Perez
|
r3731 | if not have['matplotlib']: | ||
Fernando Perez
|
r5474 | exclusions.extend([ipjoin('core', 'pylabtools'), | ||
MinRK
|
r6566 | ipjoin('core', 'tests', 'test_pylabtools'), | ||
MinRK
|
r9375 | ipjoin('kernel', 'zmq', 'pylab'), | ||
MinRK
|
r6566 | ]) | ||
Fernando Perez
|
r3731 | |||
Brian Granger
|
r7102 | if not have['cython']: | ||
exclusions.extend([ipjoin('extensions', 'cythonmagic')]) | ||||
MinRK
|
r7117 | exclusions.extend([ipjoin('extensions', 'tests', 'test_cythonmagic')]) | ||
Brian Granger
|
r7102 | |||
Stefan van der Walt
|
r7385 | if not have['oct2py']: | ||
exclusions.extend([ipjoin('extensions', 'octavemagic')]) | ||||
exclusions.extend([ipjoin('extensions', 'tests', 'test_octavemagic')]) | ||||
Brian E. Granger
|
r4703 | if not have['tornado']: | ||
Fernando Perez
|
r11026 | exclusions.append(ipjoin('html')) | ||
Brian E. Granger
|
r4703 | |||
Bradley M. Froehle
|
r8908 | if not have['jinja2']: | ||
MinRK
|
r11035 | exclusions.append(ipjoin('html', 'notebookapp')) | ||
Bradley M. Froehle
|
r8908 | |||
Fernando Perez
|
r7224 | if not have['rpy2'] or not have['numpy']: | ||
exclusions.append(ipjoin('extensions', 'rmagic')) | ||||
exclusions.append(ipjoin('extensions', 'tests', 'test_rmagic')) | ||||
Brian Granger
|
r8184 | if not have['azure']: | ||
MinRK
|
r11035 | exclusions.append(ipjoin('html', 'services', 'notebooks', 'azurenbmanager')) | ||
Brian Granger
|
r8184 | |||
Brian E. Granger
|
r11095 | if not all((have['pygments'], have['jinja2'], have['markdown'], have['sphinx'])): | ||
exclusions.append(ipjoin('nbconvert')) | ||||
Brian Granger
|
r2079 | # This is needed for the reg-exp to match on win32 in the ipdoctest plugin. | ||
if sys.platform == 'win32': | ||||
Fernando Perez
|
r2414 | exclusions = [s.replace('\\','\\\\') for s in exclusions] | ||
MinRK
|
r7121 | |||
# check for any exclusions that don't seem to exist: | ||||
MinRK
|
r7126 | parent, _ = os.path.split(get_ipython_package_dir()) | ||
MinRK
|
r7121 | for exclusion in exclusions: | ||
MinRK
|
r7463 | if exclusion.endswith(('deathrow', 'quarantine')): | ||
# ignore deathrow/quarantine, which exist in dev, but not install | ||||
continue | ||||
MinRK
|
r7126 | fullpath = pjoin(parent, exclusion) | ||
MinRK
|
r7412 | if not os.path.exists(fullpath) and not glob.glob(fullpath + '.*'): | ||
Thomas Kluyver
|
r8223 | warn("Excluding nonexistent file: %r" % exclusion) | ||
Brian Granger
|
r2079 | |||
Fernando Perez
|
r2414 | return exclusions | ||
Administrator
|
r1980 | |||
Fernando Perez
|
r2398 | class IPTester(object): | ||
"""Call that calls iptest or trial in a subprocess. | ||||
""" | ||||
Fernando Perez
|
r2399 | #: string, name of test runner that will be called | ||
runner = None | ||||
#: list, parameters for test runner | ||||
params = None | ||||
#: list, arguments of system call to be made to call test runner | ||||
call_args = None | ||||
Bradley M. Froehle
|
r8035 | #: list, subprocesses we start (for cleanup) | ||
processes = None | ||||
Thomas Kluyver
|
r5983 | #: str, coverage xml output file | ||
coverage_xml = None | ||||
Bernardo B. Marques
|
r4872 | |||
Fernando Perez
|
r2480 | def __init__(self, runner='iptest', params=None): | ||
Fernando Perez
|
r2399 | """Create new test runner.""" | ||
Fernando Perez
|
r2494 | p = os.path | ||
Fernando Perez
|
r2398 | if runner == 'iptest': | ||
Victor Zverovich
|
r11055 | iptest_app = os.path.abspath(get_ipython_module_path('IPython.testing.iptest')) | ||
Brian Granger
|
r2507 | self.runner = pycmd2argv(iptest_app) + sys.argv[1:] | ||
Brian Granger
|
r2512 | else: | ||
raise Exception('Not a valid test runner: %s' % repr(runner)) | ||||
Fernando Perez
|
r2398 | if params is None: | ||
params = [] | ||||
Fernando Perez
|
r2480 | if isinstance(params, str): | ||
Fernando Perez
|
r2398 | params = [params] | ||
self.params = params | ||||
# Assemble call | ||||
self.call_args = self.runner+self.params | ||||
Thomas Kluyver
|
r5962 | |||
Thomas Kluyver
|
r5987 | # Find the section we're testing (IPython.foo) | ||
for sect in self.params: | ||||
Thomas Kluyver
|
r11073 | if sect.startswith('IPython') or sect in special_test_suites: break | ||
Thomas Kluyver
|
r5987 | else: | ||
raise ValueError("Section not found", self.params) | ||||
Thomas Kluyver
|
r5962 | if '--with-xunit' in self.call_args: | ||
MinRK
|
r7744 | |||
self.call_args.append('--xunit-file') | ||||
# FIXME: when Windows uses subprocess.call, these extra quotes are unnecessary: | ||||
xunit_file = path.abspath(sect+'.xunit.xml') | ||||
if sys.platform == 'win32': | ||||
xunit_file = '"%s"' % xunit_file | ||||
self.call_args.append(xunit_file) | ||||
Thomas Kluyver
|
r5983 | |||
Thomas Kluyver
|
r5985 | if '--with-xml-coverage' in self.call_args: | ||
Thomas Kluyver
|
r5983 | self.coverage_xml = path.abspath(sect+".coverage.xml") | ||
Thomas Kluyver
|
r5985 | self.call_args.remove('--with-xml-coverage') | ||
Thomas Kluyver
|
r5984 | self.call_args = ["coverage", "run", "--source="+sect] + self.call_args[1:] | ||
Fernando Perez
|
r2398 | |||
Bradley M. Froehle
|
r8035 | # Store anything we start to clean up on deletion | ||
self.processes = [] | ||||
Fernando Perez
|
r2399 | |||
Bradley M. Froehle
|
r7870 | def _run_cmd(self): | ||
Bradley M. Froehle
|
r8038 | with TemporaryDirectory() as IPYTHONDIR: | ||
env = os.environ.copy() | ||||
env['IPYTHONDIR'] = IPYTHONDIR | ||||
# print >> sys.stderr, '*** CMD:', ' '.join(self.call_args) # dbg | ||||
subp = subprocess.Popen(self.call_args, env=env) | ||||
self.processes.append(subp) | ||||
# If this fails, the process will be left in self.processes and | ||||
# cleaned up later, but if the wait call succeeds, then we can | ||||
# clear the stored process. | ||||
retcode = subp.wait() | ||||
self.processes.pop() | ||||
return retcode | ||||
Bernardo B. Marques
|
r4872 | |||
Fernando Perez
|
r2398 | def run(self): | ||
"""Run the stored commands""" | ||||
try: | ||||
Thomas Kluyver
|
r5983 | retcode = self._run_cmd() | ||
MinRK
|
r8495 | except KeyboardInterrupt: | ||
return -signal.SIGINT | ||||
Fernando Perez
|
r2398 | except: | ||
import traceback | ||||
traceback.print_exc() | ||||
return 1 # signal failure | ||||
Thomas Kluyver
|
r5983 | |||
if self.coverage_xml: | ||||
Thomas Kluyver
|
r5986 | subprocess.call(["coverage", "xml", "-o", self.coverage_xml]) | ||
Thomas Kluyver
|
r5983 | return retcode | ||
Fernando Perez
|
r2398 | |||
Fernando Perez
|
r2399 | def __del__(self): | ||
"""Cleanup on exit by killing any leftover processes.""" | ||||
Bradley M. Froehle
|
r8035 | for subp in self.processes: | ||
Bradley M. Froehle
|
r8036 | if subp.poll() is not None: | ||
continue # process is already dead | ||||
Fernando Perez
|
r2399 | try: | ||
MinRK
|
r8495 | print('Cleaning up stale PID: %d' % subp.pid) | ||
Bradley M. Froehle
|
r8035 | subp.kill() | ||
except: # (OSError, WindowsError) ? | ||||
Fernando Perez
|
r2399 | # This is just a best effort, if we fail or the process was | ||
# really gone, ignore it. | ||||
Bernardo B. Marques
|
r4872 | pass | ||
MinRK
|
r8495 | else: | ||
for i in range(10): | ||||
if subp.poll() is None: | ||||
time.sleep(0.1) | ||||
else: | ||||
break | ||||
Fernando Perez
|
r2400 | |||
Bradley M. Froehle
|
r8036 | if subp.poll() is None: | ||
# The process did not die... | ||||
MinRK
|
r8495 | print('... failed. Manual cleanup may be required.') | ||
Fernando Perez
|
r2398 | |||
Thomas Kluyver
|
r11073 | |||
special_test_suites = { | ||||
'autoreload': ['IPython.extensions.autoreload', 'IPython.extensions.tests.test_autoreload'], | ||||
} | ||||
Fernando Perez
|
r11026 | |||
Thomas Kluyver
|
r8124 | def make_runners(inc_slow=False): | ||
Fernando Perez
|
r2398 | """Define the top-level packages that need to be tested. | ||
""" | ||||
Fernando Perez
|
r2483 | # Packages to be tested via nose, that only depend on the stdlib | ||
Fernando Perez
|
r11026 | nose_pkg_names = ['config', 'core', 'extensions', 'lib', 'terminal', | ||
Brian E. Granger
|
r11095 | 'testing', 'utils', 'nbformat'] | ||
Bernardo B. Marques
|
r4872 | |||
Fernando Perez
|
r11026 | if have['qt']: | ||
nose_pkg_names.append('qt') | ||||
if have['tornado']: | ||||
nose_pkg_names.append('html') | ||||
MinRK
|
r3673 | if have['zmq']: | ||
MinRK
|
r9375 | nose_pkg_names.append('kernel') | ||
nose_pkg_names.append('kernel.inprocess') | ||||
Thomas Kluyver
|
r8124 | if inc_slow: | ||
nose_pkg_names.append('parallel') | ||||
Bernardo B. Marques
|
r4872 | |||
Brian E. Granger
|
r11095 | if all((have['pygments'], have['jinja2'], have['markdown'], have['sphinx'])): | ||
nose_pkg_names.append('nbconvert') | ||||
Fernando Perez
|
r2483 | # For debugging this code, only load quick stuff | ||
Fernando Perez
|
r2494 | #nose_pkg_names = ['core', 'extensions'] # dbg | ||
Fernando Perez
|
r2483 | |||
# Make fully qualified package names prepending 'IPython.' to our name lists | ||||
nose_packages = ['IPython.%s' % m for m in nose_pkg_names ] | ||||
# Make runners | ||||
runners = [ (v, IPTester('iptest', params=v)) for v in nose_packages ] | ||||
Thomas Kluyver
|
r11073 | |||
for name in special_test_suites: | ||||
runners.append((name, IPTester('iptest', params=name))) | ||||
Bernardo B. Marques
|
r4872 | |||
Fernando Perez
|
r2398 | return runners | ||
Brian Granger
|
r1972 | def run_iptest(): | ||
"""Run the IPython test suite using nose. | ||||
Bernardo B. Marques
|
r4872 | |||
Brian Granger
|
r1972 | This function is called when this script is **not** called with the form | ||
`iptest all`. It simply calls nose with appropriate command line flags | ||||
and accepts all of the standard nose arguments. | ||||
Fernando Perez
|
r1574 | """ | ||
Thomas Kluyver
|
r6102 | # Apply our monkeypatch to Xunit | ||
Thomas Kluyver
|
r6113 | if '--with-xunit' in sys.argv and not hasattr(Xunit, 'orig_addError'): | ||
Thomas Kluyver
|
r6102 | monkeypatch_xunit() | ||
Fernando Perez
|
r1574 | |||
Bernardo B. Marques
|
r4872 | warnings.filterwarnings('ignore', | ||
Fernando Perez
|
r1574 | 'This will be removed soon. Use IPython.testing.util instead') | ||
Thomas Kluyver
|
r11073 | |||
if sys.argv[1] in special_test_suites: | ||||
sys.argv[1:2] = special_test_suites[sys.argv[1]] | ||||
special_suite = True | ||||
else: | ||||
special_suite = False | ||||
Fernando Perez
|
r1574 | |||
Fernando Perez
|
r2487 | argv = sys.argv + [ '--detailed-errors', # extra info in tracebacks | ||
Bernardo B. Marques
|
r4872 | |||
Fernando Perez
|
r2414 | '--with-ipdoctest', | ||
'--ipdoctest-tests','--ipdoctest-extension=txt', | ||||
Bernardo B. Marques
|
r4872 | |||
Fernando Perez
|
r1761 | # We add --exe because of setuptools' imbecility (it | ||
# blindly does chmod +x on ALL files). Nose does the | ||||
# right thing and it tries to avoid executables, | ||||
# setuptools unfortunately forces our hand here. This | ||||
# has been discussed on the distutils list and the | ||||
# setuptools devs refuse to fix this problem! | ||||
'--exe', | ||||
] | ||||
MinRK
|
r8201 | if '-a' not in argv and '-A' not in argv: | ||
argv = argv + ['-a', '!crash'] | ||||
Fernando Perez
|
r1574 | |||
Fernando Perez
|
r2494 | if nose.__version__ >= '0.11': | ||
# I don't fully understand why we need this one, but depending on what | ||||
# directory the test suite is run from, if we don't give it, 0 tests | ||||
# get run. Specifically, if the test suite is run from the source dir | ||||
# with an argument (like 'iptest.py IPython.core', 0 tests are run, | ||||
# even if the same call done in this directory works fine). It appears | ||||
# that if the requested package is in the current dir, nose bails early | ||||
# by default. Since it's otherwise harmless, leave it in by default | ||||
# for nose >= 0.11, though unfortunately nose 0.10 doesn't support it. | ||||
argv.append('--traverse-namespace') | ||||
Fernando Perez
|
r1574 | |||
Matthew Brett
|
r4567 | # use our plugin for doctesting. It will remove the standard doctest plugin | ||
# if it finds it enabled | ||||
Thomas Kluyver
|
r11073 | ipdt = IPythonDoctest() if special_suite else IPythonDoctest(make_exclude()) | ||
plugins = [ipdt, KnownFailure()] | ||||
Fernando Perez
|
r8489 | |||
# We need a global ipython running in this process, but the special | ||||
# in-process group spawns its own IPython kernels, so for *that* group we | ||||
# must avoid also opening the global one (otherwise there's a conflict of | ||||
# singletons). Ultimately the solution to this problem is to refactor our | ||||
# assumptions about what needs to be a singleton and what doesn't (app | ||||
# objects should, individual shells shouldn't). But for now, this | ||||
# workaround allows the test suite for the inprocess module to complete. | ||||
MinRK
|
r9375 | if not 'IPython.kernel.inprocess' in sys.argv: | ||
Fernando Perez
|
r8489 | globalipapp.start_ipython() | ||
Fernando Perez
|
r2414 | # Now nose can run | ||
Matthew Brett
|
r4567 | TestProgram(argv=argv, addplugins=plugins) | ||
Brian Granger
|
r1972 | |||
Thomas Kluyver
|
r8124 | def run_iptestall(inc_slow=False): | ||
Brian Granger
|
r1972 | """Run the entire IPython test suite by calling nose and trial. | ||
Bernardo B. Marques
|
r4872 | |||
Brian Granger
|
r1972 | This function constructs :class:`IPTester` instances for all IPython | ||
modules and package and then runs each of them. This causes the modules | ||||
and packages of IPython to be tested each in their own subprocess using | ||||
MinRK
|
r6421 | nose. | ||
Thomas Kluyver
|
r8124 | |||
Parameters | ||||
---------- | ||||
inc_slow : bool, optional | ||||
Include slow tests, like IPython.parallel. By default, these tests aren't | ||||
run. | ||||
Brian Granger
|
r1972 | """ | ||
Brian Granger
|
r2079 | |||
Thomas Kluyver
|
r8124 | runners = make_runners(inc_slow=inc_slow) | ||
Brian Granger
|
r2079 | |||
Fernando Perez
|
r2398 | # Run the test runners in a temporary dir so we can nuke it when finished | ||
# to clean up any junk files left over by accident. This also makes it | ||||
# robust against being run in non-writeable directories by mistake, as the | ||||
# temp dir will always be user-writeable. | ||||
Jörgen Stenarson
|
r4208 | curdir = os.getcwdu() | ||
Fernando Perez
|
r2398 | testdir = tempfile.gettempdir() | ||
os.chdir(testdir) | ||||
Brian Granger
|
r1972 | # Run all test runners, tracking execution time | ||
Fernando Perez
|
r2480 | failed = [] | ||
Brian Granger
|
r1972 | t_start = time.time() | ||
Fernando Perez
|
r2398 | try: | ||
Fernando Perez
|
r2480 | for (name, runner) in runners: | ||
Matthias BUSSONNIER
|
r7817 | print('*'*70) | ||
print('IPython test group:',name) | ||||
Fernando Perez
|
r2398 | res = runner.run() | ||
if res: | ||||
Fernando Perez
|
r2480 | failed.append( (name, runner) ) | ||
MinRK
|
r8495 | if res == -signal.SIGINT: | ||
print("Interrupted") | ||||
break | ||||
Fernando Perez
|
r2398 | finally: | ||
os.chdir(curdir) | ||||
Brian Granger
|
r1972 | t_end = time.time() | ||
t_tests = t_end - t_start | ||||
nrunners = len(runners) | ||||
nfail = len(failed) | ||||
# summarize results | ||||
Matthias BUSSONNIER
|
r7817 | print() | ||
print('*'*70) | ||||
print('Test suite completed for system with the following information:') | ||||
print(report()) | ||||
print('Ran %s test groups in %.3fs' % (nrunners, t_tests)) | ||||
print() | ||||
print('Status:') | ||||
Brian Granger
|
r1972 | if not failed: | ||
Matthias BUSSONNIER
|
r7817 | print('OK') | ||
Brian Granger
|
r1972 | else: | ||
# If anything went wrong, point out what command to rerun manually to | ||||
# see the actual errors and individual summary | ||||
Matthias BUSSONNIER
|
r7817 | print('ERROR - %s out of %s test groups failed.' % (nfail, nrunners)) | ||
Fernando Perez
|
r2480 | for name, failed_runner in failed: | ||
Matthias BUSSONNIER
|
r7817 | print('-'*40) | ||
print('Runner failed:',name) | ||||
print('You may wish to rerun this one individually, with:') | ||||
Jörgen Stenarson
|
r7732 | failed_call_args = [py3compat.cast_unicode(x) for x in failed_runner.call_args] | ||
Matthias BUSSONNIER
|
r7817 | print(u' '.join(failed_call_args)) | ||
print() | ||||
Thomas Kluyver
|
r3895 | # Ensure that our exit code indicates failure | ||
sys.exit(1) | ||||
Brian Granger
|
r1972 | |||
def main(): | ||||
Fernando Perez
|
r2480 | for arg in sys.argv[1:]: | ||
Thomas Kluyver
|
r11073 | if arg.startswith('IPython') or arg in special_test_suites: | ||
Fernando Perez
|
r2481 | # This is in-process | ||
Brian Granger
|
r1990 | run_iptest() | ||
Fernando Perez
|
r2480 | else: | ||
Thomas Kluyver
|
r8124 | if "--all" in sys.argv: | ||
sys.argv.remove("--all") | ||||
inc_slow = True | ||||
else: | ||||
inc_slow = False | ||||
Fernando Perez
|
r2481 | # This starts subprocesses | ||
Thomas Kluyver
|
r8124 | run_iptestall(inc_slow=inc_slow) | ||
Brian Granger
|
r1972 | |||
if __name__ == '__main__': | ||||
Fernando Perez
|
r2091 | main() | ||