iptest.py
501 lines
| 18.0 KiB
| text/x-python
|
PythonLexer
|
r1574 | # -*- coding: utf-8 -*- | ||
"""IPython Test Suite Runner. | ||||
|
r1851 | |||
|
r1972 | This module provides a main entry point to a user script to test IPython | ||
itself from the command line. There are two ways of running this script: | ||||
1. With the syntax `iptest all`. This runs our entire test suite by | ||||
|
r3486 | calling this script (with different arguments) recursively. This | ||
|
r1972 | causes modules and package to be tested in different processes, using nose | ||
or trial where appropriate. | ||||
2. With the regular nose syntax, like `iptest -vvs IPython`. In this form | ||||
the script simply calls nose, but with special command line flags and | ||||
plugins loaded. | ||||
|
r1574 | """ | ||
|
r1851 | #----------------------------------------------------------------------------- | ||
|
r5390 | # Copyright (C) 2009-2011 The IPython Development Team | ||
|
r2498 | # | ||
# Distributed under the terms of the BSD License. The full license is in | ||||
# the file COPYING, distributed as part of this software. | ||||
#----------------------------------------------------------------------------- | ||||
#----------------------------------------------------------------------------- | ||||
# Imports | ||||
|
r1851 | #----------------------------------------------------------------------------- | ||
|
r2442 | # Stdlib | ||
|
r1972 | import os | ||
import os.path as path | ||||
|
r2399 | import signal | ||
|
r1574 | import sys | ||
|
r1972 | import subprocess | ||
|
r2111 | import tempfile | ||
|
r1972 | import time | ||
|
r1574 | import warnings | ||
|
r2442 | # Note: monkeypatch! | ||
# We need to monkeypatch a small problem in nose itself first, before importing | ||||
# it for actual use. This should get into nose upstream, but its release cycle | ||||
# is slow and we need it for our parametric tests to work correctly. | ||||
|
r2480 | from IPython.testing import nosepatch | ||
|
r2442 | # Now, proceed to import nose itself | ||
|
r1574 | import nose.plugins.builtin | ||
|
r1851 | from nose.core import TestProgram | ||
|
r1574 | |||
|
r2442 | # Our own imports | ||
|
r4857 | from IPython.utils.importstring import import_item | ||
|
r2498 | from IPython.utils.path import get_ipython_module_path | ||
from IPython.utils.process import find_cmd, pycmd2argv | ||||
from IPython.utils.sysinfo import sys_info | ||||
|
r2480 | from IPython.testing import globalipapp | ||
from IPython.testing.plugin.ipdoctest import IPythonDoctest | ||||
|
r3511 | from IPython.external.decorators import KnownFailure | ||
|
r1574 | |||
|
r1979 | pjoin = path.join | ||
|
r2494 | |||
#----------------------------------------------------------------------------- | ||||
# Globals | ||||
#----------------------------------------------------------------------------- | ||||
|
r1851 | #----------------------------------------------------------------------------- | ||
|
r2398 | # Warnings control | ||
#----------------------------------------------------------------------------- | ||||
|
r2512 | |||
|
r2398 | # Twisted generates annoying warnings with Python 2.6, as will do other code | ||
# that imports 'sets' as of today | ||||
warnings.filterwarnings('ignore', 'the sets module is deprecated', | ||||
DeprecationWarning ) | ||||
|
r2461 | # This one also comes from Twisted | ||
warnings.filterwarnings('ignore', 'the sha module is deprecated', | ||||
DeprecationWarning) | ||||
|
r2483 | # Wx on Fedora11 spits these out | ||
warnings.filterwarnings('ignore', 'wxPython/wxWidgets release number mismatch', | ||||
UserWarning) | ||||
|
r2398 | #----------------------------------------------------------------------------- | ||
|
r1981 | # Logic for skipping doctests | ||
|
r1851 | #----------------------------------------------------------------------------- | ||
|
r4857 | def extract_version(mod): | ||
return mod.__version__ | ||||
|
r1851 | |||
|
r4857 | def test_for(item, min_version=None, callback=extract_version): | ||
"""Test to see if item is importable, and optionally check against a minimum | ||||
version. | ||||
|
r4872 | |||
|
r4857 | If min_version is given, the default behavior is to check against the | ||
`__version__` attribute of the item, but specifying `callback` allows you to | ||||
extract the value you are interested in. e.g:: | ||||
|
r4872 | |||
|
r4857 | In [1]: import sys | ||
|
r4872 | |||
|
r4857 | In [2]: from IPython.testing.iptest import test_for | ||
|
r4872 | |||
|
r4857 | In [3]: test_for('sys', (2,6), callback=lambda sys: sys.version_info) | ||
Out[3]: True | ||||
|
r4872 | |||
|
r4857 | """ | ||
|
r1981 | try: | ||
|
r4857 | check = import_item(item) | ||
|
r2488 | except (ImportError, RuntimeError): | ||
|
r4857 | # GTK reports Runtime error if it can't be initialized even if it's | ||
|
r2488 | # importable. | ||
|
r1981 | return False | ||
else: | ||||
|
r3640 | if min_version: | ||
|
r4857 | if callback: | ||
# extra processing step to get version to compare | ||||
check = callback(check) | ||||
|
r4872 | |||
|
r4857 | return check >= min_version | ||
|
r3640 | else: | ||
return True | ||||
|
r1981 | |||
|
r2496 | # Global dict where we can store information on what we have and what we don't | ||
# have available at test run time | ||||
have = {} | ||||
have['curses'] = test_for('_curses') | ||||
|
r3731 | have['matplotlib'] = test_for('matplotlib') | ||
|
r4857 | have['pexpect'] = test_for('IPython.external.pexpect') | ||
|
r3731 | have['pymongo'] = test_for('pymongo') | ||
|
r2496 | have['wx'] = test_for('wx') | ||
have['wx.aui'] = test_for('wx.aui') | ||||
|
r3761 | have['qt'] = test_for('IPython.external.qt') | ||
|
r5147 | have['sqlite3'] = test_for('sqlite3') | ||
|
r4856 | |||
|
r4857 | have['tornado'] = test_for('tornado.version_info', (2,1,0), callback=None) | ||
if os.name == 'nt': | ||||
min_zmq = (2,1,7) | ||||
|
r4856 | else: | ||
|
r4857 | min_zmq = (2,1,4) | ||
def version_tuple(mod): | ||||
"turn '2.1.9' into (2,1,9), and '2.1dev' into (2,1,999)" | ||||
# turn 'dev' into 999, because Python3 rejects str-int comparisons | ||||
vs = mod.__version__.replace('dev', '.999') | ||||
tup = tuple([int(v) for v in vs.split('.') ]) | ||||
return tup | ||||
have['zmq'] = test_for('zmq', min_zmq, version_tuple) | ||||
|
r1981 | |||
|
r2494 | #----------------------------------------------------------------------------- | ||
# Functions and classes | ||||
#----------------------------------------------------------------------------- | ||||
|
r1980 | |||
|
r2496 | def report(): | ||
"""Return a string with a summary report of test-related variables.""" | ||||
|
r3204 | out = [ sys_info(), '\n'] | ||
|
r2496 | |||
avail = [] | ||||
not_avail = [] | ||||
|
r4872 | |||
|
r2496 | for k, is_avail in have.items(): | ||
if is_avail: | ||||
avail.append(k) | ||||
else: | ||||
not_avail.append(k) | ||||
if avail: | ||||
out.append('\nTools and libraries available at test time:\n') | ||||
avail.sort() | ||||
out.append(' ' + ' '.join(avail)+'\n') | ||||
if not_avail: | ||||
out.append('\nTools and libraries NOT available at test time:\n') | ||||
not_avail.sort() | ||||
out.append(' ' + ' '.join(not_avail)+'\n') | ||||
|
r4872 | |||
|
r2496 | return ''.join(out) | ||
|
r2079 | def make_exclude(): | ||
|
r2454 | """Make patterns of modules and packages to exclude from testing. | ||
|
r2512 | |||
|
r2454 | For the IPythonDoctest plugin, we need to exclude certain patterns that | ||
cause testing problems. We should strive to minimize the number of | ||||
|
r2512 | skipped modules, since this means untested code. | ||
|
r2454 | These modules and packages will NOT get scanned by nose at all for tests. | ||
""" | ||||
# Simple utility to make IPython paths more readably, we need a lot of | ||||
# these below | ||||
ipjoin = lambda *paths: pjoin('IPython', *paths) | ||||
|
r4872 | |||
|
r2454 | exclusions = [ipjoin('external'), | ||
|
r2417 | pjoin('IPython_doctest_plugin'), | ||
|
r2454 | ipjoin('quarantine'), | ||
ipjoin('deathrow'), | ||||
ipjoin('testing', 'attic'), | ||||
|
r2461 | # This guy is probably attic material | ||
|
r2454 | ipjoin('testing', 'mkdoctests'), | ||
|
r2461 | # Testing inputhook will need a lot of thought, to figure out | ||
# how to have tests that don't lock up with the gui event | ||||
# loops in the picture | ||||
|
r2454 | ipjoin('lib', 'inputhook'), | ||
|
r2417 | # Config files aren't really importable stand-alone | ||
|
r2454 | ipjoin('config', 'default'), | ||
ipjoin('config', 'profile'), | ||||
|
r2417 | ] | ||
|
r5147 | if not have['sqlite3']: | ||
exclusions.append(ipjoin('core', 'tests', 'test_history')) | ||||
exclusions.append(ipjoin('core', 'history')) | ||||
|
r2496 | if not have['wx']: | ||
|
r2454 | exclusions.append(ipjoin('lib', 'inputhookwx')) | ||
|
r4872 | |||
|
r3458 | # We do this unconditionally, so that the test suite doesn't import | ||
# gtk, changing the default encoding and masking some unicode bugs. | ||||
exclusions.append(ipjoin('lib', 'inputhookgtk')) | ||||
|
r2083 | |||
|
r2079 | # These have to be skipped on win32 because the use echo, rm, cd, etc. | ||
|
r3917 | # See ticket https://github.com/ipython/ipython/issues/87 | ||
|
r2079 | if sys.platform == 'win32': | ||
|
r2454 | exclusions.append(ipjoin('testing', 'plugin', 'test_exampleip')) | ||
exclusions.append(ipjoin('testing', 'plugin', 'dtexample')) | ||||
|
r2079 | |||
|
r2496 | if not have['pexpect']: | ||
|
r2455 | exclusions.extend([ipjoin('scripts', 'irunner'), | ||
|
r3761 | ipjoin('lib', 'irunner'), | ||
|
r5694 | ipjoin('lib', 'tests', 'test_irunner'), | ||
ipjoin('frontend', 'terminal', 'console'), | ||||
]) | ||||
|
r2079 | |||
|
r3640 | if not have['zmq']: | ||
exclusions.append(ipjoin('zmq')) | ||||
|
r3761 | exclusions.append(ipjoin('frontend', 'qt')) | ||
|
r5694 | exclusions.append(ipjoin('frontend', 'html')) | ||
exclusions.append(ipjoin('frontend', 'consoleapp.py')) | ||||
exclusions.append(ipjoin('frontend', 'terminal', 'console')) | ||||
|
r3672 | exclusions.append(ipjoin('parallel')) | ||
|
r3761 | elif not have['qt']: | ||
exclusions.append(ipjoin('frontend', 'qt')) | ||||
|
r4872 | |||
|
r3674 | if not have['pymongo']: | ||
exclusions.append(ipjoin('parallel', 'controller', 'mongodb')) | ||||
|
r3875 | exclusions.append(ipjoin('parallel', 'tests', 'test_mongodb')) | ||
|
r3640 | |||
|
r3731 | if not have['matplotlib']: | ||
|
r5474 | exclusions.extend([ipjoin('core', 'pylabtools'), | ||
ipjoin('core', 'tests', 'test_pylabtools')]) | ||||
|
r3731 | |||
|
r4703 | if not have['tornado']: | ||
exclusions.append(ipjoin('frontend', 'html')) | ||||
|
r2079 | # This is needed for the reg-exp to match on win32 in the ipdoctest plugin. | ||
if sys.platform == 'win32': | ||||
|
r2414 | exclusions = [s.replace('\\','\\\\') for s in exclusions] | ||
|
r2079 | |||
|
r2414 | return exclusions | ||
|
r1980 | |||
|
r2398 | class IPTester(object): | ||
"""Call that calls iptest or trial in a subprocess. | ||||
""" | ||||
|
r2399 | #: string, name of test runner that will be called | ||
runner = None | ||||
#: list, parameters for test runner | ||||
params = None | ||||
#: list, arguments of system call to be made to call test runner | ||||
call_args = None | ||||
#: list, process ids of subprocesses we start (for cleanup) | ||||
pids = None | ||||
|
r5983 | #: str, coverage xml output file | ||
coverage_xml = None | ||||
|
r4872 | |||
|
r2480 | def __init__(self, runner='iptest', params=None): | ||
|
r2399 | """Create new test runner.""" | ||
|
r2494 | p = os.path | ||
|
r2398 | if runner == 'iptest': | ||
|
r2507 | iptest_app = get_ipython_module_path('IPython.testing.iptest') | ||
self.runner = pycmd2argv(iptest_app) + sys.argv[1:] | ||||
|
r2512 | else: | ||
raise Exception('Not a valid test runner: %s' % repr(runner)) | ||||
|
r2398 | if params is None: | ||
params = [] | ||||
|
r2480 | if isinstance(params, str): | ||
|
r2398 | params = [params] | ||
self.params = params | ||||
# Assemble call | ||||
self.call_args = self.runner+self.params | ||||
|
r5962 | |||
|
r5987 | # Find the section we're testing (IPython.foo) | ||
for sect in self.params: | ||||
if sect.startswith('IPython'): break | ||||
else: | ||||
raise ValueError("Section not found", self.params) | ||||
|
r5962 | if '--with-xunit' in self.call_args: | ||
self.call_args.append('--xunit-file=%s' % path.abspath(sect+'.xunit.xml')) | ||||
|
r5983 | |||
|
r5985 | if '--with-xml-coverage' in self.call_args: | ||
|
r5983 | self.coverage_xml = path.abspath(sect+".coverage.xml") | ||
|
r5985 | self.call_args.remove('--with-xml-coverage') | ||
|
r5984 | self.call_args = ["coverage", "run", "--source="+sect] + self.call_args[1:] | ||
|
r2398 | |||
|
r2399 | # Store pids of anything we start to clean up on deletion, if possible | ||
# (on posix only, since win32 has no os.kill) | ||||
self.pids = [] | ||||
|
r2398 | if sys.platform == 'win32': | ||
def _run_cmd(self): | ||||
# On Windows, use os.system instead of subprocess.call, because I | ||||
# was having problems with subprocess and I just don't know enough | ||||
# about win32 to debug this reliably. Os.system may be the 'old | ||||
# fashioned' way to do it, but it works just fine. If someone | ||||
# later can clean this up that's fine, as long as the tests run | ||||
# reliably in win32. | ||||
|
r2512 | # What types of problems are you having. They may be related to | ||
# running Python in unboffered mode. BG. | ||||
|
r2398 | return os.system(' '.join(self.call_args)) | ||
else: | ||||
def _run_cmd(self): | ||||
|
r2781 | # print >> sys.stderr, '*** CMD:', ' '.join(self.call_args) # dbg | ||
|
r2399 | subp = subprocess.Popen(self.call_args) | ||
self.pids.append(subp.pid) | ||||
# If this fails, the pid will be left in self.pids and cleaned up | ||||
# later, but if the wait call succeeds, then we can clear the | ||||
# stored pid. | ||||
retcode = subp.wait() | ||||
self.pids.pop() | ||||
return retcode | ||||
|
r4872 | |||
|
r2398 | def run(self): | ||
"""Run the stored commands""" | ||||
try: | ||||
|
r5983 | retcode = self._run_cmd() | ||
|
r2398 | except: | ||
import traceback | ||||
traceback.print_exc() | ||||
return 1 # signal failure | ||||
|
r5983 | |||
if self.coverage_xml: | ||||
|
r5986 | subprocess.call(["coverage", "xml", "-o", self.coverage_xml]) | ||
|
r5983 | return retcode | ||
|
r2398 | |||
|
r2399 | def __del__(self): | ||
"""Cleanup on exit by killing any leftover processes.""" | ||||
if not hasattr(os, 'kill'): | ||||
return | ||||
|
r4872 | |||
|
r2399 | for pid in self.pids: | ||
try: | ||||
print 'Cleaning stale PID:', pid | ||||
os.kill(pid, signal.SIGKILL) | ||||
except OSError: | ||||
# This is just a best effort, if we fail or the process was | ||||
# really gone, ignore it. | ||||
|
r4872 | pass | ||
|
r2400 | |||
|
r2398 | |||
def make_runners(): | ||||
"""Define the top-level packages that need to be tested. | ||||
""" | ||||
|
r2483 | # Packages to be tested via nose, that only depend on the stdlib | ||
nose_pkg_names = ['config', 'core', 'extensions', 'frontend', 'lib', | ||||
|
r4402 | 'scripts', 'testing', 'utils', 'nbformat' ] | ||
|
r4872 | |||
|
r3673 | if have['zmq']: | ||
nose_pkg_names.append('parallel') | ||||
|
r4872 | |||
|
r2483 | # For debugging this code, only load quick stuff | ||
|
r2494 | #nose_pkg_names = ['core', 'extensions'] # dbg | ||
|
r2483 | |||
# Make fully qualified package names prepending 'IPython.' to our name lists | ||||
nose_packages = ['IPython.%s' % m for m in nose_pkg_names ] | ||||
# Make runners | ||||
runners = [ (v, IPTester('iptest', params=v)) for v in nose_packages ] | ||||
|
r4872 | |||
|
r2398 | return runners | ||
|
r1972 | def run_iptest(): | ||
"""Run the IPython test suite using nose. | ||||
|
r4872 | |||
|
r1972 | This function is called when this script is **not** called with the form | ||
`iptest all`. It simply calls nose with appropriate command line flags | ||||
and accepts all of the standard nose arguments. | ||||
|
r1574 | """ | ||
|
r4872 | warnings.filterwarnings('ignore', | ||
|
r1574 | 'This will be removed soon. Use IPython.testing.util instead') | ||
|
r2487 | argv = sys.argv + [ '--detailed-errors', # extra info in tracebacks | ||
|
r4872 | |||
|
r2414 | # Loading ipdoctest causes problems with Twisted, but | ||
# our test suite runner now separates things and runs | ||||
# all Twisted tests with trial. | ||||
'--with-ipdoctest', | ||||
'--ipdoctest-tests','--ipdoctest-extension=txt', | ||||
|
r4872 | |||
|
r1761 | # We add --exe because of setuptools' imbecility (it | ||
# blindly does chmod +x on ALL files). Nose does the | ||||
# right thing and it tries to avoid executables, | ||||
# setuptools unfortunately forces our hand here. This | ||||
# has been discussed on the distutils list and the | ||||
# setuptools devs refuse to fix this problem! | ||||
'--exe', | ||||
] | ||||
|
r1574 | |||
|
r2494 | if nose.__version__ >= '0.11': | ||
# I don't fully understand why we need this one, but depending on what | ||||
# directory the test suite is run from, if we don't give it, 0 tests | ||||
# get run. Specifically, if the test suite is run from the source dir | ||||
# with an argument (like 'iptest.py IPython.core', 0 tests are run, | ||||
# even if the same call done in this directory works fine). It appears | ||||
# that if the requested package is in the current dir, nose bails early | ||||
# by default. Since it's otherwise harmless, leave it in by default | ||||
# for nose >= 0.11, though unfortunately nose 0.10 doesn't support it. | ||||
argv.append('--traverse-namespace') | ||||
|
r1574 | |||
|
r4567 | # use our plugin for doctesting. It will remove the standard doctest plugin | ||
# if it finds it enabled | ||||
|
r3511 | plugins = [IPythonDoctest(make_exclude()), KnownFailure()] | ||
|
r2414 | # We need a global ipython running in this process | ||
globalipapp.start_ipython() | ||||
# Now nose can run | ||||
|
r4567 | TestProgram(argv=argv, addplugins=plugins) | ||
|
r1972 | |||
def run_iptestall(): | ||||
"""Run the entire IPython test suite by calling nose and trial. | ||||
|
r4872 | |||
|
r1972 | This function constructs :class:`IPTester` instances for all IPython | ||
modules and package and then runs each of them. This causes the modules | ||||
and packages of IPython to be tested each in their own subprocess using | ||||
nose or twisted.trial appropriately. | ||||
""" | ||||
|
r2079 | |||
|
r1972 | runners = make_runners() | ||
|
r2079 | |||
|
r2398 | # Run the test runners in a temporary dir so we can nuke it when finished | ||
# to clean up any junk files left over by accident. This also makes it | ||||
# robust against being run in non-writeable directories by mistake, as the | ||||
# temp dir will always be user-writeable. | ||||
|
r4208 | curdir = os.getcwdu() | ||
|
r2398 | testdir = tempfile.gettempdir() | ||
os.chdir(testdir) | ||||
|
r1972 | # Run all test runners, tracking execution time | ||
|
r2480 | failed = [] | ||
|
r1972 | t_start = time.time() | ||
|
r2398 | try: | ||
|
r2480 | for (name, runner) in runners: | ||
print '*'*70 | ||||
|
r2398 | print 'IPython test group:',name | ||
res = runner.run() | ||||
if res: | ||||
|
r2480 | failed.append( (name, runner) ) | ||
|
r2398 | finally: | ||
os.chdir(curdir) | ||||
|
r1972 | t_end = time.time() | ||
t_tests = t_end - t_start | ||||
nrunners = len(runners) | ||||
nfail = len(failed) | ||||
# summarize results | ||||
|
r2480 | print '*'*70 | ||
|
r2496 | print 'Test suite completed for system with the following information:' | ||
print report() | ||||
|
r2091 | print 'Ran %s test groups in %.3fs' % (nrunners, t_tests) | ||
|
r1972 | |||
|
r2496 | print 'Status:' | ||
|
r1972 | if not failed: | ||
print 'OK' | ||||
else: | ||||
# If anything went wrong, point out what command to rerun manually to | ||||
# see the actual errors and individual summary | ||||
|
r2091 | print 'ERROR - %s out of %s test groups failed.' % (nfail, nrunners) | ||
|
r2480 | for name, failed_runner in failed: | ||
|
r1972 | print '-'*40 | ||
print 'Runner failed:',name | ||||
print 'You may wish to rerun this one individually, with:' | ||||
print ' '.join(failed_runner.call_args) | ||||
|
r3895 | # Ensure that our exit code indicates failure | ||
sys.exit(1) | ||||
|
r1972 | |||
def main(): | ||||
|
r2480 | for arg in sys.argv[1:]: | ||
if arg.startswith('IPython'): | ||||
|
r2481 | # This is in-process | ||
|
r1990 | run_iptest() | ||
|
r2480 | else: | ||
|
r2481 | # This starts subprocesses | ||
|
r2480 | run_iptestall() | ||
|
r1972 | |||
if __name__ == '__main__': | ||||
|
r2091 | main() | ||