iptest.py
365 lines
| 14.4 KiB
| text/x-python
|
PythonLexer
Fernando Perez
|
r1574 | # -*- coding: utf-8 -*- | ||
"""IPython Test Suite Runner. | ||||
Fernando Perez
|
r1851 | |||
Brian Granger
|
r1972 | This module provides a main entry point to a user script to test IPython | ||
itself from the command line. There are two ways of running this script: | ||||
1. With the syntax `iptest all`. This runs our entire test suite by | ||||
Brian Granger
|
r3486 | calling this script (with different arguments) recursively. This | ||
Brian Granger
|
r1972 | causes modules and package to be tested in different processes, using nose | ||
or trial where appropriate. | ||||
2. With the regular nose syntax, like `iptest -vvs IPython`. In this form | ||||
the script simply calls nose, but with special command line flags and | ||||
plugins loaded. | ||||
Fernando Perez
|
r1574 | """ | ||
Fernando Perez
|
r1851 | #----------------------------------------------------------------------------- | ||
Matthias BUSSONNIER
|
r5390 | # Copyright (C) 2009-2011 The IPython Development Team | ||
Brian Granger
|
r2498 | # | ||
# Distributed under the terms of the BSD License. The full license is in | ||||
# the file COPYING, distributed as part of this software. | ||||
#----------------------------------------------------------------------------- | ||||
#----------------------------------------------------------------------------- | ||||
# Imports | ||||
Fernando Perez
|
r1851 | #----------------------------------------------------------------------------- | ||
Matthias BUSSONNIER
|
r7817 | from __future__ import print_function | ||
Fernando Perez
|
r1851 | |||
Fernando Perez
|
r2442 | # Stdlib | ||
MinRK
|
r7412 | import glob | ||
Brian Granger
|
r1972 | import os | ||
import os.path as path | ||||
Fernando Perez
|
r1574 | import sys | ||
import warnings | ||||
Fernando Perez
|
r2442 | # Now, proceed to import nose itself | ||
Fernando Perez
|
r1574 | import nose.plugins.builtin | ||
Thomas Kluyver
|
r6101 | from nose.plugins.xunit import Xunit | ||
from nose import SkipTest | ||||
Fernando Perez
|
r1851 | from nose.core import TestProgram | ||
Fernando Perez
|
r1574 | |||
Fernando Perez
|
r2442 | # Our own imports | ||
MinRK
|
r4857 | from IPython.utils.importstring import import_item | ||
Thomas Kluyver
|
r12607 | from IPython.utils.path import get_ipython_package_dir | ||
MinRK
|
r7121 | from IPython.utils.warn import warn | ||
Brian Granger
|
r2498 | |||
Fernando Perez
|
r2480 | from IPython.testing import globalipapp | ||
from IPython.testing.plugin.ipdoctest import IPythonDoctest | ||||
Thomas Kluyver
|
r6101 | from IPython.external.decorators import KnownFailure, knownfailureif | ||
Fernando Perez
|
r1574 | |||
Brian Granger
|
r1979 | pjoin = path.join | ||
Fernando Perez
|
r2494 | |||
#----------------------------------------------------------------------------- | ||||
# Globals | ||||
#----------------------------------------------------------------------------- | ||||
Fernando Perez
|
r1851 | #----------------------------------------------------------------------------- | ||
Fernando Perez
|
r2398 | # Warnings control | ||
#----------------------------------------------------------------------------- | ||||
Brian Granger
|
r2512 | |||
Fernando Perez
|
r2398 | # Twisted generates annoying warnings with Python 2.6, as will do other code | ||
# that imports 'sets' as of today | ||||
warnings.filterwarnings('ignore', 'the sets module is deprecated', | ||||
DeprecationWarning ) | ||||
Fernando Perez
|
r2461 | # This one also comes from Twisted | ||
warnings.filterwarnings('ignore', 'the sha module is deprecated', | ||||
DeprecationWarning) | ||||
Fernando Perez
|
r2483 | # Wx on Fedora11 spits these out | ||
warnings.filterwarnings('ignore', 'wxPython/wxWidgets release number mismatch', | ||||
UserWarning) | ||||
Thomas Kluyver
|
r6101 | # ------------------------------------------------------------------------------ | ||
# Monkeypatch Xunit to count known failures as skipped. | ||||
# ------------------------------------------------------------------------------ | ||||
Thomas Kluyver
|
r6102 | def monkeypatch_xunit(): | ||
Thomas Kluyver
|
r6101 | try: | ||
knownfailureif(True)(lambda: None)() | ||||
except Exception as e: | ||||
KnownFailureTest = type(e) | ||||
def addError(self, test, err, capt=None): | ||||
if issubclass(err[0], KnownFailureTest): | ||||
err = (SkipTest,) + err[1:] | ||||
return self.orig_addError(test, err, capt) | ||||
Xunit.orig_addError = Xunit.addError | ||||
Xunit.addError = addError | ||||
Fernando Perez
|
r2398 | #----------------------------------------------------------------------------- | ||
Administrator
|
r1981 | # Logic for skipping doctests | ||
Fernando Perez
|
r1851 | #----------------------------------------------------------------------------- | ||
MinRK
|
r4857 | def extract_version(mod): | ||
return mod.__version__ | ||||
Fernando Perez
|
r1851 | |||
MinRK
|
r4857 | def test_for(item, min_version=None, callback=extract_version): | ||
"""Test to see if item is importable, and optionally check against a minimum | ||||
version. | ||||
Bernardo B. Marques
|
r4872 | |||
MinRK
|
r4857 | If min_version is given, the default behavior is to check against the | ||
`__version__` attribute of the item, but specifying `callback` allows you to | ||||
extract the value you are interested in. e.g:: | ||||
Bernardo B. Marques
|
r4872 | |||
MinRK
|
r4857 | In [1]: import sys | ||
Bernardo B. Marques
|
r4872 | |||
MinRK
|
r4857 | In [2]: from IPython.testing.iptest import test_for | ||
Bernardo B. Marques
|
r4872 | |||
MinRK
|
r4857 | In [3]: test_for('sys', (2,6), callback=lambda sys: sys.version_info) | ||
Out[3]: True | ||||
Bernardo B. Marques
|
r4872 | |||
MinRK
|
r4857 | """ | ||
Administrator
|
r1981 | try: | ||
MinRK
|
r4857 | check = import_item(item) | ||
Fernando Perez
|
r2488 | except (ImportError, RuntimeError): | ||
MinRK
|
r4857 | # GTK reports Runtime error if it can't be initialized even if it's | ||
Fernando Perez
|
r2488 | # importable. | ||
Administrator
|
r1981 | return False | ||
else: | ||||
MinRK
|
r3640 | if min_version: | ||
MinRK
|
r4857 | if callback: | ||
# extra processing step to get version to compare | ||||
check = callback(check) | ||||
Bernardo B. Marques
|
r4872 | |||
MinRK
|
r4857 | return check >= min_version | ||
MinRK
|
r3640 | else: | ||
return True | ||||
Administrator
|
r1981 | |||
Fernando Perez
|
r2496 | # Global dict where we can store information on what we have and what we don't | ||
# have available at test run time | ||||
have = {} | ||||
have['curses'] = test_for('_curses') | ||||
Fernando Perez
|
r3731 | have['matplotlib'] = test_for('matplotlib') | ||
Fernando Perez
|
r7224 | have['numpy'] = test_for('numpy') | ||
MinRK
|
r4857 | have['pexpect'] = test_for('IPython.external.pexpect') | ||
Fernando Perez
|
r3731 | have['pymongo'] = test_for('pymongo') | ||
MinRK
|
r6874 | have['pygments'] = test_for('pygments') | ||
MinRK
|
r3761 | have['qt'] = test_for('IPython.external.qt') | ||
Fernando Perez
|
r7224 | have['rpy2'] = test_for('rpy2') | ||
MinRK
|
r5147 | have['sqlite3'] = test_for('sqlite3') | ||
Brian Granger
|
r7102 | have['cython'] = test_for('Cython') | ||
Stefan van der Walt
|
r7385 | have['oct2py'] = test_for('oct2py') | ||
MinRK
|
r4857 | have['tornado'] = test_for('tornado.version_info', (2,1,0), callback=None) | ||
Bradley M. Froehle
|
r8908 | have['jinja2'] = test_for('jinja2') | ||
Fernando Perez
|
r7224 | have['wx'] = test_for('wx') | ||
have['wx.aui'] = test_for('wx.aui') | ||||
Brian Granger
|
r8184 | have['azure'] = test_for('azure') | ||
Brian E. Granger
|
r11095 | have['sphinx'] = test_for('sphinx') | ||
MinRK
|
r4857 | |||
MinRK
|
r9336 | min_zmq = (2,1,11) | ||
MinRK
|
r9339 | have['zmq'] = test_for('zmq.pyzmq_version_info', min_zmq, callback=lambda x: x()) | ||
Administrator
|
r1981 | |||
Fernando Perez
|
r2494 | #----------------------------------------------------------------------------- | ||
# Functions and classes | ||||
#----------------------------------------------------------------------------- | ||||
Administrator
|
r1980 | |||
Fernando Perez
|
r2496 | |||
Brian Granger
|
r2079 | def make_exclude(): | ||
Fernando Perez
|
r2454 | """Make patterns of modules and packages to exclude from testing. | ||
Brian Granger
|
r2512 | |||
Fernando Perez
|
r2454 | For the IPythonDoctest plugin, we need to exclude certain patterns that | ||
cause testing problems. We should strive to minimize the number of | ||||
Brian Granger
|
r2512 | skipped modules, since this means untested code. | ||
Fernando Perez
|
r2454 | These modules and packages will NOT get scanned by nose at all for tests. | ||
""" | ||||
# Simple utility to make IPython paths more readably, we need a lot of | ||||
# these below | ||||
ipjoin = lambda *paths: pjoin('IPython', *paths) | ||||
Bernardo B. Marques
|
r4872 | |||
Fernando Perez
|
r2454 | exclusions = [ipjoin('external'), | ||
ipjoin('quarantine'), | ||||
ipjoin('deathrow'), | ||||
Fernando Perez
|
r2461 | # This guy is probably attic material | ||
Fernando Perez
|
r2454 | ipjoin('testing', 'mkdoctests'), | ||
Fernando Perez
|
r2461 | # Testing inputhook will need a lot of thought, to figure out | ||
# how to have tests that don't lock up with the gui event | ||||
# loops in the picture | ||||
Fernando Perez
|
r2454 | ipjoin('lib', 'inputhook'), | ||
Fernando Perez
|
r2417 | # Config files aren't really importable stand-alone | ||
Fernando Perez
|
r2454 | ipjoin('config', 'profile'), | ||
Fernando Perez
|
r7801 | # The notebook 'static' directory contains JS, css and other | ||
# files for web serving. Occasionally projects may put a .py | ||||
# file in there (MathJax ships a conf.py), so we might as | ||||
# well play it safe and skip the whole thing. | ||||
MinRK
|
r11035 | ipjoin('html', 'static'), | ||
ipjoin('html', 'fabfile'), | ||||
Fernando Perez
|
r2417 | ] | ||
MinRK
|
r5147 | if not have['sqlite3']: | ||
exclusions.append(ipjoin('core', 'tests', 'test_history')) | ||||
exclusions.append(ipjoin('core', 'history')) | ||||
Fernando Perez
|
r2496 | if not have['wx']: | ||
Fernando Perez
|
r2454 | exclusions.append(ipjoin('lib', 'inputhookwx')) | ||
MinRK
|
r7455 | |||
MinRK
|
r9375 | if 'IPython.kernel.inprocess' not in sys.argv: | ||
exclusions.append(ipjoin('kernel', 'inprocess')) | ||||
MinRK
|
r7455 | # FIXME: temporarily disable autoreload tests, as they can produce | ||
# spurious failures in subsequent tests (cythonmagic). | ||||
exclusions.append(ipjoin('extensions', 'autoreload')) | ||||
exclusions.append(ipjoin('extensions', 'tests', 'test_autoreload')) | ||||
Bernardo B. Marques
|
r4872 | |||
Thomas Kluyver
|
r3458 | # We do this unconditionally, so that the test suite doesn't import | ||
# gtk, changing the default encoding and masking some unicode bugs. | ||||
exclusions.append(ipjoin('lib', 'inputhookgtk')) | ||||
MinRK
|
r9375 | exclusions.append(ipjoin('kernel', 'zmq', 'gui', 'gtkembed')) | ||
Brian Granger
|
r2083 | |||
Jonathan Frederic
|
r11507 | #Also done unconditionally, exclude nbconvert directories containing | ||
#config files used to test. Executing the config files with iptest would | ||||
#cause an exception. | ||||
exclusions.append(ipjoin('nbconvert', 'tests', 'files')) | ||||
exclusions.append(ipjoin('nbconvert', 'exporters', 'tests', 'files')) | ||||
Jonathan Frederic
|
r11497 | |||
Brian Granger
|
r2079 | # These have to be skipped on win32 because the use echo, rm, cd, etc. | ||
Thomas Kluyver
|
r3917 | # See ticket https://github.com/ipython/ipython/issues/87 | ||
Brian Granger
|
r2079 | if sys.platform == 'win32': | ||
Fernando Perez
|
r2454 | exclusions.append(ipjoin('testing', 'plugin', 'test_exampleip')) | ||
exclusions.append(ipjoin('testing', 'plugin', 'dtexample')) | ||||
Brian Granger
|
r2079 | |||
Fernando Perez
|
r2496 | if not have['pexpect']: | ||
MinRK
|
r7775 | exclusions.extend([ipjoin('lib', 'irunner'), | ||
MinRK
|
r5694 | ipjoin('lib', 'tests', 'test_irunner'), | ||
Fernando Perez
|
r11026 | ipjoin('terminal', 'console'), | ||
MinRK
|
r5694 | ]) | ||
Brian Granger
|
r2079 | |||
MinRK
|
r3640 | if not have['zmq']: | ||
MinRK
|
r12086 | exclusions.append(ipjoin('lib', 'kernel')) | ||
MinRK
|
r9375 | exclusions.append(ipjoin('kernel')) | ||
Fernando Perez
|
r11026 | exclusions.append(ipjoin('qt')) | ||
exclusions.append(ipjoin('html')) | ||||
exclusions.append(ipjoin('consoleapp.py')) | ||||
exclusions.append(ipjoin('terminal', 'console')) | ||||
MinRK
|
r3672 | exclusions.append(ipjoin('parallel')) | ||
MinRK
|
r6874 | elif not have['qt'] or not have['pygments']: | ||
Fernando Perez
|
r11026 | exclusions.append(ipjoin('qt')) | ||
Bernardo B. Marques
|
r4872 | |||
MinRK
|
r3674 | if not have['pymongo']: | ||
exclusions.append(ipjoin('parallel', 'controller', 'mongodb')) | ||||
MinRK
|
r3875 | exclusions.append(ipjoin('parallel', 'tests', 'test_mongodb')) | ||
MinRK
|
r3640 | |||
Fernando Perez
|
r3731 | if not have['matplotlib']: | ||
Fernando Perez
|
r5474 | exclusions.extend([ipjoin('core', 'pylabtools'), | ||
MinRK
|
r6566 | ipjoin('core', 'tests', 'test_pylabtools'), | ||
MinRK
|
r9375 | ipjoin('kernel', 'zmq', 'pylab'), | ||
MinRK
|
r6566 | ]) | ||
Fernando Perez
|
r3731 | |||
Brian Granger
|
r7102 | if not have['cython']: | ||
exclusions.extend([ipjoin('extensions', 'cythonmagic')]) | ||||
MinRK
|
r7117 | exclusions.extend([ipjoin('extensions', 'tests', 'test_cythonmagic')]) | ||
Brian Granger
|
r7102 | |||
Stefan van der Walt
|
r7385 | if not have['oct2py']: | ||
exclusions.extend([ipjoin('extensions', 'octavemagic')]) | ||||
exclusions.extend([ipjoin('extensions', 'tests', 'test_octavemagic')]) | ||||
Brian E. Granger
|
r4703 | if not have['tornado']: | ||
Fernando Perez
|
r11026 | exclusions.append(ipjoin('html')) | ||
MinRK
|
r12515 | exclusions.append(ipjoin('nbconvert', 'post_processors', 'serve')) | ||
exclusions.append(ipjoin('nbconvert', 'post_processors', 'tests', 'test_serve')) | ||||
Brian E. Granger
|
r4703 | |||
Bradley M. Froehle
|
r8908 | if not have['jinja2']: | ||
MinRK
|
r11035 | exclusions.append(ipjoin('html', 'notebookapp')) | ||
Bradley M. Froehle
|
r8908 | |||
Fernando Perez
|
r7224 | if not have['rpy2'] or not have['numpy']: | ||
exclusions.append(ipjoin('extensions', 'rmagic')) | ||||
exclusions.append(ipjoin('extensions', 'tests', 'test_rmagic')) | ||||
Brian Granger
|
r8184 | if not have['azure']: | ||
MinRK
|
r11035 | exclusions.append(ipjoin('html', 'services', 'notebooks', 'azurenbmanager')) | ||
Brian Granger
|
r8184 | |||
Jonathan Frederic
|
r11539 | if not all((have['pygments'], have['jinja2'], have['sphinx'])): | ||
Brian E. Granger
|
r11095 | exclusions.append(ipjoin('nbconvert')) | ||
Brian Granger
|
r2079 | # This is needed for the reg-exp to match on win32 in the ipdoctest plugin. | ||
if sys.platform == 'win32': | ||||
Fernando Perez
|
r2414 | exclusions = [s.replace('\\','\\\\') for s in exclusions] | ||
MinRK
|
r7121 | |||
# check for any exclusions that don't seem to exist: | ||||
MinRK
|
r7126 | parent, _ = os.path.split(get_ipython_package_dir()) | ||
MinRK
|
r7121 | for exclusion in exclusions: | ||
MinRK
|
r7463 | if exclusion.endswith(('deathrow', 'quarantine')): | ||
# ignore deathrow/quarantine, which exist in dev, but not install | ||||
continue | ||||
MinRK
|
r7126 | fullpath = pjoin(parent, exclusion) | ||
MinRK
|
r7412 | if not os.path.exists(fullpath) and not glob.glob(fullpath + '.*'): | ||
Thomas Kluyver
|
r8223 | warn("Excluding nonexistent file: %r" % exclusion) | ||
Brian Granger
|
r2079 | |||
Fernando Perez
|
r2414 | return exclusions | ||
Administrator
|
r1980 | |||
Thomas Kluyver
|
r11073 | special_test_suites = { | ||
'autoreload': ['IPython.extensions.autoreload', 'IPython.extensions.tests.test_autoreload'], | ||||
} | ||||
Fernando Perez
|
r2398 | |||
Brian Granger
|
r1972 | def run_iptest(): | ||
"""Run the IPython test suite using nose. | ||||
Bernardo B. Marques
|
r4872 | |||
Brian Granger
|
r1972 | This function is called when this script is **not** called with the form | ||
`iptest all`. It simply calls nose with appropriate command line flags | ||||
and accepts all of the standard nose arguments. | ||||
Fernando Perez
|
r1574 | """ | ||
Thomas Kluyver
|
r6102 | # Apply our monkeypatch to Xunit | ||
Thomas Kluyver
|
r6113 | if '--with-xunit' in sys.argv and not hasattr(Xunit, 'orig_addError'): | ||
Thomas Kluyver
|
r6102 | monkeypatch_xunit() | ||
Fernando Perez
|
r1574 | |||
Bernardo B. Marques
|
r4872 | warnings.filterwarnings('ignore', | ||
Fernando Perez
|
r1574 | 'This will be removed soon. Use IPython.testing.util instead') | ||
Thomas Kluyver
|
r11073 | |||
if sys.argv[1] in special_test_suites: | ||||
sys.argv[1:2] = special_test_suites[sys.argv[1]] | ||||
special_suite = True | ||||
else: | ||||
special_suite = False | ||||
Fernando Perez
|
r1574 | |||
Fernando Perez
|
r2487 | argv = sys.argv + [ '--detailed-errors', # extra info in tracebacks | ||
Bernardo B. Marques
|
r4872 | |||
Fernando Perez
|
r2414 | '--with-ipdoctest', | ||
'--ipdoctest-tests','--ipdoctest-extension=txt', | ||||
Bernardo B. Marques
|
r4872 | |||
Fernando Perez
|
r1761 | # We add --exe because of setuptools' imbecility (it | ||
# blindly does chmod +x on ALL files). Nose does the | ||||
# right thing and it tries to avoid executables, | ||||
# setuptools unfortunately forces our hand here. This | ||||
# has been discussed on the distutils list and the | ||||
# setuptools devs refuse to fix this problem! | ||||
'--exe', | ||||
] | ||||
MinRK
|
r8201 | if '-a' not in argv and '-A' not in argv: | ||
argv = argv + ['-a', '!crash'] | ||||
Fernando Perez
|
r1574 | |||
Fernando Perez
|
r2494 | if nose.__version__ >= '0.11': | ||
# I don't fully understand why we need this one, but depending on what | ||||
# directory the test suite is run from, if we don't give it, 0 tests | ||||
# get run. Specifically, if the test suite is run from the source dir | ||||
# with an argument (like 'iptest.py IPython.core', 0 tests are run, | ||||
# even if the same call done in this directory works fine). It appears | ||||
# that if the requested package is in the current dir, nose bails early | ||||
# by default. Since it's otherwise harmless, leave it in by default | ||||
# for nose >= 0.11, though unfortunately nose 0.10 doesn't support it. | ||||
argv.append('--traverse-namespace') | ||||
Fernando Perez
|
r1574 | |||
Matthew Brett
|
r4567 | # use our plugin for doctesting. It will remove the standard doctest plugin | ||
# if it finds it enabled | ||||
Thomas Kluyver
|
r11073 | ipdt = IPythonDoctest() if special_suite else IPythonDoctest(make_exclude()) | ||
plugins = [ipdt, KnownFailure()] | ||||
Fernando Perez
|
r8489 | |||
# We need a global ipython running in this process, but the special | ||||
# in-process group spawns its own IPython kernels, so for *that* group we | ||||
# must avoid also opening the global one (otherwise there's a conflict of | ||||
# singletons). Ultimately the solution to this problem is to refactor our | ||||
# assumptions about what needs to be a singleton and what doesn't (app | ||||
# objects should, individual shells shouldn't). But for now, this | ||||
# workaround allows the test suite for the inprocess module to complete. | ||||
MinRK
|
r9375 | if not 'IPython.kernel.inprocess' in sys.argv: | ||
Fernando Perez
|
r8489 | globalipapp.start_ipython() | ||
Fernando Perez
|
r2414 | # Now nose can run | ||
Matthew Brett
|
r4567 | TestProgram(argv=argv, addplugins=plugins) | ||
Brian Granger
|
r1972 | |||
if __name__ == '__main__': | ||||
Thomas Kluyver
|
r12607 | run_iptest() | ||