iptest.py
517 lines
| 17.7 KiB
| text/x-python
|
PythonLexer
|
r1574 | # -*- coding: utf-8 -*- | ||
"""IPython Test Suite Runner. | ||||
|
r1851 | |||
|
r1972 | This module provides a main entry point to a user script to test IPython | ||
itself from the command line. There are two ways of running this script: | ||||
1. With the syntax `iptest all`. This runs our entire test suite by | ||||
|
r3486 | calling this script (with different arguments) recursively. This | ||
|
r1972 | causes modules and package to be tested in different processes, using nose | ||
or trial where appropriate. | ||||
2. With the regular nose syntax, like `iptest -vvs IPython`. In this form | ||||
the script simply calls nose, but with special command line flags and | ||||
plugins loaded. | ||||
|
r1574 | """ | ||
|
r18242 | # Copyright (c) IPython Development Team. | ||
# Distributed under the terms of the Modified BSD License. | ||||
|
r2498 | |||
|
r7817 | from __future__ import print_function | ||
|
r1851 | |||
|
r7412 | import glob | ||
|
r12973 | from io import BytesIO | ||
|
r1972 | import os | ||
import os.path as path | ||||
|
r1574 | import sys | ||
|
r12973 | from threading import Thread, Lock, Event | ||
|
r1574 | import warnings | ||
import nose.plugins.builtin | ||||
|
r6101 | from nose.plugins.xunit import Xunit | ||
from nose import SkipTest | ||||
|
r1851 | from nose.core import TestProgram | ||
|
r12608 | from nose.plugins import Plugin | ||
|
r12973 | from nose.util import safe_str | ||
|
r1574 | |||
|
r13263 | from IPython.utils.process import is_cmd_found | ||
|
r18692 | from IPython.utils.py3compat import bytes_to_str | ||
|
r4857 | from IPython.utils.importstring import import_item | ||
|
r2480 | from IPython.testing.plugin.ipdoctest import IPythonDoctest | ||
|
r6101 | from IPython.external.decorators import KnownFailure, knownfailureif | ||
|
r1574 | |||
|
r1979 | pjoin = path.join | ||
|
r1851 | #----------------------------------------------------------------------------- | ||
|
r2398 | # Warnings control | ||
#----------------------------------------------------------------------------- | ||||
|
r2512 | |||
|
r2398 | # Twisted generates annoying warnings with Python 2.6, as will do other code | ||
# that imports 'sets' as of today | ||||
warnings.filterwarnings('ignore', 'the sets module is deprecated', | ||||
DeprecationWarning ) | ||||
|
r2461 | # This one also comes from Twisted | ||
warnings.filterwarnings('ignore', 'the sha module is deprecated', | ||||
DeprecationWarning) | ||||
|
r2483 | # Wx on Fedora11 spits these out | ||
warnings.filterwarnings('ignore', 'wxPython/wxWidgets release number mismatch', | ||||
UserWarning) | ||||
|
r6101 | # ------------------------------------------------------------------------------ | ||
# Monkeypatch Xunit to count known failures as skipped. | ||||
# ------------------------------------------------------------------------------ | ||||
|
r6102 | def monkeypatch_xunit(): | ||
|
r6101 | try: | ||
knownfailureif(True)(lambda: None)() | ||||
except Exception as e: | ||||
KnownFailureTest = type(e) | ||||
def addError(self, test, err, capt=None): | ||||
if issubclass(err[0], KnownFailureTest): | ||||
err = (SkipTest,) + err[1:] | ||||
return self.orig_addError(test, err, capt) | ||||
Xunit.orig_addError = Xunit.addError | ||||
Xunit.addError = addError | ||||
|
r2398 | #----------------------------------------------------------------------------- | ||
|
r12608 | # Check which dependencies are installed and greater than minimum version. | ||
|
r1851 | #----------------------------------------------------------------------------- | ||
|
r4857 | def extract_version(mod): | ||
return mod.__version__ | ||||
|
r1851 | |||
|
r4857 | def test_for(item, min_version=None, callback=extract_version): | ||
"""Test to see if item is importable, and optionally check against a minimum | ||||
version. | ||||
|
r4872 | |||
|
r4857 | If min_version is given, the default behavior is to check against the | ||
`__version__` attribute of the item, but specifying `callback` allows you to | ||||
extract the value you are interested in. e.g:: | ||||
|
r4872 | |||
|
r4857 | In [1]: import sys | ||
|
r4872 | |||
|
r4857 | In [2]: from IPython.testing.iptest import test_for | ||
|
r4872 | |||
|
r4857 | In [3]: test_for('sys', (2,6), callback=lambda sys: sys.version_info) | ||
Out[3]: True | ||||
|
r4872 | |||
|
r4857 | """ | ||
|
r1981 | try: | ||
|
r4857 | check = import_item(item) | ||
|
r2488 | except (ImportError, RuntimeError): | ||
|
r4857 | # GTK reports Runtime error if it can't be initialized even if it's | ||
|
r2488 | # importable. | ||
|
r1981 | return False | ||
else: | ||||
|
r3640 | if min_version: | ||
|
r4857 | if callback: | ||
# extra processing step to get version to compare | ||||
check = callback(check) | ||||
|
r4872 | |||
|
r4857 | return check >= min_version | ||
|
r3640 | else: | ||
return True | ||||
|
r1981 | |||
|
r2496 | # Global dict where we can store information on what we have and what we don't | ||
# have available at test run time | ||||
have = {} | ||||
have['curses'] = test_for('_curses') | ||||
|
r3731 | have['matplotlib'] = test_for('matplotlib') | ||
|
r7224 | have['numpy'] = test_for('numpy') | ||
|
r20815 | have['pexpect'] = test_for('pexpect') | ||
|
r3731 | have['pymongo'] = test_for('pymongo') | ||
|
r6874 | have['pygments'] = test_for('pygments') | ||
|
r3761 | have['qt'] = test_for('IPython.external.qt') | ||
|
r5147 | have['sqlite3'] = test_for('sqlite3') | ||
|
r18739 | have['tornado'] = test_for('tornado.version_info', (4,0), callback=None) | ||
|
r8908 | have['jinja2'] = test_for('jinja2') | ||
|
r17585 | have['mistune'] = test_for('mistune') | ||
|
r13177 | have['requests'] = test_for('requests') | ||
|
r11095 | have['sphinx'] = test_for('sphinx') | ||
|
r16981 | have['jsonschema'] = test_for('jsonschema') | ||
|
r18486 | have['terminado'] = test_for('terminado') | ||
|
r13263 | have['casperjs'] = is_cmd_found('casperjs') | ||
|
r16828 | have['phantomjs'] = is_cmd_found('phantomjs') | ||
have['slimerjs'] = is_cmd_found('slimerjs') | ||||
|
r4857 | |||
|
r20354 | min_zmq = (13,) | ||
|
r9336 | |||
|
r9339 | have['zmq'] = test_for('zmq.pyzmq_version_info', min_zmq, callback=lambda x: x()) | ||
|
r1981 | |||
|
r2494 | #----------------------------------------------------------------------------- | ||
|
r12608 | # Test suite definitions | ||
|
r2494 | #----------------------------------------------------------------------------- | ||
|
r1980 | |||
|
r12608 | test_group_names = ['parallel', 'kernel', 'kernel.inprocess', 'config', 'core', | ||
'extensions', 'lib', 'terminal', 'testing', 'utils', | ||||
|
r13279 | 'nbformat', 'qt', 'html', 'nbconvert' | ||
|
r12608 | ] | ||
class TestSection(object): | ||||
def __init__(self, name, includes): | ||||
self.name = name | ||||
self.includes = includes | ||||
self.excludes = [] | ||||
self.dependencies = [] | ||||
self.enabled = True | ||||
def exclude(self, module): | ||||
if not module.startswith('IPython'): | ||||
module = self.includes[0] + "." + module | ||||
self.excludes.append(module.replace('.', os.sep)) | ||||
def requires(self, *packages): | ||||
self.dependencies.extend(packages) | ||||
@property | ||||
def will_run(self): | ||||
return self.enabled and all(have[p] for p in self.dependencies) | ||||
|
r20861 | shims = { | ||
'parallel': 'ipython_parallel', | ||||
|
r20956 | 'kernel': 'ipython_kernel', | ||
'kernel.inprocess': 'ipython_kernel.inprocess', | ||||
|
r20999 | 'config': 'traitlets', | ||
|
r20861 | } | ||
|
r12608 | # Name -> (include, exclude, dependencies_met) | ||
|
r20861 | test_sections = {n:TestSection(n, [shims.get(n, 'IPython.%s' % n)]) for n in test_group_names} | ||
|
r12608 | |||
# Exclusions and dependencies | ||||
# --------------------------- | ||||
# core: | ||||
sec = test_sections['core'] | ||||
if not have['sqlite3']: | ||||
sec.exclude('tests.test_history') | ||||
sec.exclude('history') | ||||
if not have['matplotlib']: | ||||
sec.exclude('pylabtools'), | ||||
sec.exclude('tests.test_pylabtools') | ||||
# lib: | ||||
sec = test_sections['lib'] | ||||
if not have['zmq']: | ||||
sec.exclude('kernel') | ||||
# We do this unconditionally, so that the test suite doesn't import | ||||
# gtk, changing the default encoding and masking some unicode bugs. | ||||
sec.exclude('inputhookgtk') | ||||
|
r13757 | # We also do this unconditionally, because wx can interfere with Unix signals. | ||
# There are currently no tests for it anyway. | ||||
sec.exclude('inputhookwx') | ||||
|
r12608 | # Testing inputhook will need a lot of thought, to figure out | ||
# how to have tests that don't lock up with the gui event | ||||
# loops in the picture | ||||
sec.exclude('inputhook') | ||||
# testing: | ||||
|
r12777 | sec = test_sections['testing'] | ||
# These have to be skipped on win32 because they use echo, rm, cd, etc. | ||||
|
r12608 | # See ticket https://github.com/ipython/ipython/issues/87 | ||
if sys.platform == 'win32': | ||||
sec.exclude('plugin.test_exampleip') | ||||
sec.exclude('plugin.dtexample') | ||||
# terminal: | ||||
if (not have['pexpect']) or (not have['zmq']): | ||||
test_sections['terminal'].exclude('console') | ||||
# parallel | ||||
sec = test_sections['parallel'] | ||||
sec.requires('zmq') | ||||
if not have['pymongo']: | ||||
sec.exclude('controller.mongodb') | ||||
sec.exclude('tests.test_mongodb') | ||||
# kernel: | ||||
sec = test_sections['kernel'] | ||||
sec.requires('zmq') | ||||
# The in-process kernel tests are done in a separate section | ||||
sec.exclude('inprocess') | ||||
# importing gtk sets the default encoding, which we want to avoid | ||||
|
r20957 | sec.exclude('gui.gtkembed') | ||
sec.exclude('gui.gtk3embed') | ||||
|
r12608 | if not have['matplotlib']: | ||
|
r20957 | sec.exclude('pylab') | ||
|
r12608 | |||
# kernel.inprocess: | ||||
test_sections['kernel.inprocess'].requires('zmq') | ||||
# extensions: | ||||
sec = test_sections['extensions'] | ||||
|
r17610 | # This is deprecated in favour of rpy2 | ||
sec.exclude('rmagic') | ||||
|
r12608 | # autoreload does some strange stuff, so move it to its own test section | ||
sec.exclude('autoreload') | ||||
sec.exclude('tests.test_autoreload') | ||||
test_sections['autoreload'] = TestSection('autoreload', | ||||
['IPython.extensions.autoreload', 'IPython.extensions.tests.test_autoreload']) | ||||
test_group_names.append('autoreload') | ||||
# qt: | ||||
test_sections['qt'].requires('zmq', 'qt', 'pygments') | ||||
# html: | ||||
sec = test_sections['html'] | ||||
|
r18242 | sec.requires('zmq', 'tornado', 'requests', 'sqlite3', 'jsonschema') | ||
|
r12608 | # The notebook 'static' directory contains JS, css and other | ||
# files for web serving. Occasionally projects may put a .py | ||||
# file in there (MathJax ships a conf.py), so we might as | ||||
# well play it safe and skip the whole thing. | ||||
sec.exclude('static') | ||||
|
r18351 | sec.exclude('tasks') | ||
|
r12608 | if not have['jinja2']: | ||
sec.exclude('notebookapp') | ||||
|
r13948 | if not have['pygments'] or not have['jinja2']: | ||
sec.exclude('nbconvert') | ||||
|
r18486 | if not have['terminado']: | ||
sec.exclude('terminal') | ||||
|
r12608 | |||
# nbconvert: | ||||
sec = test_sections['nbconvert'] | ||||
|
r18242 | sec.requires('pygments', 'jinja2', 'jsonschema', 'mistune') | ||
|
r12608 | # Exclude nbconvert directories containing config files used to test. | ||
# Executing the config files with iptest would cause an exception. | ||||
sec.exclude('tests.files') | ||||
sec.exclude('exporters.tests.files') | ||||
|
r12625 | if not have['tornado']: | ||
sec.exclude('nbconvert.post_processors.serve') | ||||
sec.exclude('nbconvert.post_processors.tests.test_serve') | ||||
|
r2496 | |||
|
r16981 | # nbformat: | ||
|
r18242 | test_sections['nbformat'].requires('jsonschema') | ||
|
r16981 | |||
|
r12608 | #----------------------------------------------------------------------------- | ||
# Functions and classes | ||||
#----------------------------------------------------------------------------- | ||||
|
r2512 | |||
|
r12608 | def check_exclusions_exist(): | ||
|
r12612 | from IPython.utils.path import get_ipython_package_dir | ||
from IPython.utils.warn import warn | ||||
|
r12608 | parent = os.path.dirname(get_ipython_package_dir()) | ||
for sec in test_sections: | ||||
for pattern in sec.exclusions: | ||||
fullpath = pjoin(parent, pattern) | ||||
if not os.path.exists(fullpath) and not glob.glob(fullpath + '.*'): | ||||
warn("Excluding nonexistent file: %r" % pattern) | ||||
|
r2512 | |||
|
r12608 | |||
class ExclusionPlugin(Plugin): | ||||
"""A nose plugin to effect our exclusions of files and directories. | ||||
|
r2454 | """ | ||
|
r12608 | name = 'exclusions' | ||
score = 3000 # Should come before any other plugins | ||||
|
r9375 | |||
|
r12608 | def __init__(self, exclude_patterns=None): | ||
""" | ||||
Parameters | ||||
---------- | ||||
exclude_patterns : sequence of strings, optional | ||||
|
r12756 | Filenames containing these patterns (as raw strings, not as regular | ||
expressions) are excluded from the tests. | ||||
|
r12608 | """ | ||
|
r12756 | self.exclude_patterns = exclude_patterns or [] | ||
|
r12608 | super(ExclusionPlugin, self).__init__() | ||
def options(self, parser, env=os.environ): | ||||
Plugin.options(self, parser, env) | ||||
|
r7121 | |||
|
r12608 | def configure(self, options, config): | ||
Plugin.configure(self, options, config) | ||||
# Override nose trying to disable plugin. | ||||
self.enabled = True | ||||
def wantFile(self, filename): | ||||
"""Return whether the given filename should be scanned for tests. | ||||
""" | ||||
|
r12756 | if any(pat in filename for pat in self.exclude_patterns): | ||
|
r12608 | return False | ||
return None | ||||
def wantDirectory(self, directory): | ||||
"""Return whether the given directory should be scanned for tests. | ||||
""" | ||||
|
r12756 | if any(pat in directory for pat in self.exclude_patterns): | ||
|
r12608 | return False | ||
return None | ||||
|
r2398 | |||
|
r12973 | class StreamCapturer(Thread): | ||
|
r13517 | daemon = True # Don't hang if main thread crashes | ||
|
r12973 | started = False | ||
|
r18692 | def __init__(self, echo=False): | ||
|
r12973 | super(StreamCapturer, self).__init__() | ||
|
r18692 | self.echo = echo | ||
|
r12973 | self.streams = [] | ||
self.buffer = BytesIO() | ||||
|
r13405 | self.readfd, self.writefd = os.pipe() | ||
|
r12973 | self.buffer_lock = Lock() | ||
self.stop = Event() | ||||
def run(self): | ||||
self.started = True | ||||
|
r13405 | |||
|
r12973 | while not self.stop.is_set(): | ||
|
r13423 | chunk = os.read(self.readfd, 1024) | ||
|
r13405 | |||
|
r13423 | with self.buffer_lock: | ||
self.buffer.write(chunk) | ||||
|
r18692 | if self.echo: | ||
sys.stdout.write(bytes_to_str(chunk)) | ||||
|
r12973 | |||
|
r13405 | os.close(self.readfd) | ||
os.close(self.writefd) | ||||
|
r13423 | |||
|
r12973 | def reset_buffer(self): | ||
with self.buffer_lock: | ||||
self.buffer.truncate(0) | ||||
self.buffer.seek(0) | ||||
|
r13423 | |||
|
r12973 | def get_buffer(self): | ||
with self.buffer_lock: | ||||
return self.buffer.getvalue() | ||||
|
r13423 | |||
|
r12973 | def ensure_started(self): | ||
if not self.started: | ||||
self.start() | ||||
|
r13423 | def halt(self): | ||
"""Safely stop the thread.""" | ||||
if not self.started: | ||||
return | ||||
self.stop.set() | ||||
|
r20318 | os.write(self.writefd, b'\0') # Ensure we're not locked in a read() | ||
|
r13423 | self.join() | ||
|
r12973 | class SubprocessStreamCapturePlugin(Plugin): | ||
name='subprocstreams' | ||||
def __init__(self): | ||||
Plugin.__init__(self) | ||||
self.stream_capturer = StreamCapturer() | ||||
|
r13824 | self.destination = os.environ.get('IPTEST_SUBPROC_STREAMS', 'capture') | ||
|
r12973 | # This is ugly, but distant parts of the test machinery need to be able | ||
|
r13405 | # to redirect streams, so we make the object globally accessible. | ||
|
r13824 | nose.iptest_stdstreams_fileno = self.get_write_fileno | ||
def get_write_fileno(self): | ||||
if self.destination == 'capture': | ||||
self.stream_capturer.ensure_started() | ||||
return self.stream_capturer.writefd | ||||
elif self.destination == 'discard': | ||||
return os.open(os.devnull, os.O_WRONLY) | ||||
else: | ||||
return sys.__stdout__.fileno() | ||||
|
r12973 | |||
def configure(self, options, config): | ||||
Plugin.configure(self, options, config) | ||||
# Override nose trying to disable plugin. | ||||
|
r13824 | if self.destination == 'capture': | ||
self.enabled = True | ||||
|
r12973 | |||
def startTest(self, test): | ||||
# Reset log capture | ||||
self.stream_capturer.reset_buffer() | ||||
def formatFailure(self, test, err): | ||||
# Show output | ||||
ec, ev, tb = err | ||||
|
r13155 | captured = self.stream_capturer.get_buffer().decode('utf-8', 'replace') | ||
if captured.strip(): | ||||
ev = safe_str(ev) | ||||
out = [ev, '>> begin captured subprocess output <<', | ||||
captured, | ||||
'>> end captured subprocess output <<'] | ||||
return ec, '\n'.join(out), tb | ||||
return err | ||||
|
r12973 | |||
formatError = formatFailure | ||||
def finalize(self, result): | ||||
|
r13423 | self.stream_capturer.halt() | ||
|
r12973 | |||
|
r1972 | def run_iptest(): | ||
"""Run the IPython test suite using nose. | ||||
|
r4872 | |||
|
r1972 | This function is called when this script is **not** called with the form | ||
`iptest all`. It simply calls nose with appropriate command line flags | ||||
and accepts all of the standard nose arguments. | ||||
|
r1574 | """ | ||
|
r6102 | # Apply our monkeypatch to Xunit | ||
|
r6113 | if '--with-xunit' in sys.argv and not hasattr(Xunit, 'orig_addError'): | ||
|
r6102 | monkeypatch_xunit() | ||
|
r1574 | |||
|
r4872 | warnings.filterwarnings('ignore', | ||
|
r1574 | 'This will be removed soon. Use IPython.testing.util instead') | ||
|
r11073 | |||
|
r12622 | arg1 = sys.argv[1] | ||
if arg1 in test_sections: | ||||
section = test_sections[arg1] | ||||
sys.argv[1:2] = section.includes | ||||
elif arg1.startswith('IPython.') and arg1[8:] in test_sections: | ||||
section = test_sections[arg1[8:]] | ||||
|
r12620 | sys.argv[1:2] = section.includes | ||
else: | ||||
section = TestSection(arg1, includes=[arg1]) | ||||
|
r1574 | |||
|
r2487 | argv = sys.argv + [ '--detailed-errors', # extra info in tracebacks | ||
|
r4872 | |||
|
r2414 | '--with-ipdoctest', | ||
'--ipdoctest-tests','--ipdoctest-extension=txt', | ||||
|
r4872 | |||
|
r1761 | # We add --exe because of setuptools' imbecility (it | ||
# blindly does chmod +x on ALL files). Nose does the | ||||
# right thing and it tries to avoid executables, | ||||
# setuptools unfortunately forces our hand here. This | ||||
# has been discussed on the distutils list and the | ||||
# setuptools devs refuse to fix this problem! | ||||
'--exe', | ||||
] | ||||
|
r8201 | if '-a' not in argv and '-A' not in argv: | ||
argv = argv + ['-a', '!crash'] | ||||
|
r1574 | |||
|
r2494 | if nose.__version__ >= '0.11': | ||
# I don't fully understand why we need this one, but depending on what | ||||
# directory the test suite is run from, if we don't give it, 0 tests | ||||
# get run. Specifically, if the test suite is run from the source dir | ||||
# with an argument (like 'iptest.py IPython.core', 0 tests are run, | ||||
# even if the same call done in this directory works fine). It appears | ||||
# that if the requested package is in the current dir, nose bails early | ||||
# by default. Since it's otherwise harmless, leave it in by default | ||||
# for nose >= 0.11, though unfortunately nose 0.10 doesn't support it. | ||||
argv.append('--traverse-namespace') | ||||
|
r1574 | |||
|
r4567 | # use our plugin for doctesting. It will remove the standard doctest plugin | ||
# if it finds it enabled | ||||
|
r12973 | plugins = [ExclusionPlugin(section.excludes), IPythonDoctest(), KnownFailure(), | ||
SubprocessStreamCapturePlugin() ] | ||||
|
r12608 | |||
# Use working directory set by parent process (see iptestcontroller) | ||||
if 'IPTEST_WORKING_DIR' in os.environ: | ||||
os.chdir(os.environ['IPTEST_WORKING_DIR']) | ||||
|
r8489 | |||
# We need a global ipython running in this process, but the special | ||||
# in-process group spawns its own IPython kernels, so for *that* group we | ||||
# must avoid also opening the global one (otherwise there's a conflict of | ||||
# singletons). Ultimately the solution to this problem is to refactor our | ||||
# assumptions about what needs to be a singleton and what doesn't (app | ||||
# objects should, individual shells shouldn't). But for now, this | ||||
# workaround allows the test suite for the inprocess module to complete. | ||||
|
r12620 | if 'kernel.inprocess' not in section.name: | ||
|
r12612 | from IPython.testing import globalipapp | ||
|
r8489 | globalipapp.start_ipython() | ||
|
r2414 | # Now nose can run | ||
|
r4567 | TestProgram(argv=argv, addplugins=plugins) | ||
|
r1972 | |||
if __name__ == '__main__': | ||||
|
r12607 | run_iptest() | ||
|
r12612 | |||