##// END OF EJS Templates
remove cython extension....
Matthias Bussonnier -
Show More
@@ -1,345 +1,42 b''
1 1 # -*- coding: utf-8 -*-
2 2 """
3 =====================
4 Cython related magics
5 =====================
3 The cython magic has been integrated into Cython itself,
4 which is now released in version 0.21.
6 5
7 Magic command interface for interactive work with Cython
8
9 .. note::
10
11 The ``Cython`` package needs to be installed separately. It
12 can be obtained using ``easy_install`` or ``pip``.
13
14 Usage
15 =====
16
17 To enable the magics below, execute ``%load_ext cythonmagic``.
18
19 ``%%cython``
20
21 {CYTHON_DOC}
22
23 ``%%cython_inline``
24
25 {CYTHON_INLINE_DOC}
26
27 ``%%cython_pyximport``
28
29 {CYTHON_PYXIMPORT_DOC}
30
31 Author:
32 * Brian Granger
33
34 Parts of this code were taken from Cython.inline.
6 cf github `cython` organisation, `cython` repo, under the
7 file `Cython/Build/IpythonMagic.py`
35 8 """
36 9 #-----------------------------------------------------------------------------
37 10 # Copyright (C) 2010-2011, IPython Development Team.
38 11 #
39 12 # Distributed under the terms of the Modified BSD License.
40 13 #
41 14 # The full license is in the file COPYING.txt, distributed with this software.
42 15 #-----------------------------------------------------------------------------
43 16
44 17 from __future__ import print_function
45 18
46 import imp
47 import io
48 import os
49 import re
50 import sys
51 import time
52
53 19 try:
54 reload
55 except NameError: # Python 3
56 from imp import reload
20 import Cython
21 except:
22 Cython = None
57 23
58 24 try:
59 import hashlib
60 except ImportError:
61 import md5 as hashlib
62
63 from distutils.core import Distribution, Extension
64 from distutils.command.build_ext import build_ext
65
66 from IPython.core import display
67 from IPython.core import magic_arguments
68 from IPython.core.magic import Magics, magics_class, cell_magic
69 from IPython.utils import py3compat
70 from IPython.utils.path import get_ipython_cache_dir
71 from IPython.utils.text import dedent
72
73 import Cython
74 from Cython.Compiler.Errors import CompileError
75 from Cython.Build.Dependencies import cythonize
76
77
78 @magics_class
79 class CythonMagics(Magics):
80
81 def __init__(self, shell):
82 super(CythonMagics,self).__init__(shell)
83 self._reloads = {}
84 self._code_cache = {}
85
86 def _import_all(self, module):
87 for k,v in module.__dict__.items():
88 if not k.startswith('__'):
89 self.shell.push({k:v})
90
91 @cell_magic
92 def cython_inline(self, line, cell):
93 """Compile and run a Cython code cell using Cython.inline.
94
95 This magic simply passes the body of the cell to Cython.inline
96 and returns the result. If the variables `a` and `b` are defined
97 in the user's namespace, here is a simple example that returns
98 their sum::
99
100 %%cython_inline
101 return a+b
102
103 For most purposes, we recommend the usage of the `%%cython` magic.
104 """
105 locs = self.shell.user_global_ns
106 globs = self.shell.user_ns
107 return Cython.inline(cell, locals=locs, globals=globs)
108
109 @cell_magic
110 def cython_pyximport(self, line, cell):
111 """Compile and import a Cython code cell using pyximport.
25 from Cython.Build.IpythonMagic import CythonMagics
26 except :
27 pass
112 28
113 The contents of the cell are written to a `.pyx` file in the current
114 working directory, which is then imported using `pyximport`. This
115 magic requires a module name to be passed::
116
117 %%cython_pyximport modulename
118 def f(x):
119 return 2.0*x
120
121 The compiled module is then imported and all of its symbols are
122 injected into the user's namespace. For most purposes, we recommend
123 the usage of the `%%cython` magic.
124 """
125 module_name = line.strip()
126 if not module_name:
127 raise ValueError('module name must be given')
128 fname = module_name + '.pyx'
129 with io.open(fname, 'w', encoding='utf-8') as f:
130 f.write(cell)
131 if 'pyximport' not in sys.modules:
132 import pyximport
133 pyximport.install(reload_support=True)
134 if module_name in self._reloads:
135 module = self._reloads[module_name]
136 reload(module)
137 else:
138 __import__(module_name)
139 module = sys.modules[module_name]
140 self._reloads[module_name] = module
141 self._import_all(module)
142
143 @magic_arguments.magic_arguments()
144 @magic_arguments.argument(
145 '-c', '--compile-args', action='append', default=[],
146 help="Extra flags to pass to compiler via the `extra_compile_args` "
147 "Extension flag (can be specified multiple times)."
148 )
149 @magic_arguments.argument(
150 '--link-args', action='append', default=[],
151 help="Extra flags to pass to linker via the `extra_link_args` "
152 "Extension flag (can be specified multiple times)."
153 )
154 @magic_arguments.argument(
155 '-l', '--lib', action='append', default=[],
156 help="Add a library to link the extension against (can be specified "
157 "multiple times)."
158 )
159 @magic_arguments.argument(
160 '-n', '--name',
161 help="Specify a name for the Cython module."
162 )
163 @magic_arguments.argument(
164 '-L', dest='library_dirs', metavar='dir', action='append', default=[],
165 help="Add a path to the list of libary directories (can be specified "
166 "multiple times)."
167 )
168 @magic_arguments.argument(
169 '-I', '--include', action='append', default=[],
170 help="Add a path to the list of include directories (can be specified "
171 "multiple times)."
172 )
173 @magic_arguments.argument(
174 '-+', '--cplus', action='store_true', default=False,
175 help="Output a C++ rather than C file."
176 )
177 @magic_arguments.argument(
178 '-f', '--force', action='store_true', default=False,
179 help="Force the compilation of a new module, even if the source has been "
180 "previously compiled."
181 )
182 @magic_arguments.argument(
183 '-a', '--annotate', action='store_true', default=False,
184 help="Produce a colorized HTML version of the source."
185 )
186 @cell_magic
187 def cython(self, line, cell):
188 """Compile and import everything from a Cython code cell.
189
190 The contents of the cell are written to a `.pyx` file in the
191 directory `IPYTHONDIR/cython` using a filename with the hash of the
192 code. This file is then cythonized and compiled. The resulting module
193 is imported and all of its symbols are injected into the user's
194 namespace. The usage is similar to that of `%%cython_pyximport` but
195 you don't have to pass a module name::
196
197 %%cython
198 def f(x):
199 return 2.0*x
200
201 To compile OpenMP codes, pass the required `--compile-args`
202 and `--link-args`. For example with gcc::
203
204 %%cython --compile-args=-fopenmp --link-args=-fopenmp
205 ...
206 """
207 args = magic_arguments.parse_argstring(self.cython, line)
208 code = cell if cell.endswith('\n') else cell+'\n'
209 lib_dir = os.path.join(get_ipython_cache_dir(), 'cython')
210 quiet = True
211 key = code, sys.version_info, sys.executable, Cython.__version__
212
213 if not os.path.exists(lib_dir):
214 os.makedirs(lib_dir)
215
216 if args.force:
217 # Force a new module name by adding the current time to the
218 # key which is hashed to determine the module name.
219 key += time.time(),
220
221 if args.name:
222 module_name = py3compat.unicode_to_str(args.name)
223 else:
224 module_name = "_cython_magic_" + hashlib.md5(str(key).encode('utf-8')).hexdigest()
225 module_path = os.path.join(lib_dir, module_name + self.so_ext)
226
227 have_module = os.path.isfile(module_path)
228 need_cythonize = not have_module
229
230 if args.annotate:
231 html_file = os.path.join(lib_dir, module_name + '.html')
232 if not os.path.isfile(html_file):
233 need_cythonize = True
234
235 if need_cythonize:
236 c_include_dirs = args.include
237 if 'numpy' in code:
238 import numpy
239 c_include_dirs.append(numpy.get_include())
240 pyx_file = os.path.join(lib_dir, module_name + '.pyx')
241 pyx_file = py3compat.cast_bytes_py2(pyx_file, encoding=sys.getfilesystemencoding())
242 with io.open(pyx_file, 'w', encoding='utf-8') as f:
243 f.write(code)
244 extension = Extension(
245 name = module_name,
246 sources = [pyx_file],
247 include_dirs = c_include_dirs,
248 library_dirs = args.library_dirs,
249 extra_compile_args = args.compile_args,
250 extra_link_args = args.link_args,
251 libraries = args.lib,
252 language = 'c++' if args.cplus else 'c',
253 )
254 build_extension = self._get_build_extension()
255 try:
256 opts = dict(
257 quiet=quiet,
258 annotate = args.annotate,
259 force = True,
260 )
261 build_extension.extensions = cythonize([extension], **opts)
262 except CompileError:
263 return
264
265 if not have_module:
266 build_extension.build_temp = os.path.dirname(pyx_file)
267 build_extension.build_lib = lib_dir
268 build_extension.run()
269 self._code_cache[key] = module_name
270
271 module = imp.load_dynamic(module_name, module_path)
272 self._import_all(module)
273
274 if args.annotate:
275 try:
276 with io.open(html_file, encoding='utf-8') as f:
277 annotated_html = f.read()
278 except IOError as e:
279 # File could not be opened. Most likely the user has a version
280 # of Cython before 0.15.1 (when `cythonize` learned the
281 # `force` keyword argument) and has already compiled this
282 # exact source without annotation.
283 print('Cython completed successfully but the annotated '
284 'source could not be read.', file=sys.stderr)
285 print(e, file=sys.stderr)
286 else:
287 return display.HTML(self.clean_annotated_html(annotated_html))
288
289 @property
290 def so_ext(self):
291 """The extension suffix for compiled modules."""
292 try:
293 return self._so_ext
294 except AttributeError:
295 self._so_ext = self._get_build_extension().get_ext_filename('')
296 return self._so_ext
297
298 def _clear_distutils_mkpath_cache(self):
299 """clear distutils mkpath cache
300
301 prevents distutils from skipping re-creation of dirs that have been removed
302 """
303 try:
304 from distutils.dir_util import _path_created
305 except ImportError:
306 pass
307 else:
308 _path_created.clear()
309
310 def _get_build_extension(self):
311 self._clear_distutils_mkpath_cache()
312 dist = Distribution()
313 config_files = dist.find_config_files()
314 try:
315 config_files.remove('setup.cfg')
316 except ValueError:
317 pass
318 dist.parse_config_files(config_files)
319 build_extension = build_ext(dist)
320 build_extension.finalize_options()
321 return build_extension
322
323 @staticmethod
324 def clean_annotated_html(html):
325 """Clean up the annotated HTML source.
326
327 Strips the link to the generated C or C++ file, which we do not
328 present to the user.
329 """
330 r = re.compile('<p>Raw output: <a href="(.*)">(.*)</a>')
331 html = '\n'.join(l for l in html.splitlines() if not r.match(l))
332 return html
333
334 __doc__ = __doc__.format(
335 # rST doesn't see the -+ flag as part of an option list, so we
336 # hide it from the module-level docstring.
337 CYTHON_DOC = dedent(CythonMagics.cython.__doc__\
338 .replace('-+, --cplus','--cplus ')),
339 CYTHON_INLINE_DOC = dedent(CythonMagics.cython_inline.__doc__),
340 CYTHON_PYXIMPORT_DOC = dedent(CythonMagics.cython_pyximport.__doc__),
341 )
342 29
30 ## still load the magic in IPython 3.x, remove completely in future versions.
343 31 def load_ipython_extension(ip):
344 32 """Load the extension in IPython."""
345 ip.register_magics(CythonMagics)
33
34 print("""The Cython magic has been move to the Cython package, hence """)
35 print("""`%load_ext cythonmagic` is deprecated; Please use `%load_ext cython` instead.""")
36
37 if Cython is None or tuple(map(int,Cython.__version__.split('.'))) < (0,21) :
38 print("You need Cython version >=0.21 to use the Cython magic")
39 return
40 if CythonMagics:
41 print("""\nThough, because I am nice, I'll still try to load it for you this time.""")
42 ip.register_magics(CythonMagics)
@@ -1,527 +1,523 b''
1 1 # -*- coding: utf-8 -*-
2 2 """IPython Test Suite Runner.
3 3
4 4 This module provides a main entry point to a user script to test IPython
5 5 itself from the command line. There are two ways of running this script:
6 6
7 7 1. With the syntax `iptest all`. This runs our entire test suite by
8 8 calling this script (with different arguments) recursively. This
9 9 causes modules and package to be tested in different processes, using nose
10 10 or trial where appropriate.
11 11 2. With the regular nose syntax, like `iptest -vvs IPython`. In this form
12 12 the script simply calls nose, but with special command line flags and
13 13 plugins loaded.
14 14
15 15 """
16 16
17 17 #-----------------------------------------------------------------------------
18 18 # Copyright (C) 2009-2011 The IPython Development Team
19 19 #
20 20 # Distributed under the terms of the BSD License. The full license is in
21 21 # the file COPYING, distributed as part of this software.
22 22 #-----------------------------------------------------------------------------
23 23
24 24 #-----------------------------------------------------------------------------
25 25 # Imports
26 26 #-----------------------------------------------------------------------------
27 27 from __future__ import print_function
28 28
29 29 # Stdlib
30 30 import glob
31 31 from io import BytesIO
32 32 import os
33 33 import os.path as path
34 34 import sys
35 35 from threading import Thread, Lock, Event
36 36 import warnings
37 37
38 38 # Now, proceed to import nose itself
39 39 import nose.plugins.builtin
40 40 from nose.plugins.xunit import Xunit
41 41 from nose import SkipTest
42 42 from nose.core import TestProgram
43 43 from nose.plugins import Plugin
44 44 from nose.util import safe_str
45 45
46 46 # Our own imports
47 47 from IPython.utils.process import is_cmd_found
48 48 from IPython.utils.importstring import import_item
49 49 from IPython.testing.plugin.ipdoctest import IPythonDoctest
50 50 from IPython.external.decorators import KnownFailure, knownfailureif
51 51
52 52 pjoin = path.join
53 53
54 54
55 55 #-----------------------------------------------------------------------------
56 56 # Globals
57 57 #-----------------------------------------------------------------------------
58 58
59 59
60 60 #-----------------------------------------------------------------------------
61 61 # Warnings control
62 62 #-----------------------------------------------------------------------------
63 63
64 64 # Twisted generates annoying warnings with Python 2.6, as will do other code
65 65 # that imports 'sets' as of today
66 66 warnings.filterwarnings('ignore', 'the sets module is deprecated',
67 67 DeprecationWarning )
68 68
69 69 # This one also comes from Twisted
70 70 warnings.filterwarnings('ignore', 'the sha module is deprecated',
71 71 DeprecationWarning)
72 72
73 73 # Wx on Fedora11 spits these out
74 74 warnings.filterwarnings('ignore', 'wxPython/wxWidgets release number mismatch',
75 75 UserWarning)
76 76
77 77 # ------------------------------------------------------------------------------
78 78 # Monkeypatch Xunit to count known failures as skipped.
79 79 # ------------------------------------------------------------------------------
80 80 def monkeypatch_xunit():
81 81 try:
82 82 knownfailureif(True)(lambda: None)()
83 83 except Exception as e:
84 84 KnownFailureTest = type(e)
85 85
86 86 def addError(self, test, err, capt=None):
87 87 if issubclass(err[0], KnownFailureTest):
88 88 err = (SkipTest,) + err[1:]
89 89 return self.orig_addError(test, err, capt)
90 90
91 91 Xunit.orig_addError = Xunit.addError
92 92 Xunit.addError = addError
93 93
94 94 #-----------------------------------------------------------------------------
95 95 # Check which dependencies are installed and greater than minimum version.
96 96 #-----------------------------------------------------------------------------
97 97 def extract_version(mod):
98 98 return mod.__version__
99 99
100 100 def test_for(item, min_version=None, callback=extract_version):
101 101 """Test to see if item is importable, and optionally check against a minimum
102 102 version.
103 103
104 104 If min_version is given, the default behavior is to check against the
105 105 `__version__` attribute of the item, but specifying `callback` allows you to
106 106 extract the value you are interested in. e.g::
107 107
108 108 In [1]: import sys
109 109
110 110 In [2]: from IPython.testing.iptest import test_for
111 111
112 112 In [3]: test_for('sys', (2,6), callback=lambda sys: sys.version_info)
113 113 Out[3]: True
114 114
115 115 """
116 116 try:
117 117 check = import_item(item)
118 118 except (ImportError, RuntimeError):
119 119 # GTK reports Runtime error if it can't be initialized even if it's
120 120 # importable.
121 121 return False
122 122 else:
123 123 if min_version:
124 124 if callback:
125 125 # extra processing step to get version to compare
126 126 check = callback(check)
127 127
128 128 return check >= min_version
129 129 else:
130 130 return True
131 131
132 132 # Global dict where we can store information on what we have and what we don't
133 133 # have available at test run time
134 134 have = {}
135 135
136 136 have['curses'] = test_for('_curses')
137 137 have['matplotlib'] = test_for('matplotlib')
138 138 have['numpy'] = test_for('numpy')
139 139 have['pexpect'] = test_for('IPython.external.pexpect')
140 140 have['pymongo'] = test_for('pymongo')
141 141 have['pygments'] = test_for('pygments')
142 142 have['qt'] = test_for('IPython.external.qt')
143 143 have['sqlite3'] = test_for('sqlite3')
144 have['cython'] = test_for('Cython')
145 144 have['tornado'] = test_for('tornado.version_info', (3,1,0), callback=None)
146 145 have['jinja2'] = test_for('jinja2')
147 146 have['mistune'] = test_for('mistune')
148 147 have['requests'] = test_for('requests')
149 148 have['sphinx'] = test_for('sphinx')
150 149 have['jsonschema'] = test_for('jsonschema')
151 150 have['jsonpointer'] = test_for('jsonpointer')
152 151 have['casperjs'] = is_cmd_found('casperjs')
153 152 have['phantomjs'] = is_cmd_found('phantomjs')
154 153 have['slimerjs'] = is_cmd_found('slimerjs')
155 154
156 155 min_zmq = (2,1,11)
157 156
158 157 have['zmq'] = test_for('zmq.pyzmq_version_info', min_zmq, callback=lambda x: x())
159 158
160 159 #-----------------------------------------------------------------------------
161 160 # Test suite definitions
162 161 #-----------------------------------------------------------------------------
163 162
164 163 test_group_names = ['parallel', 'kernel', 'kernel.inprocess', 'config', 'core',
165 164 'extensions', 'lib', 'terminal', 'testing', 'utils',
166 165 'nbformat', 'qt', 'html', 'nbconvert'
167 166 ]
168 167
169 168 class TestSection(object):
170 169 def __init__(self, name, includes):
171 170 self.name = name
172 171 self.includes = includes
173 172 self.excludes = []
174 173 self.dependencies = []
175 174 self.enabled = True
176 175
177 176 def exclude(self, module):
178 177 if not module.startswith('IPython'):
179 178 module = self.includes[0] + "." + module
180 179 self.excludes.append(module.replace('.', os.sep))
181 180
182 181 def requires(self, *packages):
183 182 self.dependencies.extend(packages)
184 183
185 184 @property
186 185 def will_run(self):
187 186 return self.enabled and all(have[p] for p in self.dependencies)
188 187
189 188 # Name -> (include, exclude, dependencies_met)
190 189 test_sections = {n:TestSection(n, ['IPython.%s' % n]) for n in test_group_names}
191 190
192 191 # Exclusions and dependencies
193 192 # ---------------------------
194 193
195 194 # core:
196 195 sec = test_sections['core']
197 196 if not have['sqlite3']:
198 197 sec.exclude('tests.test_history')
199 198 sec.exclude('history')
200 199 if not have['matplotlib']:
201 200 sec.exclude('pylabtools'),
202 201 sec.exclude('tests.test_pylabtools')
203 202
204 203 # lib:
205 204 sec = test_sections['lib']
206 205 if not have['zmq']:
207 206 sec.exclude('kernel')
208 207 # We do this unconditionally, so that the test suite doesn't import
209 208 # gtk, changing the default encoding and masking some unicode bugs.
210 209 sec.exclude('inputhookgtk')
211 210 # We also do this unconditionally, because wx can interfere with Unix signals.
212 211 # There are currently no tests for it anyway.
213 212 sec.exclude('inputhookwx')
214 213 # Testing inputhook will need a lot of thought, to figure out
215 214 # how to have tests that don't lock up with the gui event
216 215 # loops in the picture
217 216 sec.exclude('inputhook')
218 217
219 218 # testing:
220 219 sec = test_sections['testing']
221 220 # These have to be skipped on win32 because they use echo, rm, cd, etc.
222 221 # See ticket https://github.com/ipython/ipython/issues/87
223 222 if sys.platform == 'win32':
224 223 sec.exclude('plugin.test_exampleip')
225 224 sec.exclude('plugin.dtexample')
226 225
227 226 # terminal:
228 227 if (not have['pexpect']) or (not have['zmq']):
229 228 test_sections['terminal'].exclude('console')
230 229
231 230 # parallel
232 231 sec = test_sections['parallel']
233 232 sec.requires('zmq')
234 233 if not have['pymongo']:
235 234 sec.exclude('controller.mongodb')
236 235 sec.exclude('tests.test_mongodb')
237 236
238 237 # kernel:
239 238 sec = test_sections['kernel']
240 239 sec.requires('zmq')
241 240 # The in-process kernel tests are done in a separate section
242 241 sec.exclude('inprocess')
243 242 # importing gtk sets the default encoding, which we want to avoid
244 243 sec.exclude('zmq.gui.gtkembed')
245 244 sec.exclude('zmq.gui.gtk3embed')
246 245 if not have['matplotlib']:
247 246 sec.exclude('zmq.pylab')
248 247
249 248 # kernel.inprocess:
250 249 test_sections['kernel.inprocess'].requires('zmq')
251 250
252 251 # extensions:
253 252 sec = test_sections['extensions']
254 if not have['cython']:
255 sec.exclude('cythonmagic')
256 sec.exclude('tests.test_cythonmagic')
257 253 # This is deprecated in favour of rpy2
258 254 sec.exclude('rmagic')
259 255 # autoreload does some strange stuff, so move it to its own test section
260 256 sec.exclude('autoreload')
261 257 sec.exclude('tests.test_autoreload')
262 258 test_sections['autoreload'] = TestSection('autoreload',
263 259 ['IPython.extensions.autoreload', 'IPython.extensions.tests.test_autoreload'])
264 260 test_group_names.append('autoreload')
265 261
266 262 # qt:
267 263 test_sections['qt'].requires('zmq', 'qt', 'pygments')
268 264
269 265 # html:
270 266 sec = test_sections['html']
271 267 sec.requires('zmq', 'tornado', 'requests', 'sqlite3', 'jsonschema', 'jsonpointer')
272 268 # The notebook 'static' directory contains JS, css and other
273 269 # files for web serving. Occasionally projects may put a .py
274 270 # file in there (MathJax ships a conf.py), so we might as
275 271 # well play it safe and skip the whole thing.
276 272 sec.exclude('static')
277 273 sec.exclude('fabfile')
278 274 if not have['jinja2']:
279 275 sec.exclude('notebookapp')
280 276 if not have['pygments'] or not have['jinja2']:
281 277 sec.exclude('nbconvert')
282 278
283 279 # config:
284 280 # Config files aren't really importable stand-alone
285 281 test_sections['config'].exclude('profile')
286 282
287 283 # nbconvert:
288 284 sec = test_sections['nbconvert']
289 285 sec.requires('pygments', 'jinja2', 'jsonschema', 'jsonpointer', 'mistune')
290 286 # Exclude nbconvert directories containing config files used to test.
291 287 # Executing the config files with iptest would cause an exception.
292 288 sec.exclude('tests.files')
293 289 sec.exclude('exporters.tests.files')
294 290 if not have['tornado']:
295 291 sec.exclude('nbconvert.post_processors.serve')
296 292 sec.exclude('nbconvert.post_processors.tests.test_serve')
297 293
298 294 # nbformat:
299 295 test_sections['nbformat'].requires('jsonschema', 'jsonpointer')
300 296
301 297 #-----------------------------------------------------------------------------
302 298 # Functions and classes
303 299 #-----------------------------------------------------------------------------
304 300
305 301 def check_exclusions_exist():
306 302 from IPython.utils.path import get_ipython_package_dir
307 303 from IPython.utils.warn import warn
308 304 parent = os.path.dirname(get_ipython_package_dir())
309 305 for sec in test_sections:
310 306 for pattern in sec.exclusions:
311 307 fullpath = pjoin(parent, pattern)
312 308 if not os.path.exists(fullpath) and not glob.glob(fullpath + '.*'):
313 309 warn("Excluding nonexistent file: %r" % pattern)
314 310
315 311
316 312 class ExclusionPlugin(Plugin):
317 313 """A nose plugin to effect our exclusions of files and directories.
318 314 """
319 315 name = 'exclusions'
320 316 score = 3000 # Should come before any other plugins
321 317
322 318 def __init__(self, exclude_patterns=None):
323 319 """
324 320 Parameters
325 321 ----------
326 322
327 323 exclude_patterns : sequence of strings, optional
328 324 Filenames containing these patterns (as raw strings, not as regular
329 325 expressions) are excluded from the tests.
330 326 """
331 327 self.exclude_patterns = exclude_patterns or []
332 328 super(ExclusionPlugin, self).__init__()
333 329
334 330 def options(self, parser, env=os.environ):
335 331 Plugin.options(self, parser, env)
336 332
337 333 def configure(self, options, config):
338 334 Plugin.configure(self, options, config)
339 335 # Override nose trying to disable plugin.
340 336 self.enabled = True
341 337
342 338 def wantFile(self, filename):
343 339 """Return whether the given filename should be scanned for tests.
344 340 """
345 341 if any(pat in filename for pat in self.exclude_patterns):
346 342 return False
347 343 return None
348 344
349 345 def wantDirectory(self, directory):
350 346 """Return whether the given directory should be scanned for tests.
351 347 """
352 348 if any(pat in directory for pat in self.exclude_patterns):
353 349 return False
354 350 return None
355 351
356 352
357 353 class StreamCapturer(Thread):
358 354 daemon = True # Don't hang if main thread crashes
359 355 started = False
360 356 def __init__(self):
361 357 super(StreamCapturer, self).__init__()
362 358 self.streams = []
363 359 self.buffer = BytesIO()
364 360 self.readfd, self.writefd = os.pipe()
365 361 self.buffer_lock = Lock()
366 362 self.stop = Event()
367 363
368 364 def run(self):
369 365 self.started = True
370 366
371 367 while not self.stop.is_set():
372 368 chunk = os.read(self.readfd, 1024)
373 369
374 370 with self.buffer_lock:
375 371 self.buffer.write(chunk)
376 372
377 373 os.close(self.readfd)
378 374 os.close(self.writefd)
379 375
380 376 def reset_buffer(self):
381 377 with self.buffer_lock:
382 378 self.buffer.truncate(0)
383 379 self.buffer.seek(0)
384 380
385 381 def get_buffer(self):
386 382 with self.buffer_lock:
387 383 return self.buffer.getvalue()
388 384
389 385 def ensure_started(self):
390 386 if not self.started:
391 387 self.start()
392 388
393 389 def halt(self):
394 390 """Safely stop the thread."""
395 391 if not self.started:
396 392 return
397 393
398 394 self.stop.set()
399 395 os.write(self.writefd, b'wake up') # Ensure we're not locked in a read()
400 396 self.join()
401 397
402 398 class SubprocessStreamCapturePlugin(Plugin):
403 399 name='subprocstreams'
404 400 def __init__(self):
405 401 Plugin.__init__(self)
406 402 self.stream_capturer = StreamCapturer()
407 403 self.destination = os.environ.get('IPTEST_SUBPROC_STREAMS', 'capture')
408 404 # This is ugly, but distant parts of the test machinery need to be able
409 405 # to redirect streams, so we make the object globally accessible.
410 406 nose.iptest_stdstreams_fileno = self.get_write_fileno
411 407
412 408 def get_write_fileno(self):
413 409 if self.destination == 'capture':
414 410 self.stream_capturer.ensure_started()
415 411 return self.stream_capturer.writefd
416 412 elif self.destination == 'discard':
417 413 return os.open(os.devnull, os.O_WRONLY)
418 414 else:
419 415 return sys.__stdout__.fileno()
420 416
421 417 def configure(self, options, config):
422 418 Plugin.configure(self, options, config)
423 419 # Override nose trying to disable plugin.
424 420 if self.destination == 'capture':
425 421 self.enabled = True
426 422
427 423 def startTest(self, test):
428 424 # Reset log capture
429 425 self.stream_capturer.reset_buffer()
430 426
431 427 def formatFailure(self, test, err):
432 428 # Show output
433 429 ec, ev, tb = err
434 430 captured = self.stream_capturer.get_buffer().decode('utf-8', 'replace')
435 431 if captured.strip():
436 432 ev = safe_str(ev)
437 433 out = [ev, '>> begin captured subprocess output <<',
438 434 captured,
439 435 '>> end captured subprocess output <<']
440 436 return ec, '\n'.join(out), tb
441 437
442 438 return err
443 439
444 440 formatError = formatFailure
445 441
446 442 def finalize(self, result):
447 443 self.stream_capturer.halt()
448 444
449 445
450 446 def run_iptest():
451 447 """Run the IPython test suite using nose.
452 448
453 449 This function is called when this script is **not** called with the form
454 450 `iptest all`. It simply calls nose with appropriate command line flags
455 451 and accepts all of the standard nose arguments.
456 452 """
457 453 # Apply our monkeypatch to Xunit
458 454 if '--with-xunit' in sys.argv and not hasattr(Xunit, 'orig_addError'):
459 455 monkeypatch_xunit()
460 456
461 457 warnings.filterwarnings('ignore',
462 458 'This will be removed soon. Use IPython.testing.util instead')
463 459
464 460 arg1 = sys.argv[1]
465 461 if arg1 in test_sections:
466 462 section = test_sections[arg1]
467 463 sys.argv[1:2] = section.includes
468 464 elif arg1.startswith('IPython.') and arg1[8:] in test_sections:
469 465 section = test_sections[arg1[8:]]
470 466 sys.argv[1:2] = section.includes
471 467 else:
472 468 section = TestSection(arg1, includes=[arg1])
473 469
474 470
475 471 argv = sys.argv + [ '--detailed-errors', # extra info in tracebacks
476 472
477 473 '--with-ipdoctest',
478 474 '--ipdoctest-tests','--ipdoctest-extension=txt',
479 475
480 476 # We add --exe because of setuptools' imbecility (it
481 477 # blindly does chmod +x on ALL files). Nose does the
482 478 # right thing and it tries to avoid executables,
483 479 # setuptools unfortunately forces our hand here. This
484 480 # has been discussed on the distutils list and the
485 481 # setuptools devs refuse to fix this problem!
486 482 '--exe',
487 483 ]
488 484 if '-a' not in argv and '-A' not in argv:
489 485 argv = argv + ['-a', '!crash']
490 486
491 487 if nose.__version__ >= '0.11':
492 488 # I don't fully understand why we need this one, but depending on what
493 489 # directory the test suite is run from, if we don't give it, 0 tests
494 490 # get run. Specifically, if the test suite is run from the source dir
495 491 # with an argument (like 'iptest.py IPython.core', 0 tests are run,
496 492 # even if the same call done in this directory works fine). It appears
497 493 # that if the requested package is in the current dir, nose bails early
498 494 # by default. Since it's otherwise harmless, leave it in by default
499 495 # for nose >= 0.11, though unfortunately nose 0.10 doesn't support it.
500 496 argv.append('--traverse-namespace')
501 497
502 498 # use our plugin for doctesting. It will remove the standard doctest plugin
503 499 # if it finds it enabled
504 500 plugins = [ExclusionPlugin(section.excludes), IPythonDoctest(), KnownFailure(),
505 501 SubprocessStreamCapturePlugin() ]
506 502
507 503 # Use working directory set by parent process (see iptestcontroller)
508 504 if 'IPTEST_WORKING_DIR' in os.environ:
509 505 os.chdir(os.environ['IPTEST_WORKING_DIR'])
510 506
511 507 # We need a global ipython running in this process, but the special
512 508 # in-process group spawns its own IPython kernels, so for *that* group we
513 509 # must avoid also opening the global one (otherwise there's a conflict of
514 510 # singletons). Ultimately the solution to this problem is to refactor our
515 511 # assumptions about what needs to be a singleton and what doesn't (app
516 512 # objects should, individual shells shouldn't). But for now, this
517 513 # workaround allows the test suite for the inprocess module to complete.
518 514 if 'kernel.inprocess' not in section.name:
519 515 from IPython.testing import globalipapp
520 516 globalipapp.start_ipython()
521 517
522 518 # Now nose can run
523 519 TestProgram(argv=argv, addplugins=plugins)
524 520
525 521 if __name__ == '__main__':
526 522 run_iptest()
527 523
1 NO CONTENT: file was removed
General Comments 0
You need to be logged in to leave comments. Login now