##// END OF EJS Templates
Merge pull request #6510 from Carreau/removecython...
Thomas Kluyver -
r18304:dbd30bfa merge
parent child Browse files
Show More
@@ -0,0 +1,1 b''
1 * The ``%cython`` magic, is now part of the Cython module. Use `%load_ext Cython` with a version of Cython >= 0.21 to have access to the magic now.
@@ -1,345 +1,43 b''
1 1 # -*- coding: utf-8 -*-
2 2 """
3 =====================
4 Cython related magics
5 =====================
3 The cython magic has been integrated into Cython itself,
4 which is now released in version 0.21.
6 5
7 Magic command interface for interactive work with Cython
8
9 .. note::
10
11 The ``Cython`` package needs to be installed separately. It
12 can be obtained using ``easy_install`` or ``pip``.
13
14 Usage
15 =====
16
17 To enable the magics below, execute ``%load_ext cythonmagic``.
18
19 ``%%cython``
20
21 {CYTHON_DOC}
22
23 ``%%cython_inline``
24
25 {CYTHON_INLINE_DOC}
26
27 ``%%cython_pyximport``
28
29 {CYTHON_PYXIMPORT_DOC}
30
31 Author:
32 * Brian Granger
33
34 Parts of this code were taken from Cython.inline.
6 cf github `Cython` organisation, `Cython` repo, under the
7 file `Cython/Build/IpythonMagic.py`
35 8 """
36 9 #-----------------------------------------------------------------------------
37 10 # Copyright (C) 2010-2011, IPython Development Team.
38 11 #
39 12 # Distributed under the terms of the Modified BSD License.
40 13 #
41 14 # The full license is in the file COPYING.txt, distributed with this software.
42 15 #-----------------------------------------------------------------------------
43 16
44 17 from __future__ import print_function
45 18
46 import imp
47 import io
48 import os
49 import re
50 import sys
51 import time
19 import IPython.utils.version as version
52 20
53 21 try:
54 reload
55 except NameError: # Python 3
56 from imp import reload
22 import Cython
23 except:
24 Cython = None
57 25
58 26 try:
59 import hashlib
60 except ImportError:
61 import md5 as hashlib
62
63 from distutils.core import Distribution, Extension
64 from distutils.command.build_ext import build_ext
65
66 from IPython.core import display
67 from IPython.core import magic_arguments
68 from IPython.core.magic import Magics, magics_class, cell_magic
69 from IPython.utils import py3compat
70 from IPython.utils.path import get_ipython_cache_dir
71 from IPython.utils.text import dedent
72
73 import Cython
74 from Cython.Compiler.Errors import CompileError
75 from Cython.Build.Dependencies import cythonize
76
77
78 @magics_class
79 class CythonMagics(Magics):
80
81 def __init__(self, shell):
82 super(CythonMagics,self).__init__(shell)
83 self._reloads = {}
84 self._code_cache = {}
85
86 def _import_all(self, module):
87 for k,v in module.__dict__.items():
88 if not k.startswith('__'):
89 self.shell.push({k:v})
90
91 @cell_magic
92 def cython_inline(self, line, cell):
93 """Compile and run a Cython code cell using Cython.inline.
94
95 This magic simply passes the body of the cell to Cython.inline
96 and returns the result. If the variables `a` and `b` are defined
97 in the user's namespace, here is a simple example that returns
98 their sum::
99
100 %%cython_inline
101 return a+b
102
103 For most purposes, we recommend the usage of the `%%cython` magic.
104 """
105 locs = self.shell.user_global_ns
106 globs = self.shell.user_ns
107 return Cython.inline(cell, locals=locs, globals=globs)
108
109 @cell_magic
110 def cython_pyximport(self, line, cell):
111 """Compile and import a Cython code cell using pyximport.
27 from Cython.Build.IpythonMagic import CythonMagics
28 except :
29 pass
112 30
113 The contents of the cell are written to a `.pyx` file in the current
114 working directory, which is then imported using `pyximport`. This
115 magic requires a module name to be passed::
116
117 %%cython_pyximport modulename
118 def f(x):
119 return 2.0*x
120
121 The compiled module is then imported and all of its symbols are
122 injected into the user's namespace. For most purposes, we recommend
123 the usage of the `%%cython` magic.
124 """
125 module_name = line.strip()
126 if not module_name:
127 raise ValueError('module name must be given')
128 fname = module_name + '.pyx'
129 with io.open(fname, 'w', encoding='utf-8') as f:
130 f.write(cell)
131 if 'pyximport' not in sys.modules:
132 import pyximport
133 pyximport.install(reload_support=True)
134 if module_name in self._reloads:
135 module = self._reloads[module_name]
136 reload(module)
137 else:
138 __import__(module_name)
139 module = sys.modules[module_name]
140 self._reloads[module_name] = module
141 self._import_all(module)
142
143 @magic_arguments.magic_arguments()
144 @magic_arguments.argument(
145 '-c', '--compile-args', action='append', default=[],
146 help="Extra flags to pass to compiler via the `extra_compile_args` "
147 "Extension flag (can be specified multiple times)."
148 )
149 @magic_arguments.argument(
150 '--link-args', action='append', default=[],
151 help="Extra flags to pass to linker via the `extra_link_args` "
152 "Extension flag (can be specified multiple times)."
153 )
154 @magic_arguments.argument(
155 '-l', '--lib', action='append', default=[],
156 help="Add a library to link the extension against (can be specified "
157 "multiple times)."
158 )
159 @magic_arguments.argument(
160 '-n', '--name',
161 help="Specify a name for the Cython module."
162 )
163 @magic_arguments.argument(
164 '-L', dest='library_dirs', metavar='dir', action='append', default=[],
165 help="Add a path to the list of libary directories (can be specified "
166 "multiple times)."
167 )
168 @magic_arguments.argument(
169 '-I', '--include', action='append', default=[],
170 help="Add a path to the list of include directories (can be specified "
171 "multiple times)."
172 )
173 @magic_arguments.argument(
174 '-+', '--cplus', action='store_true', default=False,
175 help="Output a C++ rather than C file."
176 )
177 @magic_arguments.argument(
178 '-f', '--force', action='store_true', default=False,
179 help="Force the compilation of a new module, even if the source has been "
180 "previously compiled."
181 )
182 @magic_arguments.argument(
183 '-a', '--annotate', action='store_true', default=False,
184 help="Produce a colorized HTML version of the source."
185 )
186 @cell_magic
187 def cython(self, line, cell):
188 """Compile and import everything from a Cython code cell.
189
190 The contents of the cell are written to a `.pyx` file in the
191 directory `IPYTHONDIR/cython` using a filename with the hash of the
192 code. This file is then cythonized and compiled. The resulting module
193 is imported and all of its symbols are injected into the user's
194 namespace. The usage is similar to that of `%%cython_pyximport` but
195 you don't have to pass a module name::
196
197 %%cython
198 def f(x):
199 return 2.0*x
200
201 To compile OpenMP codes, pass the required `--compile-args`
202 and `--link-args`. For example with gcc::
203
204 %%cython --compile-args=-fopenmp --link-args=-fopenmp
205 ...
206 """
207 args = magic_arguments.parse_argstring(self.cython, line)
208 code = cell if cell.endswith('\n') else cell+'\n'
209 lib_dir = os.path.join(get_ipython_cache_dir(), 'cython')
210 quiet = True
211 key = code, sys.version_info, sys.executable, Cython.__version__
212
213 if not os.path.exists(lib_dir):
214 os.makedirs(lib_dir)
215
216 if args.force:
217 # Force a new module name by adding the current time to the
218 # key which is hashed to determine the module name.
219 key += time.time(),
220
221 if args.name:
222 module_name = py3compat.unicode_to_str(args.name)
223 else:
224 module_name = "_cython_magic_" + hashlib.md5(str(key).encode('utf-8')).hexdigest()
225 module_path = os.path.join(lib_dir, module_name + self.so_ext)
226
227 have_module = os.path.isfile(module_path)
228 need_cythonize = not have_module
229
230 if args.annotate:
231 html_file = os.path.join(lib_dir, module_name + '.html')
232 if not os.path.isfile(html_file):
233 need_cythonize = True
234
235 if need_cythonize:
236 c_include_dirs = args.include
237 if 'numpy' in code:
238 import numpy
239 c_include_dirs.append(numpy.get_include())
240 pyx_file = os.path.join(lib_dir, module_name + '.pyx')
241 pyx_file = py3compat.cast_bytes_py2(pyx_file, encoding=sys.getfilesystemencoding())
242 with io.open(pyx_file, 'w', encoding='utf-8') as f:
243 f.write(code)
244 extension = Extension(
245 name = module_name,
246 sources = [pyx_file],
247 include_dirs = c_include_dirs,
248 library_dirs = args.library_dirs,
249 extra_compile_args = args.compile_args,
250 extra_link_args = args.link_args,
251 libraries = args.lib,
252 language = 'c++' if args.cplus else 'c',
253 )
254 build_extension = self._get_build_extension()
255 try:
256 opts = dict(
257 quiet=quiet,
258 annotate = args.annotate,
259 force = True,
260 )
261 build_extension.extensions = cythonize([extension], **opts)
262 except CompileError:
263 return
264
265 if not have_module:
266 build_extension.build_temp = os.path.dirname(pyx_file)
267 build_extension.build_lib = lib_dir
268 build_extension.run()
269 self._code_cache[key] = module_name
270
271 module = imp.load_dynamic(module_name, module_path)
272 self._import_all(module)
273
274 if args.annotate:
275 try:
276 with io.open(html_file, encoding='utf-8') as f:
277 annotated_html = f.read()
278 except IOError as e:
279 # File could not be opened. Most likely the user has a version
280 # of Cython before 0.15.1 (when `cythonize` learned the
281 # `force` keyword argument) and has already compiled this
282 # exact source without annotation.
283 print('Cython completed successfully but the annotated '
284 'source could not be read.', file=sys.stderr)
285 print(e, file=sys.stderr)
286 else:
287 return display.HTML(self.clean_annotated_html(annotated_html))
288
289 @property
290 def so_ext(self):
291 """The extension suffix for compiled modules."""
292 try:
293 return self._so_ext
294 except AttributeError:
295 self._so_ext = self._get_build_extension().get_ext_filename('')
296 return self._so_ext
297
298 def _clear_distutils_mkpath_cache(self):
299 """clear distutils mkpath cache
300
301 prevents distutils from skipping re-creation of dirs that have been removed
302 """
303 try:
304 from distutils.dir_util import _path_created
305 except ImportError:
306 pass
307 else:
308 _path_created.clear()
309
310 def _get_build_extension(self):
311 self._clear_distutils_mkpath_cache()
312 dist = Distribution()
313 config_files = dist.find_config_files()
314 try:
315 config_files.remove('setup.cfg')
316 except ValueError:
317 pass
318 dist.parse_config_files(config_files)
319 build_extension = build_ext(dist)
320 build_extension.finalize_options()
321 return build_extension
322
323 @staticmethod
324 def clean_annotated_html(html):
325 """Clean up the annotated HTML source.
326
327 Strips the link to the generated C or C++ file, which we do not
328 present to the user.
329 """
330 r = re.compile('<p>Raw output: <a href="(.*)">(.*)</a>')
331 html = '\n'.join(l for l in html.splitlines() if not r.match(l))
332 return html
333
334 __doc__ = __doc__.format(
335 # rST doesn't see the -+ flag as part of an option list, so we
336 # hide it from the module-level docstring.
337 CYTHON_DOC = dedent(CythonMagics.cython.__doc__\
338 .replace('-+, --cplus','--cplus ')),
339 CYTHON_INLINE_DOC = dedent(CythonMagics.cython_inline.__doc__),
340 CYTHON_PYXIMPORT_DOC = dedent(CythonMagics.cython_pyximport.__doc__),
341 )
342 31
32 ## still load the magic in IPython 3.x, remove completely in future versions.
343 33 def load_ipython_extension(ip):
344 34 """Load the extension in IPython."""
345 ip.register_magics(CythonMagics)
35
36 print("""The Cython magic has been move to the Cython package, hence """)
37 print("""`%load_ext cythonmagic` is deprecated; Please use `%load_ext Cython` instead.""")
38
39 if Cython is None or not version.check_version(Cython.__version__, "0.21"):
40 print("You need Cython version >=0.21 to use the Cython magic")
41 return
42 print("""\nThough, because I am nice, I'll still try to load it for you this time.""")
43 Cython.load_ipython_extension(ip)
@@ -1,516 +1,512 b''
1 1 # -*- coding: utf-8 -*-
2 2 """IPython Test Suite Runner.
3 3
4 4 This module provides a main entry point to a user script to test IPython
5 5 itself from the command line. There are two ways of running this script:
6 6
7 7 1. With the syntax `iptest all`. This runs our entire test suite by
8 8 calling this script (with different arguments) recursively. This
9 9 causes modules and package to be tested in different processes, using nose
10 10 or trial where appropriate.
11 11 2. With the regular nose syntax, like `iptest -vvs IPython`. In this form
12 12 the script simply calls nose, but with special command line flags and
13 13 plugins loaded.
14 14
15 15 """
16 16
17 17 # Copyright (c) IPython Development Team.
18 18 # Distributed under the terms of the Modified BSD License.
19 19
20 20 from __future__ import print_function
21 21
22 22 import glob
23 23 from io import BytesIO
24 24 import os
25 25 import os.path as path
26 26 import sys
27 27 from threading import Thread, Lock, Event
28 28 import warnings
29 29
30 30 import nose.plugins.builtin
31 31 from nose.plugins.xunit import Xunit
32 32 from nose import SkipTest
33 33 from nose.core import TestProgram
34 34 from nose.plugins import Plugin
35 35 from nose.util import safe_str
36 36
37 37 from IPython.utils.process import is_cmd_found
38 38 from IPython.utils.importstring import import_item
39 39 from IPython.testing.plugin.ipdoctest import IPythonDoctest
40 40 from IPython.external.decorators import KnownFailure, knownfailureif
41 41
42 42 pjoin = path.join
43 43
44 44
45 45 #-----------------------------------------------------------------------------
46 46 # Globals
47 47 #-----------------------------------------------------------------------------
48 48
49 49
50 50 #-----------------------------------------------------------------------------
51 51 # Warnings control
52 52 #-----------------------------------------------------------------------------
53 53
54 54 # Twisted generates annoying warnings with Python 2.6, as will do other code
55 55 # that imports 'sets' as of today
56 56 warnings.filterwarnings('ignore', 'the sets module is deprecated',
57 57 DeprecationWarning )
58 58
59 59 # This one also comes from Twisted
60 60 warnings.filterwarnings('ignore', 'the sha module is deprecated',
61 61 DeprecationWarning)
62 62
63 63 # Wx on Fedora11 spits these out
64 64 warnings.filterwarnings('ignore', 'wxPython/wxWidgets release number mismatch',
65 65 UserWarning)
66 66
67 67 # ------------------------------------------------------------------------------
68 68 # Monkeypatch Xunit to count known failures as skipped.
69 69 # ------------------------------------------------------------------------------
70 70 def monkeypatch_xunit():
71 71 try:
72 72 knownfailureif(True)(lambda: None)()
73 73 except Exception as e:
74 74 KnownFailureTest = type(e)
75 75
76 76 def addError(self, test, err, capt=None):
77 77 if issubclass(err[0], KnownFailureTest):
78 78 err = (SkipTest,) + err[1:]
79 79 return self.orig_addError(test, err, capt)
80 80
81 81 Xunit.orig_addError = Xunit.addError
82 82 Xunit.addError = addError
83 83
84 84 #-----------------------------------------------------------------------------
85 85 # Check which dependencies are installed and greater than minimum version.
86 86 #-----------------------------------------------------------------------------
87 87 def extract_version(mod):
88 88 return mod.__version__
89 89
90 90 def test_for(item, min_version=None, callback=extract_version):
91 91 """Test to see if item is importable, and optionally check against a minimum
92 92 version.
93 93
94 94 If min_version is given, the default behavior is to check against the
95 95 `__version__` attribute of the item, but specifying `callback` allows you to
96 96 extract the value you are interested in. e.g::
97 97
98 98 In [1]: import sys
99 99
100 100 In [2]: from IPython.testing.iptest import test_for
101 101
102 102 In [3]: test_for('sys', (2,6), callback=lambda sys: sys.version_info)
103 103 Out[3]: True
104 104
105 105 """
106 106 try:
107 107 check = import_item(item)
108 108 except (ImportError, RuntimeError):
109 109 # GTK reports Runtime error if it can't be initialized even if it's
110 110 # importable.
111 111 return False
112 112 else:
113 113 if min_version:
114 114 if callback:
115 115 # extra processing step to get version to compare
116 116 check = callback(check)
117 117
118 118 return check >= min_version
119 119 else:
120 120 return True
121 121
122 122 # Global dict where we can store information on what we have and what we don't
123 123 # have available at test run time
124 124 have = {}
125 125
126 126 have['curses'] = test_for('_curses')
127 127 have['matplotlib'] = test_for('matplotlib')
128 128 have['numpy'] = test_for('numpy')
129 129 have['pexpect'] = test_for('IPython.external.pexpect')
130 130 have['pymongo'] = test_for('pymongo')
131 131 have['pygments'] = test_for('pygments')
132 132 have['qt'] = test_for('IPython.external.qt')
133 133 have['sqlite3'] = test_for('sqlite3')
134 have['cython'] = test_for('Cython')
135 134 have['tornado'] = test_for('tornado.version_info', (3,1,0), callback=None)
136 135 have['jinja2'] = test_for('jinja2')
137 136 have['mistune'] = test_for('mistune')
138 137 have['requests'] = test_for('requests')
139 138 have['sphinx'] = test_for('sphinx')
140 139 have['jsonschema'] = test_for('jsonschema')
141 140 have['casperjs'] = is_cmd_found('casperjs')
142 141 have['phantomjs'] = is_cmd_found('phantomjs')
143 142 have['slimerjs'] = is_cmd_found('slimerjs')
144 143
145 144 min_zmq = (2,1,11)
146 145
147 146 have['zmq'] = test_for('zmq.pyzmq_version_info', min_zmq, callback=lambda x: x())
148 147
149 148 #-----------------------------------------------------------------------------
150 149 # Test suite definitions
151 150 #-----------------------------------------------------------------------------
152 151
153 152 test_group_names = ['parallel', 'kernel', 'kernel.inprocess', 'config', 'core',
154 153 'extensions', 'lib', 'terminal', 'testing', 'utils',
155 154 'nbformat', 'qt', 'html', 'nbconvert'
156 155 ]
157 156
158 157 class TestSection(object):
159 158 def __init__(self, name, includes):
160 159 self.name = name
161 160 self.includes = includes
162 161 self.excludes = []
163 162 self.dependencies = []
164 163 self.enabled = True
165 164
166 165 def exclude(self, module):
167 166 if not module.startswith('IPython'):
168 167 module = self.includes[0] + "." + module
169 168 self.excludes.append(module.replace('.', os.sep))
170 169
171 170 def requires(self, *packages):
172 171 self.dependencies.extend(packages)
173 172
174 173 @property
175 174 def will_run(self):
176 175 return self.enabled and all(have[p] for p in self.dependencies)
177 176
178 177 # Name -> (include, exclude, dependencies_met)
179 178 test_sections = {n:TestSection(n, ['IPython.%s' % n]) for n in test_group_names}
180 179
181 180 # Exclusions and dependencies
182 181 # ---------------------------
183 182
184 183 # core:
185 184 sec = test_sections['core']
186 185 if not have['sqlite3']:
187 186 sec.exclude('tests.test_history')
188 187 sec.exclude('history')
189 188 if not have['matplotlib']:
190 189 sec.exclude('pylabtools'),
191 190 sec.exclude('tests.test_pylabtools')
192 191
193 192 # lib:
194 193 sec = test_sections['lib']
195 194 if not have['zmq']:
196 195 sec.exclude('kernel')
197 196 # We do this unconditionally, so that the test suite doesn't import
198 197 # gtk, changing the default encoding and masking some unicode bugs.
199 198 sec.exclude('inputhookgtk')
200 199 # We also do this unconditionally, because wx can interfere with Unix signals.
201 200 # There are currently no tests for it anyway.
202 201 sec.exclude('inputhookwx')
203 202 # Testing inputhook will need a lot of thought, to figure out
204 203 # how to have tests that don't lock up with the gui event
205 204 # loops in the picture
206 205 sec.exclude('inputhook')
207 206
208 207 # testing:
209 208 sec = test_sections['testing']
210 209 # These have to be skipped on win32 because they use echo, rm, cd, etc.
211 210 # See ticket https://github.com/ipython/ipython/issues/87
212 211 if sys.platform == 'win32':
213 212 sec.exclude('plugin.test_exampleip')
214 213 sec.exclude('plugin.dtexample')
215 214
216 215 # terminal:
217 216 if (not have['pexpect']) or (not have['zmq']):
218 217 test_sections['terminal'].exclude('console')
219 218
220 219 # parallel
221 220 sec = test_sections['parallel']
222 221 sec.requires('zmq')
223 222 if not have['pymongo']:
224 223 sec.exclude('controller.mongodb')
225 224 sec.exclude('tests.test_mongodb')
226 225
227 226 # kernel:
228 227 sec = test_sections['kernel']
229 228 sec.requires('zmq')
230 229 # The in-process kernel tests are done in a separate section
231 230 sec.exclude('inprocess')
232 231 # importing gtk sets the default encoding, which we want to avoid
233 232 sec.exclude('zmq.gui.gtkembed')
234 233 sec.exclude('zmq.gui.gtk3embed')
235 234 if not have['matplotlib']:
236 235 sec.exclude('zmq.pylab')
237 236
238 237 # kernel.inprocess:
239 238 test_sections['kernel.inprocess'].requires('zmq')
240 239
241 240 # extensions:
242 241 sec = test_sections['extensions']
243 if not have['cython']:
244 sec.exclude('cythonmagic')
245 sec.exclude('tests.test_cythonmagic')
246 242 # This is deprecated in favour of rpy2
247 243 sec.exclude('rmagic')
248 244 # autoreload does some strange stuff, so move it to its own test section
249 245 sec.exclude('autoreload')
250 246 sec.exclude('tests.test_autoreload')
251 247 test_sections['autoreload'] = TestSection('autoreload',
252 248 ['IPython.extensions.autoreload', 'IPython.extensions.tests.test_autoreload'])
253 249 test_group_names.append('autoreload')
254 250
255 251 # qt:
256 252 test_sections['qt'].requires('zmq', 'qt', 'pygments')
257 253
258 254 # html:
259 255 sec = test_sections['html']
260 256 sec.requires('zmq', 'tornado', 'requests', 'sqlite3', 'jsonschema')
261 257 # The notebook 'static' directory contains JS, css and other
262 258 # files for web serving. Occasionally projects may put a .py
263 259 # file in there (MathJax ships a conf.py), so we might as
264 260 # well play it safe and skip the whole thing.
265 261 sec.exclude('static')
266 262 sec.exclude('fabfile')
267 263 if not have['jinja2']:
268 264 sec.exclude('notebookapp')
269 265 if not have['pygments'] or not have['jinja2']:
270 266 sec.exclude('nbconvert')
271 267
272 268 # config:
273 269 # Config files aren't really importable stand-alone
274 270 test_sections['config'].exclude('profile')
275 271
276 272 # nbconvert:
277 273 sec = test_sections['nbconvert']
278 274 sec.requires('pygments', 'jinja2', 'jsonschema', 'mistune')
279 275 # Exclude nbconvert directories containing config files used to test.
280 276 # Executing the config files with iptest would cause an exception.
281 277 sec.exclude('tests.files')
282 278 sec.exclude('exporters.tests.files')
283 279 if not have['tornado']:
284 280 sec.exclude('nbconvert.post_processors.serve')
285 281 sec.exclude('nbconvert.post_processors.tests.test_serve')
286 282
287 283 # nbformat:
288 284 test_sections['nbformat'].requires('jsonschema')
289 285
290 286 #-----------------------------------------------------------------------------
291 287 # Functions and classes
292 288 #-----------------------------------------------------------------------------
293 289
294 290 def check_exclusions_exist():
295 291 from IPython.utils.path import get_ipython_package_dir
296 292 from IPython.utils.warn import warn
297 293 parent = os.path.dirname(get_ipython_package_dir())
298 294 for sec in test_sections:
299 295 for pattern in sec.exclusions:
300 296 fullpath = pjoin(parent, pattern)
301 297 if not os.path.exists(fullpath) and not glob.glob(fullpath + '.*'):
302 298 warn("Excluding nonexistent file: %r" % pattern)
303 299
304 300
305 301 class ExclusionPlugin(Plugin):
306 302 """A nose plugin to effect our exclusions of files and directories.
307 303 """
308 304 name = 'exclusions'
309 305 score = 3000 # Should come before any other plugins
310 306
311 307 def __init__(self, exclude_patterns=None):
312 308 """
313 309 Parameters
314 310 ----------
315 311
316 312 exclude_patterns : sequence of strings, optional
317 313 Filenames containing these patterns (as raw strings, not as regular
318 314 expressions) are excluded from the tests.
319 315 """
320 316 self.exclude_patterns = exclude_patterns or []
321 317 super(ExclusionPlugin, self).__init__()
322 318
323 319 def options(self, parser, env=os.environ):
324 320 Plugin.options(self, parser, env)
325 321
326 322 def configure(self, options, config):
327 323 Plugin.configure(self, options, config)
328 324 # Override nose trying to disable plugin.
329 325 self.enabled = True
330 326
331 327 def wantFile(self, filename):
332 328 """Return whether the given filename should be scanned for tests.
333 329 """
334 330 if any(pat in filename for pat in self.exclude_patterns):
335 331 return False
336 332 return None
337 333
338 334 def wantDirectory(self, directory):
339 335 """Return whether the given directory should be scanned for tests.
340 336 """
341 337 if any(pat in directory for pat in self.exclude_patterns):
342 338 return False
343 339 return None
344 340
345 341
346 342 class StreamCapturer(Thread):
347 343 daemon = True # Don't hang if main thread crashes
348 344 started = False
349 345 def __init__(self):
350 346 super(StreamCapturer, self).__init__()
351 347 self.streams = []
352 348 self.buffer = BytesIO()
353 349 self.readfd, self.writefd = os.pipe()
354 350 self.buffer_lock = Lock()
355 351 self.stop = Event()
356 352
357 353 def run(self):
358 354 self.started = True
359 355
360 356 while not self.stop.is_set():
361 357 chunk = os.read(self.readfd, 1024)
362 358
363 359 with self.buffer_lock:
364 360 self.buffer.write(chunk)
365 361
366 362 os.close(self.readfd)
367 363 os.close(self.writefd)
368 364
369 365 def reset_buffer(self):
370 366 with self.buffer_lock:
371 367 self.buffer.truncate(0)
372 368 self.buffer.seek(0)
373 369
374 370 def get_buffer(self):
375 371 with self.buffer_lock:
376 372 return self.buffer.getvalue()
377 373
378 374 def ensure_started(self):
379 375 if not self.started:
380 376 self.start()
381 377
382 378 def halt(self):
383 379 """Safely stop the thread."""
384 380 if not self.started:
385 381 return
386 382
387 383 self.stop.set()
388 384 os.write(self.writefd, b'wake up') # Ensure we're not locked in a read()
389 385 self.join()
390 386
391 387 class SubprocessStreamCapturePlugin(Plugin):
392 388 name='subprocstreams'
393 389 def __init__(self):
394 390 Plugin.__init__(self)
395 391 self.stream_capturer = StreamCapturer()
396 392 self.destination = os.environ.get('IPTEST_SUBPROC_STREAMS', 'capture')
397 393 # This is ugly, but distant parts of the test machinery need to be able
398 394 # to redirect streams, so we make the object globally accessible.
399 395 nose.iptest_stdstreams_fileno = self.get_write_fileno
400 396
401 397 def get_write_fileno(self):
402 398 if self.destination == 'capture':
403 399 self.stream_capturer.ensure_started()
404 400 return self.stream_capturer.writefd
405 401 elif self.destination == 'discard':
406 402 return os.open(os.devnull, os.O_WRONLY)
407 403 else:
408 404 return sys.__stdout__.fileno()
409 405
410 406 def configure(self, options, config):
411 407 Plugin.configure(self, options, config)
412 408 # Override nose trying to disable plugin.
413 409 if self.destination == 'capture':
414 410 self.enabled = True
415 411
416 412 def startTest(self, test):
417 413 # Reset log capture
418 414 self.stream_capturer.reset_buffer()
419 415
420 416 def formatFailure(self, test, err):
421 417 # Show output
422 418 ec, ev, tb = err
423 419 captured = self.stream_capturer.get_buffer().decode('utf-8', 'replace')
424 420 if captured.strip():
425 421 ev = safe_str(ev)
426 422 out = [ev, '>> begin captured subprocess output <<',
427 423 captured,
428 424 '>> end captured subprocess output <<']
429 425 return ec, '\n'.join(out), tb
430 426
431 427 return err
432 428
433 429 formatError = formatFailure
434 430
435 431 def finalize(self, result):
436 432 self.stream_capturer.halt()
437 433
438 434
439 435 def run_iptest():
440 436 """Run the IPython test suite using nose.
441 437
442 438 This function is called when this script is **not** called with the form
443 439 `iptest all`. It simply calls nose with appropriate command line flags
444 440 and accepts all of the standard nose arguments.
445 441 """
446 442 # Apply our monkeypatch to Xunit
447 443 if '--with-xunit' in sys.argv and not hasattr(Xunit, 'orig_addError'):
448 444 monkeypatch_xunit()
449 445
450 446 warnings.filterwarnings('ignore',
451 447 'This will be removed soon. Use IPython.testing.util instead')
452 448
453 449 arg1 = sys.argv[1]
454 450 if arg1 in test_sections:
455 451 section = test_sections[arg1]
456 452 sys.argv[1:2] = section.includes
457 453 elif arg1.startswith('IPython.') and arg1[8:] in test_sections:
458 454 section = test_sections[arg1[8:]]
459 455 sys.argv[1:2] = section.includes
460 456 else:
461 457 section = TestSection(arg1, includes=[arg1])
462 458
463 459
464 460 argv = sys.argv + [ '--detailed-errors', # extra info in tracebacks
465 461
466 462 '--with-ipdoctest',
467 463 '--ipdoctest-tests','--ipdoctest-extension=txt',
468 464
469 465 # We add --exe because of setuptools' imbecility (it
470 466 # blindly does chmod +x on ALL files). Nose does the
471 467 # right thing and it tries to avoid executables,
472 468 # setuptools unfortunately forces our hand here. This
473 469 # has been discussed on the distutils list and the
474 470 # setuptools devs refuse to fix this problem!
475 471 '--exe',
476 472 ]
477 473 if '-a' not in argv and '-A' not in argv:
478 474 argv = argv + ['-a', '!crash']
479 475
480 476 if nose.__version__ >= '0.11':
481 477 # I don't fully understand why we need this one, but depending on what
482 478 # directory the test suite is run from, if we don't give it, 0 tests
483 479 # get run. Specifically, if the test suite is run from the source dir
484 480 # with an argument (like 'iptest.py IPython.core', 0 tests are run,
485 481 # even if the same call done in this directory works fine). It appears
486 482 # that if the requested package is in the current dir, nose bails early
487 483 # by default. Since it's otherwise harmless, leave it in by default
488 484 # for nose >= 0.11, though unfortunately nose 0.10 doesn't support it.
489 485 argv.append('--traverse-namespace')
490 486
491 487 # use our plugin for doctesting. It will remove the standard doctest plugin
492 488 # if it finds it enabled
493 489 plugins = [ExclusionPlugin(section.excludes), IPythonDoctest(), KnownFailure(),
494 490 SubprocessStreamCapturePlugin() ]
495 491
496 492 # Use working directory set by parent process (see iptestcontroller)
497 493 if 'IPTEST_WORKING_DIR' in os.environ:
498 494 os.chdir(os.environ['IPTEST_WORKING_DIR'])
499 495
500 496 # We need a global ipython running in this process, but the special
501 497 # in-process group spawns its own IPython kernels, so for *that* group we
502 498 # must avoid also opening the global one (otherwise there's a conflict of
503 499 # singletons). Ultimately the solution to this problem is to refactor our
504 500 # assumptions about what needs to be a singleton and what doesn't (app
505 501 # objects should, individual shells shouldn't). But for now, this
506 502 # workaround allows the test suite for the inprocess module to complete.
507 503 if 'kernel.inprocess' not in section.name:
508 504 from IPython.testing import globalipapp
509 505 globalipapp.start_ipython()
510 506
511 507 # Now nose can run
512 508 TestProgram(argv=argv, addplugins=plugins)
513 509
514 510 if __name__ == '__main__':
515 511 run_iptest()
516 512
@@ -1,7 +1,7 b''
1 1 .. _extensions_cythonmagic:
2 2
3 3 ===========
4 4 cythonmagic
5 5 ===========
6 6
7 .. automodule:: IPython.extensions.cythonmagic
7 The `cython` magic has been moved in the `Cython` package.
@@ -1,102 +1,104 b''
1 1 .. _extensions_overview:
2 2
3 3 ==================
4 4 IPython extensions
5 5 ==================
6 6
7 7 A level above configuration are IPython extensions, Python modules which modify
8 8 the behaviour of the shell. They are referred to by an importable module name,
9 9 and can be placed anywhere you'd normally import from, or in
10 10 ``.ipython/extensions/``.
11 11
12 12 Getting extensions
13 13 ==================
14 14
15 15 A few important extensions are :ref:`bundled with IPython <bundled_extensions>`.
16 16 Others can be found on the `extensions index
17 17 <https://github.com/ipython/ipython/wiki/Extensions-Index>`_ on the wiki, and
18 18 the `Framework :: IPython tag <https://pypi.python.org/pypi?:action=browse&c=586>`_
19 19 on PyPI.
20 20
21 21 Extensions on PyPI can be installed using ``pip``, like any other Python package.
22 22 Other simple extensions can be installed with the ``%install_ext`` magic. The
23 23 latter does no validation, so be careful using it on untrusted networks like
24 24 public wifi.
25 25
26 26 Using extensions
27 27 ================
28 28
29 29 To load an extension while IPython is running, use the ``%load_ext`` magic:
30 30
31 31 .. sourcecode:: ipython
32 32
33 33 In [1]: %load_ext myextension
34 34
35 35 To load it each time IPython starts, list it in your configuration file::
36 36
37 37 c.InteractiveShellApp.extensions = [
38 38 'myextension'
39 39 ]
40 40
41 41 Writing extensions
42 42 ==================
43 43
44 44 An IPython extension is an importable Python module that has a couple of special
45 45 functions to load and unload it. Here is a template::
46 46
47 47 # myextension.py
48 48
49 49 def load_ipython_extension(ipython):
50 50 # The `ipython` argument is the currently active `InteractiveShell`
51 51 # instance, which can be used in any way. This allows you to register
52 52 # new magics or aliases, for example.
53 53
54 54 def unload_ipython_extension(ipython):
55 55 # If you want your extension to be unloadable, put that logic here.
56 56
57 57 This :func:`load_ipython_extension` function is called after your extension is
58 58 imported, and the currently active :class:`~IPython.core.interactiveshell.InteractiveShell`
59 59 instance is passed as the only argument. You can do anything you want with
60 60 IPython at that point.
61 61
62 62 :func:`load_ipython_extension` will be called again if you load or reload
63 63 the extension again. It is up to the extension author to add code to manage
64 64 that.
65 65
66 66 Useful :class:`InteractiveShell` methods include :meth:`~IPython.core.interactiveshell.InteractiveShell.register_magic_function`,
67 67 :meth:`~IPython.core.interactiveshell.InteractiveShell.push` (to add variables to the user namespace) and
68 68 :meth:`~IPython.core.interactiveshell.InteractiveShell.drop_by_id` (to remove variables on unloading).
69 69
70 70 .. seealso::
71 71
72 72 :ref:`defining_magics`
73 73
74 74 You can put your extension modules anywhere you want, as long as they can be
75 75 imported by Python's standard import mechanism. However, to make it easy to
76 76 write extensions, you can also put your extensions in :file:`extensions/`
77 77 within the :ref:`IPython directory <ipythondir>`. This directory is
78 78 added to :data:`sys.path` automatically.
79 79
80 80 When your extension is ready for general use, please add it to the `extensions
81 81 index <https://github.com/ipython/ipython/wiki/Extensions-Index>`_. We also
82 82 encourage you to upload it to PyPI and use the ``Framework :: IPython``
83 83 classifier, so that users can install it with standard packaging tools.
84 84
85 85 .. _bundled_extensions:
86 86
87 87 Extensions bundled with IPython
88 88 ===============================
89 89
90 90 .. toctree::
91 91 :maxdepth: 1
92 92
93 93 autoreload
94 94 cythonmagic
95 95 storemagic
96 96 sympyprinting
97 97
98 98 * ``octavemagic`` used to be bundled, but is now part of `oct2py <http://blink1073.github.io/oct2py/docs/>`_.
99 99 Use ``%load_ext oct2py.ipython`` to load it.
100 100 * ``rmagic`` is now part of `rpy2 <http://rpy.sourceforge.net/>`_. Use
101 101 ``%load_ext rpy2.ipython`` to load it, and see :mod:`rpy2.ipython.rmagic` for
102 102 details of how to use it.
103 * ``cythonmagic``used to be bundled, but is now part of `cython <https://github.com/cython/cython/>`_
104 Use ``%load_ext Cython`` to load it.
@@ -1,156 +1,156 b''
1 1 =====================================
2 2 Introduction to IPython configuration
3 3 =====================================
4 4
5 5 .. _setting_config:
6 6
7 7 Setting configurable options
8 8 ============================
9 9
10 10 Many of IPython's classes have configurable attributes (see
11 11 :doc:`options/index` for the list). These can be
12 12 configured in several ways.
13 13
14 14 Python config files
15 15 -------------------
16 16
17 17 To create the blank config files, run::
18 18
19 19 ipython profile create [profilename]
20 20
21 21 If you leave out the profile name, the files will be created for the
22 22 ``default`` profile (see :ref:`profiles`). These will typically be
23 23 located in :file:`~/.ipython/profile_default/`, and will be named
24 24 :file:`ipython_config.py`, :file:`ipython_notebook_config.py`, etc.
25 25 The settings in :file:`ipython_config.py` apply to all IPython commands.
26 26
27 27 The files typically start by getting the root config object::
28 28
29 29 c = get_config()
30 30
31 31 You can then configure class attributes like this::
32 32
33 33 c.InteractiveShell.automagic = False
34 34
35 35 Be careful with spelling--incorrect names will simply be ignored, with
36 36 no error.
37 37
38 38 To add to a collection which may have already been defined elsewhere,
39 39 you can use methods like those found on lists, dicts and sets: append,
40 40 extend, :meth:`~IPython.config.loader.LazyConfigValue.prepend` (like
41 41 extend, but at the front), add and update (which works both for dicts
42 42 and sets)::
43 43
44 c.InteractiveShellApp.extensions.append('cythonmagic')
44 c.InteractiveShellApp.extensions.append('Cython')
45 45
46 46 .. versionadded:: 2.0
47 47 list, dict and set methods for config values
48 48
49 49 Example config file
50 50 ```````````````````
51 51
52 52 ::
53 53
54 54 # sample ipython_config.py
55 55 c = get_config()
56 56
57 57 c.TerminalIPythonApp.display_banner = True
58 58 c.InteractiveShellApp.log_level = 20
59 59 c.InteractiveShellApp.extensions = [
60 60 'myextension'
61 61 ]
62 62 c.InteractiveShellApp.exec_lines = [
63 63 'import numpy',
64 64 'import scipy'
65 65 ]
66 66 c.InteractiveShellApp.exec_files = [
67 67 'mycode.py',
68 68 'fancy.ipy'
69 69 ]
70 70 c.InteractiveShell.autoindent = True
71 71 c.InteractiveShell.colors = 'LightBG'
72 72 c.InteractiveShell.confirm_exit = False
73 73 c.InteractiveShell.deep_reload = True
74 74 c.InteractiveShell.editor = 'nano'
75 75 c.InteractiveShell.xmode = 'Context'
76 76
77 77 c.PromptManager.in_template = 'In [\#]: '
78 78 c.PromptManager.in2_template = ' .\D.: '
79 79 c.PromptManager.out_template = 'Out[\#]: '
80 80 c.PromptManager.justify = True
81 81
82 82 c.PrefilterManager.multi_line_specials = True
83 83
84 84 c.AliasManager.user_aliases = [
85 85 ('la', 'ls -al')
86 86 ]
87 87
88 88
89 89 Command line arguments
90 90 ----------------------
91 91
92 92 Every configurable value can be set from the command line, using this
93 93 syntax::
94 94
95 95 ipython --ClassName.attribute=value
96 96
97 97 Many frequently used options have short aliases and flags, such as
98 98 ``--matplotlib`` (to integrate with a matplotlib GUI event loop) or
99 99 ``--pdb`` (automatic post-mortem debugging of exceptions).
100 100
101 101 To see all of these abbreviated options, run::
102 102
103 103 ipython --help
104 104 ipython notebook --help
105 105 # etc.
106 106
107 107 Options specified at the command line, in either format, override
108 108 options set in a configuration file.
109 109
110 110 The config magic
111 111 ----------------
112 112
113 113 You can also modify config from inside IPython, using a magic command::
114 114
115 115 %config IPCompleter.greedy = True
116 116
117 117 At present, this only affects the current session - changes you make to
118 118 config are not saved anywhere. Also, some options are only read when
119 119 IPython starts, so they can't be changed like this.
120 120
121 121 .. _profiles:
122 122
123 123 Profiles
124 124 ========
125 125
126 126 IPython can use multiple profiles, with separate configuration and
127 127 history. By default, if you don't specify a profile, IPython always runs
128 128 in the ``default`` profile. To use a new profile::
129 129
130 130 ipython profile create foo # create the profile foo
131 131 ipython --profile=foo # start IPython using the new profile
132 132
133 133 Profiles are typically stored in :ref:`ipythondir`, but you can also keep
134 134 a profile in the current working directory, for example to distribute it
135 135 with a project. To find a profile directory on the filesystem::
136 136
137 137 ipython locate profile foo
138 138
139 139 .. _ipythondir:
140 140
141 141 The IPython directory
142 142 =====================
143 143
144 144 IPython stores its files---config, command history and extensions---in
145 145 the directory :file:`~/.ipython/` by default.
146 146
147 147 .. envvar:: IPYTHONDIR
148 148
149 149 If set, this environment variable should be the path to a directory,
150 150 which IPython will use for user data. IPython will create it if it
151 151 does not exist.
152 152
153 153 .. option:: --ipython-dir=<path>
154 154
155 155 This command line option can also be used to override the default
156 156 IPython directory.
1 NO CONTENT: file was removed
1 NO CONTENT: file was removed
General Comments 0
You need to be logged in to leave comments. Login now