##// END OF EJS Templates
Merge pull request #3540 from minrk/cythoncache...
Matthias Bussonnier -
r11225:4920fb07 merge
parent child Browse files
Show More
@@ -1,312 +1,325 b''
1 # -*- coding: utf-8 -*-
1 # -*- coding: utf-8 -*-
2 """
2 """
3 =====================
3 =====================
4 Cython related magics
4 Cython related magics
5 =====================
5 =====================
6
6
7 Usage
7 Usage
8 =====
8 =====
9
9
10 ``%%cython``
10 ``%%cython``
11
11
12 {CYTHON_DOC}
12 {CYTHON_DOC}
13
13
14 ``%%cython_inline``
14 ``%%cython_inline``
15
15
16 {CYTHON_INLINE_DOC}
16 {CYTHON_INLINE_DOC}
17
17
18 ``%%cython_pyximport``
18 ``%%cython_pyximport``
19
19
20 {CYTHON_PYXIMPORT_DOC}
20 {CYTHON_PYXIMPORT_DOC}
21
21
22 Author:
22 Author:
23 * Brian Granger
23 * Brian Granger
24
24
25 Parts of this code were taken from Cython.inline.
25 Parts of this code were taken from Cython.inline.
26 """
26 """
27 #-----------------------------------------------------------------------------
27 #-----------------------------------------------------------------------------
28 # Copyright (C) 2010-2011, IPython Development Team.
28 # Copyright (C) 2010-2011, IPython Development Team.
29 #
29 #
30 # Distributed under the terms of the Modified BSD License.
30 # Distributed under the terms of the Modified BSD License.
31 #
31 #
32 # The full license is in the file COPYING.txt, distributed with this software.
32 # The full license is in the file COPYING.txt, distributed with this software.
33 #-----------------------------------------------------------------------------
33 #-----------------------------------------------------------------------------
34
34
35 from __future__ import print_function
35 from __future__ import print_function
36
36
37 import imp
37 import imp
38 import io
38 import io
39 import os
39 import os
40 import re
40 import re
41 import sys
41 import sys
42 import time
42 import time
43
43
44 try:
44 try:
45 reload
45 reload
46 except NameError: # Python 3
46 except NameError: # Python 3
47 from imp import reload
47 from imp import reload
48
48
49 try:
49 try:
50 import hashlib
50 import hashlib
51 except ImportError:
51 except ImportError:
52 import md5 as hashlib
52 import md5 as hashlib
53
53
54 from distutils.core import Distribution, Extension
54 from distutils.core import Distribution, Extension
55 from distutils.command.build_ext import build_ext
55 from distutils.command.build_ext import build_ext
56
56
57 from IPython.core import display
57 from IPython.core import display
58 from IPython.core import magic_arguments
58 from IPython.core import magic_arguments
59 from IPython.core.magic import Magics, magics_class, cell_magic
59 from IPython.core.magic import Magics, magics_class, cell_magic
60 from IPython.utils import py3compat
60 from IPython.utils import py3compat
61 from IPython.utils.path import get_ipython_cache_dir
61 from IPython.utils.path import get_ipython_cache_dir
62
62
63 import Cython
63 import Cython
64 from Cython.Compiler.Errors import CompileError
64 from Cython.Compiler.Errors import CompileError
65 from Cython.Build.Dependencies import cythonize
65 from Cython.Build.Dependencies import cythonize
66
66
67
67
68 @magics_class
68 @magics_class
69 class CythonMagics(Magics):
69 class CythonMagics(Magics):
70
70
71 def __init__(self, shell):
71 def __init__(self, shell):
72 super(CythonMagics,self).__init__(shell)
72 super(CythonMagics,self).__init__(shell)
73 self._reloads = {}
73 self._reloads = {}
74 self._code_cache = {}
74 self._code_cache = {}
75
75
76 def _import_all(self, module):
76 def _import_all(self, module):
77 for k,v in module.__dict__.items():
77 for k,v in module.__dict__.items():
78 if not k.startswith('__'):
78 if not k.startswith('__'):
79 self.shell.push({k:v})
79 self.shell.push({k:v})
80
80
81 @cell_magic
81 @cell_magic
82 def cython_inline(self, line, cell):
82 def cython_inline(self, line, cell):
83 """Compile and run a Cython code cell using Cython.inline.
83 """Compile and run a Cython code cell using Cython.inline.
84
84
85 This magic simply passes the body of the cell to Cython.inline
85 This magic simply passes the body of the cell to Cython.inline
86 and returns the result. If the variables `a` and `b` are defined
86 and returns the result. If the variables `a` and `b` are defined
87 in the user's namespace, here is a simple example that returns
87 in the user's namespace, here is a simple example that returns
88 their sum::
88 their sum::
89
89
90 %%cython_inline
90 %%cython_inline
91 return a+b
91 return a+b
92
92
93 For most purposes, we recommend the usage of the `%%cython` magic.
93 For most purposes, we recommend the usage of the `%%cython` magic.
94 """
94 """
95 locs = self.shell.user_global_ns
95 locs = self.shell.user_global_ns
96 globs = self.shell.user_ns
96 globs = self.shell.user_ns
97 return Cython.inline(cell, locals=locs, globals=globs)
97 return Cython.inline(cell, locals=locs, globals=globs)
98
98
99 @cell_magic
99 @cell_magic
100 def cython_pyximport(self, line, cell):
100 def cython_pyximport(self, line, cell):
101 """Compile and import a Cython code cell using pyximport.
101 """Compile and import a Cython code cell using pyximport.
102
102
103 The contents of the cell are written to a `.pyx` file in the current
103 The contents of the cell are written to a `.pyx` file in the current
104 working directory, which is then imported using `pyximport`. This
104 working directory, which is then imported using `pyximport`. This
105 magic requires a module name to be passed::
105 magic requires a module name to be passed::
106
106
107 %%cython_pyximport modulename
107 %%cython_pyximport modulename
108 def f(x):
108 def f(x):
109 return 2.0*x
109 return 2.0*x
110
110
111 The compiled module is then imported and all of its symbols are
111 The compiled module is then imported and all of its symbols are
112 injected into the user's namespace. For most purposes, we recommend
112 injected into the user's namespace. For most purposes, we recommend
113 the usage of the `%%cython` magic.
113 the usage of the `%%cython` magic.
114 """
114 """
115 module_name = line.strip()
115 module_name = line.strip()
116 if not module_name:
116 if not module_name:
117 raise ValueError('module name must be given')
117 raise ValueError('module name must be given')
118 fname = module_name + '.pyx'
118 fname = module_name + '.pyx'
119 with io.open(fname, 'w', encoding='utf-8') as f:
119 with io.open(fname, 'w', encoding='utf-8') as f:
120 f.write(cell)
120 f.write(cell)
121 if 'pyximport' not in sys.modules:
121 if 'pyximport' not in sys.modules:
122 import pyximport
122 import pyximport
123 pyximport.install(reload_support=True)
123 pyximport.install(reload_support=True)
124 if module_name in self._reloads:
124 if module_name in self._reloads:
125 module = self._reloads[module_name]
125 module = self._reloads[module_name]
126 reload(module)
126 reload(module)
127 else:
127 else:
128 __import__(module_name)
128 __import__(module_name)
129 module = sys.modules[module_name]
129 module = sys.modules[module_name]
130 self._reloads[module_name] = module
130 self._reloads[module_name] = module
131 self._import_all(module)
131 self._import_all(module)
132
132
133 @magic_arguments.magic_arguments()
133 @magic_arguments.magic_arguments()
134 @magic_arguments.argument(
134 @magic_arguments.argument(
135 '-c', '--compile-args', action='append', default=[],
135 '-c', '--compile-args', action='append', default=[],
136 help="Extra flags to pass to compiler via the `extra_compile_args` "
136 help="Extra flags to pass to compiler via the `extra_compile_args` "
137 "Extension flag (can be specified multiple times)."
137 "Extension flag (can be specified multiple times)."
138 )
138 )
139 @magic_arguments.argument(
139 @magic_arguments.argument(
140 '--link-args', action='append', default=[],
140 '--link-args', action='append', default=[],
141 help="Extra flags to pass to linker via the `extra_link_args` "
141 help="Extra flags to pass to linker via the `extra_link_args` "
142 "Extension flag (can be specified multiple times)."
142 "Extension flag (can be specified multiple times)."
143 )
143 )
144 @magic_arguments.argument(
144 @magic_arguments.argument(
145 '-l', '--lib', action='append', default=[],
145 '-l', '--lib', action='append', default=[],
146 help="Add a library to link the extension against (can be specified "
146 help="Add a library to link the extension against (can be specified "
147 "multiple times)."
147 "multiple times)."
148 )
148 )
149 @magic_arguments.argument(
149 @magic_arguments.argument(
150 '-L', dest='library_dirs', metavar='dir', action='append', default=[],
150 '-L', dest='library_dirs', metavar='dir', action='append', default=[],
151 help="Add a path to the list of libary directories (can be specified "
151 help="Add a path to the list of libary directories (can be specified "
152 "multiple times)."
152 "multiple times)."
153 )
153 )
154 @magic_arguments.argument(
154 @magic_arguments.argument(
155 '-I', '--include', action='append', default=[],
155 '-I', '--include', action='append', default=[],
156 help="Add a path to the list of include directories (can be specified "
156 help="Add a path to the list of include directories (can be specified "
157 "multiple times)."
157 "multiple times)."
158 )
158 )
159 @magic_arguments.argument(
159 @magic_arguments.argument(
160 '-+', '--cplus', action='store_true', default=False,
160 '-+', '--cplus', action='store_true', default=False,
161 help="Output a C++ rather than C file."
161 help="Output a C++ rather than C file."
162 )
162 )
163 @magic_arguments.argument(
163 @magic_arguments.argument(
164 '-f', '--force', action='store_true', default=False,
164 '-f', '--force', action='store_true', default=False,
165 help="Force the compilation of a new module, even if the source has been "
165 help="Force the compilation of a new module, even if the source has been "
166 "previously compiled."
166 "previously compiled."
167 )
167 )
168 @magic_arguments.argument(
168 @magic_arguments.argument(
169 '-a', '--annotate', action='store_true', default=False,
169 '-a', '--annotate', action='store_true', default=False,
170 help="Produce a colorized HTML version of the source."
170 help="Produce a colorized HTML version of the source."
171 )
171 )
172 @cell_magic
172 @cell_magic
173 def cython(self, line, cell):
173 def cython(self, line, cell):
174 """Compile and import everything from a Cython code cell.
174 """Compile and import everything from a Cython code cell.
175
175
176 The contents of the cell are written to a `.pyx` file in the
176 The contents of the cell are written to a `.pyx` file in the
177 directory `IPYTHONDIR/cython` using a filename with the hash of the
177 directory `IPYTHONDIR/cython` using a filename with the hash of the
178 code. This file is then cythonized and compiled. The resulting module
178 code. This file is then cythonized and compiled. The resulting module
179 is imported and all of its symbols are injected into the user's
179 is imported and all of its symbols are injected into the user's
180 namespace. The usage is similar to that of `%%cython_pyximport` but
180 namespace. The usage is similar to that of `%%cython_pyximport` but
181 you don't have to pass a module name::
181 you don't have to pass a module name::
182
182
183 %%cython
183 %%cython
184 def f(x):
184 def f(x):
185 return 2.0*x
185 return 2.0*x
186
186
187 To compile OpenMP codes, pass the required `--compile-args`
187 To compile OpenMP codes, pass the required `--compile-args`
188 and `--link-args`. For example with gcc::
188 and `--link-args`. For example with gcc::
189
189
190 %%cython --compile-args=-fopenmp --link-args=-fopenmp
190 %%cython --compile-args=-fopenmp --link-args=-fopenmp
191 ...
191 ...
192 """
192 """
193 args = magic_arguments.parse_argstring(self.cython, line)
193 args = magic_arguments.parse_argstring(self.cython, line)
194 code = cell if cell.endswith('\n') else cell+'\n'
194 code = cell if cell.endswith('\n') else cell+'\n'
195 lib_dir = os.path.join(get_ipython_cache_dir(), 'cython')
195 lib_dir = os.path.join(get_ipython_cache_dir(), 'cython')
196 quiet = True
196 quiet = True
197 key = code, sys.version_info, sys.executable, Cython.__version__
197 key = code, sys.version_info, sys.executable, Cython.__version__
198
198
199 if not os.path.exists(lib_dir):
199 if not os.path.exists(lib_dir):
200 os.makedirs(lib_dir)
200 os.makedirs(lib_dir)
201
201
202 if args.force:
202 if args.force:
203 # Force a new module name by adding the current time to the
203 # Force a new module name by adding the current time to the
204 # key which is hashed to determine the module name.
204 # key which is hashed to determine the module name.
205 key += time.time(),
205 key += time.time(),
206
206
207 module_name = "_cython_magic_" + hashlib.md5(str(key).encode('utf-8')).hexdigest()
207 module_name = "_cython_magic_" + hashlib.md5(str(key).encode('utf-8')).hexdigest()
208 module_path = os.path.join(lib_dir, module_name + self.so_ext)
208 module_path = os.path.join(lib_dir, module_name + self.so_ext)
209
209
210 have_module = os.path.isfile(module_path)
210 have_module = os.path.isfile(module_path)
211 need_cythonize = not have_module
211 need_cythonize = not have_module
212
212
213 if args.annotate:
213 if args.annotate:
214 html_file = os.path.join(lib_dir, module_name + '.html')
214 html_file = os.path.join(lib_dir, module_name + '.html')
215 if not os.path.isfile(html_file):
215 if not os.path.isfile(html_file):
216 need_cythonize = True
216 need_cythonize = True
217
217
218 if need_cythonize:
218 if need_cythonize:
219 c_include_dirs = args.include
219 c_include_dirs = args.include
220 if 'numpy' in code:
220 if 'numpy' in code:
221 import numpy
221 import numpy
222 c_include_dirs.append(numpy.get_include())
222 c_include_dirs.append(numpy.get_include())
223 pyx_file = os.path.join(lib_dir, module_name + '.pyx')
223 pyx_file = os.path.join(lib_dir, module_name + '.pyx')
224 pyx_file = py3compat.cast_bytes_py2(pyx_file, encoding=sys.getfilesystemencoding())
224 pyx_file = py3compat.cast_bytes_py2(pyx_file, encoding=sys.getfilesystemencoding())
225 with io.open(pyx_file, 'w', encoding='utf-8') as f:
225 with io.open(pyx_file, 'w', encoding='utf-8') as f:
226 f.write(code)
226 f.write(code)
227 extension = Extension(
227 extension = Extension(
228 name = module_name,
228 name = module_name,
229 sources = [pyx_file],
229 sources = [pyx_file],
230 include_dirs = c_include_dirs,
230 include_dirs = c_include_dirs,
231 library_dirs = args.library_dirs,
231 library_dirs = args.library_dirs,
232 extra_compile_args = args.compile_args,
232 extra_compile_args = args.compile_args,
233 extra_link_args = args.link_args,
233 extra_link_args = args.link_args,
234 libraries = args.lib,
234 libraries = args.lib,
235 language = 'c++' if args.cplus else 'c',
235 language = 'c++' if args.cplus else 'c',
236 )
236 )
237 build_extension = self._get_build_extension()
237 build_extension = self._get_build_extension()
238 try:
238 try:
239 opts = dict(
239 opts = dict(
240 quiet=quiet,
240 quiet=quiet,
241 annotate = args.annotate,
241 annotate = args.annotate,
242 force = True,
242 force = True,
243 )
243 )
244 build_extension.extensions = cythonize([extension], **opts)
244 build_extension.extensions = cythonize([extension], **opts)
245 except CompileError:
245 except CompileError:
246 return
246 return
247
247
248 if not have_module:
248 if not have_module:
249 build_extension.build_temp = os.path.dirname(pyx_file)
249 build_extension.build_temp = os.path.dirname(pyx_file)
250 build_extension.build_lib = lib_dir
250 build_extension.build_lib = lib_dir
251 build_extension.run()
251 build_extension.run()
252 self._code_cache[key] = module_name
252 self._code_cache[key] = module_name
253
253
254 module = imp.load_dynamic(module_name, module_path)
254 module = imp.load_dynamic(module_name, module_path)
255 self._import_all(module)
255 self._import_all(module)
256
256
257 if args.annotate:
257 if args.annotate:
258 try:
258 try:
259 with io.open(html_file, encoding='utf-8') as f:
259 with io.open(html_file, encoding='utf-8') as f:
260 annotated_html = f.read()
260 annotated_html = f.read()
261 except IOError as e:
261 except IOError as e:
262 # File could not be opened. Most likely the user has a version
262 # File could not be opened. Most likely the user has a version
263 # of Cython before 0.15.1 (when `cythonize` learned the
263 # of Cython before 0.15.1 (when `cythonize` learned the
264 # `force` keyword argument) and has already compiled this
264 # `force` keyword argument) and has already compiled this
265 # exact source without annotation.
265 # exact source without annotation.
266 print('Cython completed successfully but the annotated '
266 print('Cython completed successfully but the annotated '
267 'source could not be read.', file=sys.stderr)
267 'source could not be read.', file=sys.stderr)
268 print(e, file=sys.stderr)
268 print(e, file=sys.stderr)
269 else:
269 else:
270 return display.HTML(self.clean_annotated_html(annotated_html))
270 return display.HTML(self.clean_annotated_html(annotated_html))
271
271
272 @property
272 @property
273 def so_ext(self):
273 def so_ext(self):
274 """The extension suffix for compiled modules."""
274 """The extension suffix for compiled modules."""
275 try:
275 try:
276 return self._so_ext
276 return self._so_ext
277 except AttributeError:
277 except AttributeError:
278 self._so_ext = self._get_build_extension().get_ext_filename('')
278 self._so_ext = self._get_build_extension().get_ext_filename('')
279 return self._so_ext
279 return self._so_ext
280
280
281 def _clear_distutils_mkpath_cache(self):
282 """clear distutils mkpath cache
283
284 prevents distutils from skipping re-creation of dirs that have been removed
285 """
286 try:
287 from distutils.dir_util import _path_created
288 except ImportError:
289 pass
290 else:
291 _path_created.clear()
292
281 def _get_build_extension(self):
293 def _get_build_extension(self):
294 self._clear_distutils_mkpath_cache()
282 dist = Distribution()
295 dist = Distribution()
283 config_files = dist.find_config_files()
296 config_files = dist.find_config_files()
284 try:
297 try:
285 config_files.remove('setup.cfg')
298 config_files.remove('setup.cfg')
286 except ValueError:
299 except ValueError:
287 pass
300 pass
288 dist.parse_config_files(config_files)
301 dist.parse_config_files(config_files)
289 build_extension = build_ext(dist)
302 build_extension = build_ext(dist)
290 build_extension.finalize_options()
303 build_extension.finalize_options()
291 return build_extension
304 return build_extension
292
305
293 @staticmethod
306 @staticmethod
294 def clean_annotated_html(html):
307 def clean_annotated_html(html):
295 """Clean up the annotated HTML source.
308 """Clean up the annotated HTML source.
296
309
297 Strips the link to the generated C or C++ file, which we do not
310 Strips the link to the generated C or C++ file, which we do not
298 present to the user.
311 present to the user.
299 """
312 """
300 r = re.compile('<p>Raw output: <a href="(.*)">(.*)</a>')
313 r = re.compile('<p>Raw output: <a href="(.*)">(.*)</a>')
301 html = '\n'.join(l for l in html.splitlines() if not r.match(l))
314 html = '\n'.join(l for l in html.splitlines() if not r.match(l))
302 return html
315 return html
303
316
304 __doc__ = __doc__.format(
317 __doc__ = __doc__.format(
305 CYTHON_DOC = ' '*8 + CythonMagics.cython.__doc__,
318 CYTHON_DOC = ' '*8 + CythonMagics.cython.__doc__,
306 CYTHON_INLINE_DOC = ' '*8 + CythonMagics.cython_inline.__doc__,
319 CYTHON_INLINE_DOC = ' '*8 + CythonMagics.cython_inline.__doc__,
307 CYTHON_PYXIMPORT_DOC = ' '*8 + CythonMagics.cython_pyximport.__doc__,
320 CYTHON_PYXIMPORT_DOC = ' '*8 + CythonMagics.cython_pyximport.__doc__,
308 )
321 )
309
322
310 def load_ipython_extension(ip):
323 def load_ipython_extension(ip):
311 """Load the extension in IPython."""
324 """Load the extension in IPython."""
312 ip.register_magics(CythonMagics)
325 ip.register_magics(CythonMagics)
General Comments 0
You need to be logged in to leave comments. Login now