Merge pull request #1770 from ellisonbg/cython_magic

Cython related magic functions: offers the new cell magics %%cython_inline, %%cython_pyximport and %%cython to make it very easy to put cython-accelerated code in a cell and have it loaded interactively.

IPython/extensions/cythonmagic.py

@@ -0,0 +1,181 b''
+# -*- coding: utf-8 -*-
+"""
+Cython related magics.
+
+Author:
+* Brian Granger
+
+Parts of this code were taken from Cython.inline.
+"""
+#-----------------------------------------------------------------------------
+# Copyright (C) 2010-2011, IPython Development Team.
+#
+# Distributed under the terms of the Modified BSD License.
+#
+# The full license is in the file COPYING.txt, distributed with this software.
+#-----------------------------------------------------------------------------
+
+import io
+import os, sys
+from importlib import import_module
+import imp
+
+try:
+    import hashlib
+except ImportError:
+    import md5 as hashlib
+
+from distutils.core import Distribution, Extension
+from distutils.command.build_ext import build_ext
+
+from IPython.core.magic import Magics, magics_class, cell_magic
+from IPython.testing.skipdoctest import skip_doctest
+from IPython.core.magic_arguments import (
+    argument, magic_arguments, parse_argstring
+)
+from IPython.utils import py3compat
+
+import Cython
+from Cython.Compiler.Errors import CompileError
+from Cython.Compiler.Main import Context, default_options
+from Cython.Build.Dependencies import cythonize
+
+
+@magics_class
+class CythonMagics(Magics):
+
+    def __init__(self, shell):
+        super(CythonMagics,self).__init__(shell)
+        self._reloads = {}
+        self._code_cache = {}
+
+    def _import_all(self, module):
+        for k,v in module.__dict__.items():
+            if not k.startswith('__'):
+                self.shell.push({k:v})
+
+    @cell_magic
+    def cython_inline(self, line, cell):
+        """Compile and run a Cython code cell using Cython.inline.
+
+        This magic simply passes the body of the cell to Cython.inline
+        and returns the result. If the variables `a` and `b` are defined
+        in the user's namespace, here is a simple example that returns
+        their sum::
+        
+            %%cython_inline
+            return a+b
+
+        For most purposes, we recommend the usage of the `%%cython` magic.
+        """
+        locs = self.shell.user_global_ns
+        globs = self.shell.user_ns
+        return Cython.inline(cell, locals=locs, globals=globs)
+
+    @cell_magic
+    def cython_pyximport(self, line, cell):
+        """Compile and import a Cython code cell using pyximport.
+
+        The contents of the cell are written to a `.pyx` file in the current
+        working directory, which is then imported using `pyximport`. This
+        magic requires a module name to be passed::
+        
+            %%cython_pyximport modulename
+            def f(x):
+                return 2.0*x
+
+        The compiled module is then imported and all of its symbols are injected into
+        the user's namespace. For most purposes, we recommend the usage of the
+        `%%cython` magic.
+        """
+        module_name = line.strip()
+        if not module_name:
+            raise ValueError('module name must be given')
+        fname = module_name + '.pyx'
+        with io.open(fname, 'w', encoding='utf-8') as f:
+            f.write(cell)
+        if 'pyximport' not in sys.modules:
+            import pyximport
+            pyximport.install(reload_support=True)
+        if module_name in self._reloads:
+            module = self._reloads[module_name]
+            reload(module)
+        else:
+            module = import_module(module_name)
+            self._reloads[module_name] = module
+        self._import_all(module)
+
+    @magic_arguments()
+    @argument(
+        '-f', '--force', action='store_true', default=False,
+        help="Force the compilation of the pyx module even if it hasn't changed"
+    )
+    @cell_magic
+    def cython(self, line, cell):
+        """Compile and import everything from a Cython code cell.
+
+        The contents of the cell are written to a `.pyx` file in the
+        directory `IPYTHONDIR/cython` using a filename with the hash of the code.
+        This file is then cythonized and compiled. The resulting module
+        is imported and all of its symbols are injected into the user's
+        namespace. The usage is similar to that of `%%cython_pyximport` but
+        you don't have to pass a module name::
+
+        %%cython
+        def f(x):
+            return 2.0*x
+        """
+        args = parse_argstring(self.cython, line)
+        code = cell if cell.endswith('\n') else cell+'\n'
+        lib_dir=os.path.join(self.shell.ipython_dir, 'cython')
+        cython_include_dirs=['.']
+        force=args.force
+        quiet=True
+        ctx = Context(cython_include_dirs, default_options)
+        key = code, sys.version_info, sys.executable, Cython.__version__
+        module_name = "_cython_magic_" + hashlib.md5(str(key).encode('utf-8')).hexdigest()
+        so_ext = [ ext for ext,_,mod_type in imp.get_suffixes() if mod_type == imp.C_EXTENSION ][0]
+        module_path = os.path.join(lib_dir, module_name+so_ext)
+
+        if not os.path.exists(lib_dir):
+            os.makedirs(lib_dir)
+
+        if force or not os.path.isfile(module_path):
+            cflags = []
+            c_include_dirs = []
+            if 'numpy' in code:
+                import numpy
+                c_include_dirs.append(numpy.get_include())
+            pyx_file = os.path.join(lib_dir, module_name + '.pyx')
+            pyx_file = py3compat.unicode_to_str(pyx_file, encoding=sys.getfilesystemencoding())
+            with io.open(pyx_file, 'w', encoding='utf-8') as f:
+                f.write(code)
+            extension = Extension(
+                name = module_name,
+                sources = [pyx_file],
+                include_dirs = c_include_dirs,
+                extra_compile_args = cflags
+            )
+            build_extension = build_ext(Distribution())
+            build_extension.finalize_options()
+            try:
+                build_extension.extensions = cythonize([extension], ctx=ctx, quiet=quiet)
+            except CompileError:
+                return
+            build_extension.build_temp = os.path.dirname(pyx_file)
+            build_extension.build_lib  = lib_dir
+            build_extension.run()
+            self._code_cache[key] = module_name
+
+        module = imp.load_dynamic(module_name, module_path)
+        self._import_all(module)
+
+
+_loaded = False
+
+def load_ipython_extension(ip):
+    """Load the extension in IPython."""
+    global _loaded
+    if not _loaded:
+        ip.register_magics(CythonMagics)
+        _loaded = True

IPython/extensions/tests/test_cythonmagic.py

@@ -0,0 +1,47 b''
+# -*- coding: utf-8 -*-
+"""Tests for the Cython magics extension."""
+
+import os
+import nose.tools as nt
+
+from IPython.utils import py3compat
+
+code = py3compat.str_to_unicode("""def f(x):
+    return 2*x
+""")
+
+try:
+    import Cython
+except:
+    __test__ = False
+
+def setup():
+    ip = get_ipython()
+    ip.extension_manager.load_extension('cythonmagic')
+
+def test_cython_inline():
+    ip = get_ipython()
+    ip.ex('a=10; b=20')
+    result = ip.run_cell_magic('cython_inline','','return a+b')
+    nt.assert_equals(result, 30)
+
+def test_cython_pyximport():
+    module_name = '_test_cython_pyximport'
+    ip = get_ipython()
+    ip.run_cell_magic('cython_pyximport', module_name, code)
+    ip.ex('g = f(10)')
+    nt.assert_equals(ip.user_ns['g'], 20.0)
+    try:
+        os.remove(module_name+'.pyx')
+    except OSError:
+        pass
+
+def test_cython():
+    ip = get_ipython()
+    ip.run_cell_magic('cython', '', code)
+    ip.ex('g = f(10)')
+    nt.assert_equals(ip.user_ns['g'], 20.0)
+    
+
+
+

docs/examples/notebooks/cython_extension.ipynb

@@ -0,0 +1,226 b''
+{
+ "metadata": {
+  "name": "cython_extension"
+ },
+ "nbformat": 3,
+ "worksheets": [
+  {
+   "cells": [
+    {
+     "cell_type": "heading",
+     "level": 1,
+     "source": [
+      "Cython Magic Functions Extension"
+     ]
+    },
+    {
+     "cell_type": "heading",
+     "level": 2,
+     "source": [
+      "Loading the extension"
+     ]
+    },
+    {
+     "cell_type": "markdown",
+     "source": [
+      "IPtyhon has a `cythonmagic` extension that contains a number of magic functions for working with Cython code. This extension can be loaded using the `%load_ext` magic as follows:"
+     ]
+    },
+    {
+     "cell_type": "code",
+     "collapsed": false,
+     "input": [
+      "%load_ext cythonmagic"
+     ],
+     "language": "python",
+     "outputs": [],
+     "prompt_number": 1
+    },
+    {
+     "cell_type": "heading",
+     "level": 2,
+     "source": [
+      "The %cython_inline magic"
+     ]
+    },
+    {
+     "cell_type": "markdown",
+     "source": [
+      "The `%%cython_inline` magic uses `Cython.inline` to compile a Cython expression. This allows you to enter and run a function body with Cython code. Use a bare `return` statement to return values. "
+     ]
+    },
+    {
+     "cell_type": "code",
+     "collapsed": true,
+     "input": [
+      "a = 10",
+      "b = 20"
+     ],
+     "language": "python",
+     "outputs": [],
+     "prompt_number": 8
+    },
+    {
+     "cell_type": "code",
+     "collapsed": false,
+     "input": [
+      "%%cython_inline",
+      "return a+b"
+     ],
+     "language": "python",
+     "outputs": [
+      {
+       "output_type": "pyout",
+       "prompt_number": 9,
+       "text": [
+        "30"
+       ]
+      }
+     ],
+     "prompt_number": 9
+    },
+    {
+     "cell_type": "heading",
+     "level": 2,
+     "source": [
+      "The %cython_pyximport magic"
+     ]
+    },
+    {
+     "cell_type": "markdown",
+     "source": [
+      "The `%%cython_pyximport` magic allows you to enter arbitrary Cython code into a cell. That Cython code is written as a `.pyx` file in the current working directory and then imported using `pyximport`.  You have the specify the name of the module that the Code will appear in. All symbols from the module are imported automatically by the magic function."
+     ]
+    },
+    {
+     "cell_type": "code",
+     "collapsed": false,
+     "input": [
+      "%%cython_pyximport foo",
+      "def f(x):",
+      "    return 4.0*x"
+     ],
+     "language": "python",
+     "outputs": [],
+     "prompt_number": 18
+    },
+    {
+     "cell_type": "code",
+     "collapsed": false,
+     "input": [
+      "f(10)"
+     ],
+     "language": "python",
+     "outputs": [
+      {
+       "output_type": "pyout",
+       "prompt_number": 19,
+       "text": [
+        "40.0"
+       ]
+      }
+     ],
+     "prompt_number": 19
+    },
+    {
+     "cell_type": "heading",
+     "level": 2,
+     "source": [
+      "The %cython magic"
+     ]
+    },
+    {
+     "cell_type": "markdown",
+     "source": [
+      "Probably the most important magic is the `%cython` magic.  This is similar to the `%%cython_pyximport` magic, but doesn't require you to specify a module name. Instead, the `%%cython` magic uses manages everything using temporary files in the `~/.cython/magic` directory.  All of the symbols in the Cython module are imported automatically by the magic.",
+      "",
+      "Here is a simple example of a Black-Scholes options pricing algorithm written in Cython:"
+     ]
+    },
+    {
+     "cell_type": "code",
+     "collapsed": false,
+     "input": [
+      "%%cython",
+      "cimport cython",
+      "from libc.math cimport exp, sqrt, pow, log, erf",
+      "",
+      "@cython.cdivision(True)",
+      "cdef double std_norm_cdf(double x) nogil:",
+      "    return 0.5*(1+erf(x/sqrt(2.0)))",
+      "",
+      "@cython.cdivision(True)",
+      "def black_scholes(double s, double k, double t, double v,",
+      "                 double rf, double div, double cp):",
+      "    \"\"\"Price an option using the Black-Scholes model.",
+      "    ",
+      "    s : initial stock price",
+      "    k : strike price",
+      "    t : expiration time",
+      "    v : volatility",
+      "    rf : risk-free rate",
+      "    div : dividend",
+      "    cp : +1/-1 for call/put",
+      "    \"\"\"",
+      "    cdef double d1, d2, optprice",
+      "    with nogil:",
+      "        d1 = (log(s/k)+(rf-div+0.5*pow(v,2))*t)/(v*sqrt(t))",
+      "        d2 = d1 - v*sqrt(t)",
+      "        optprice = cp*s*exp(-div*t)*std_norm_cdf(cp*d1) - \\",
+      "            cp*k*exp(-rf*t)*std_norm_cdf(cp*d2)",
+      "    return optprice"
+     ],
+     "language": "python",
+     "outputs": [],
+     "prompt_number": 6
+    },
+    {
+     "cell_type": "code",
+     "collapsed": false,
+     "input": [
+      "black_scholes(100.0, 100.0, 1.0, 0.3, 0.03, 0.0, -1)"
+     ],
+     "language": "python",
+     "outputs": [
+      {
+       "output_type": "pyout",
+       "prompt_number": 7,
+       "text": [
+        "10.327861752731728"
+       ]
+      }
+     ],
+     "prompt_number": 7
+    },
+    {
+     "cell_type": "code",
+     "collapsed": false,
+     "input": [
+      "%timeit black_scholes(100.0, 100.0, 1.0, 0.3, 0.03, 0.0, -1)"
+     ],
+     "language": "python",
+     "outputs": [
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        "1000000 loops, best of 3: 621 ns per loop",
+        ""
+       ]
+      }
+     ],
+     "prompt_number": 14
+    },
+    {
+     "cell_type": "code",
+     "collapsed": true,
+     "input": [
+      ""
+     ],
+     "language": "python",
+     "outputs": []
+    }
+   ]
+  }
+ ]
+}
  No newline at end of file

docs/source/config/extensions/cythonmagic.txt

@@ -0,0 +1,7 b''
+.. _extensions_cythonmagic:
+
+===========
+cythonmagic
+===========
+
+.. automodule:: IPython.extensions.cythonmagic

IPython/testing/iptest.py

@@ -149,6 +149,7 b" have['wx'] = test_for('wx')"
 have['wx.aui'] = test_for('wx.aui')
 have['qt'] = test_for('IPython.external.qt')
 have['sqlite3'] = test_for('sqlite3')
+have['cython'] = test_for('Cython')
 
 have['tornado'] = test_for('tornado.version_info', (2,1,0), callback=None)
 
@@ -269,6 +270,9 b' def make_exclude():'
                            ipjoin('zmq', 'pylab'),
         ])
 
+    if not have['cython']:
+        exclusions.extend([ipjoin('extensions', 'cythonmagic')])
+
     if not have['tornado']:
         exclusions.append(ipjoin('frontend', 'html'))
 

docs/source/config/extensions/index.txt

@@ -71,6 +71,7 b' Extensions bundled with IPython'
    :maxdepth: 1
 
    autoreload
+   cythonmagic
    parallelmagic
    storemagic
    sympyprinting