diff --git a/examples/notebooks/Importing Notebooks.ipynb b/examples/notebooks/Importing Notebooks.ipynb new file mode 100644 index 0000000..ae20fad --- /dev/null +++ b/examples/notebooks/Importing Notebooks.ipynb @@ -0,0 +1,796 @@ +{ + "metadata": { + "gist_id": "6011986", + "name": "" + }, + "nbformat": 3, + "nbformat_minor": 0, + "worksheets": [ + { + "cells": [ + { + "cell_type": "heading", + "level": 1, + "metadata": {}, + "source": [ + "Importing IPython Notebooks as Modules" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "It is a common problem that people want to import code from IPython Notebooks.\n", + "This is made difficult by the fact that Notebooks are not plain Python files,\n", + "and thus cannot be imported by the regular Python machinery.\n", + "\n", + "There is a flag in the notebook server that provides a certain workaround for a small set of cases,\n", + "but I think it's gross so I won't even discuss it.\n", + "\n", + "Fortunately, Python provides some fairly sophisticated [hooks](http://www.python.org/dev/peps/pep-0302/) into the import machinery,\n", + "so we can actually make IPython notebooks importable without much difficulty,\n", + "and only using public APIs.\n", + "\n", + "Forgive me if some of this is gross or wrong,\n", + "I haven't really written import hooks before." + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "import io, os, sys, types" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 1 + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "from IPython.nbformat import current\n", + "from IPython.core.interactiveshell import InteractiveShell" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 2 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Import hooks typically take the form of two objects:\n", + "\n", + "1. a Module **Loader**, which takes a module name (e.g. `'IPython.display'`), and returns a Module\n", + "2. a Module **Finder**, which figures out whether a module might exist, and tells Python what **Loader** to use" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "def find_notebook(fullname, path=None):\n", + " \"\"\"find a notebook, given its fully qualified name and an optional path\n", + " \n", + " This turns \"foo.bar\" into \"foo/bar.ipynb\"\n", + " and tries turning \"Foo_Bar\" into \"Foo Bar\" if Foo_Bar\n", + " does not exist.\n", + " \"\"\"\n", + " name = fullname.rsplit('.', 1)[-1]\n", + " if not path:\n", + " path = ['']\n", + " for d in path:\n", + " nb_path = os.path.join(d, name + \".ipynb\")\n", + " if os.path.isfile(nb_path):\n", + " return nb_path\n", + " # let import Notebook_Name find \"Notebook Name.ipynb\"\n", + " nb_path = nb_path.replace(\"_\", \" \")\n", + " if os.path.isfile(nb_path):\n", + " return nb_path\n", + " " + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 3 + }, + { + "cell_type": "heading", + "level": 2, + "metadata": {}, + "source": [ + "Notebook Loader" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Here we have our Notebook Loader.\n", + "It's actually quite simple - once we figure out the filename of the module,\n", + "all it does is:\n", + "\n", + "1. load the notebook document into memory\n", + "2. create an empty Module\n", + "3. execute every cell in the Module namespace\n", + "\n", + "Since IPython cells can have extended syntax,\n", + "the IPython transform is applied to turn each of these cells into their pure-Python counterparts before executing them.\n", + "If all of your notebook cells are pure-Python,\n", + "this step is unnecessary." + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "class NotebookLoader(object):\n", + " \"\"\"Module Loader for IPython Notebooks\"\"\"\n", + " def __init__(self, path=None):\n", + " self.shell = InteractiveShell.instance()\n", + " self.path = path\n", + " \n", + " def load_module(self, fullname):\n", + " \"\"\"import a notebook as a module\"\"\"\n", + " path = find_notebook(fullname, self.path)\n", + " \n", + " print (\"importing IPython notebook from %s\" % path)\n", + " \n", + " # load the notebook object\n", + " with io.open(path, 'r', encoding='utf-8') as f:\n", + " nb = current.read(f, 'json')\n", + " \n", + " \n", + " # create the module and add it to sys.modules\n", + " # if name in sys.modules:\n", + " # return sys.modules[name]\n", + " mod = types.ModuleType(fullname)\n", + " mod.__file__ = path\n", + " mod.__loader__ = self\n", + " sys.modules[fullname] = mod\n", + " \n", + " # extra work to ensure that magics that would affect the user_ns\n", + " # actually affect the notebook module's ns\n", + " save_user_ns = self.shell.user_ns\n", + " self.shell.user_ns = mod.__dict__\n", + " \n", + " try:\n", + " for cell in nb.worksheets[0].cells:\n", + " if cell.cell_type == 'code' and cell.language == 'python':\n", + " # transform the input to executable Python\n", + " code = self.shell.input_transformer_manager.transform_cell(cell.input)\n", + " # run the code in themodule\n", + " exec code in mod.__dict__\n", + " finally:\n", + " self.shell.user_ns = save_user_ns\n", + " return mod\n" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 4 + }, + { + "cell_type": "heading", + "level": 2, + "metadata": {}, + "source": [ + "The Module Finder" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The finder is a simple object that tells you whether a name can be imported,\n", + "and returns the appropriate loader.\n", + "All this one does is check, when you do:\n", + "\n", + "```python\n", + "import mynotebook\n", + "```\n", + "\n", + "it checks whether `mynotebook.ipynb` exists.\n", + "If a notebook is found, then it returns a NotebookLoader.\n", + "\n", + "Any extra logic is just for resolving paths within packages." + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "class NotebookFinder(object):\n", + " \"\"\"Module finder that locates IPython Notebooks\"\"\"\n", + " def __init__(self):\n", + " self.loaders = {}\n", + " \n", + " def find_module(self, fullname, path=None):\n", + " nb_path = find_notebook(fullname, path)\n", + " if not nb_path:\n", + " return\n", + " \n", + " key = path\n", + " if path:\n", + " # lists aren't hashable\n", + " key = os.path.sep.join(path)\n", + " \n", + " if key not in self.loaders:\n", + " self.loaders[key] = NotebookLoader(path)\n", + " return self.loaders[key]\n" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 5 + }, + { + "cell_type": "heading", + "level": 2, + "metadata": {}, + "source": [ + "Register the hook" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now we register the `NotebookFinder` with `sys.meta_path`" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "sys.meta_path.append(NotebookFinder())" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 6 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "After this point, my notebooks should be importable.\n", + "\n", + "Let's look at what we have in the CWD:" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "ls nbimp" + ], + "language": "python", + "metadata": {}, + "outputs": [ + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "__init__.py __init__.pyc bs.ipynb mynotebook.ipynb \u001b[34mnbs\u001b[m\u001b[m/\r\n" + ] + } + ], + "prompt_number": 7 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "So I should be able to `import nbimp.mynotebook`.\n" + ] + }, + { + "cell_type": "heading", + "level": 3, + "metadata": {}, + "source": [ + "Aside: displaying notebooks" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Here is some simple code to display the contents of a notebook\n", + "with syntax highlighting, etc." + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "from pygments import highlight\n", + "from pygments.lexers import PythonLexer\n", + "from pygments.formatters import HtmlFormatter\n", + "\n", + "from IPython.display import display, HTML\n", + "\n", + "formatter = HtmlFormatter()\n", + "lexer = PythonLexer()\n", + "\n", + "# publish the CSS for pygments highlighting\n", + "display(HTML(\"\"\"\n", + "\n", + "\"\"\" % formatter.get_style_defs()\n", + "))" + ], + "language": "python", + "metadata": {}, + "outputs": [ + { + "html": [ + "\n", + "\n" + ], + "metadata": {}, + "output_type": "display_data", + "text": [ + "" + ] + } + ], + "prompt_number": 8 + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "def show_notebook(fname):\n", + " \"\"\"display a short summary of the cells of a notebook\"\"\"\n", + " with io.open(fname, 'r', encoding='utf-8') as f:\n", + " nb = current.read(f, 'json')\n", + " html = []\n", + " for cell in nb.worksheets[0].cells:\n", + " html.append(\"

%s cell

\" % cell.cell_type)\n", + " if cell.cell_type == 'code':\n", + " html.append(highlight(cell.input, lexer, formatter))\n", + " else:\n", + " html.append(\"
%s
\" % cell.source)\n", + " display(HTML('\\n'.join(html)))\n", + "\n", + "show_notebook(os.path.join(\"nbimp\", \"mynotebook.ipynb\"))" + ], + "language": "python", + "metadata": {}, + "outputs": [ + { + "html": [ + "

heading cell

\n", + "
My Notebook
\n", + "

code cell

\n", + "
def foo():\n",
+        "    return "foo"\n",
+        "
\n", + "\n", + "

code cell

\n", + "
def has_ip_syntax():\n",
+        "    listing = !ls\n",
+        "    return listing\n",
+        "
\n", + "\n", + "

code cell

\n", + "
def whatsmyname():\n",
+        "    return __name__\n",
+        "
\n" + ], + "metadata": {}, + "output_type": "display_data", + "text": [ + "" + ] + } + ], + "prompt_number": 9 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "So my notebook has a heading cell and some code cells,\n", + "one of which contains some IPython syntax.\n", + "\n", + "Let's see what happens when we import it" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "from nbimp import mynotebook" + ], + "language": "python", + "metadata": {}, + "outputs": [ + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "importing IPython notebook from nbimp/mynotebook.ipynb\n" + ] + } + ], + "prompt_number": 10 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Hooray, it imported! Does it work?" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "mynotebook.foo()" + ], + "language": "python", + "metadata": {}, + "outputs": [ + { + "metadata": {}, + "output_type": "pyout", + "prompt_number": 11, + "text": [ + "'foo'" + ] + } + ], + "prompt_number": 11 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Hooray again!\n", + "\n", + "Even the function that contains IPython syntax works:" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "mynotebook.has_ip_syntax()" + ], + "language": "python", + "metadata": {}, + "outputs": [ + { + "metadata": {}, + "output_type": "pyout", + "prompt_number": 12, + "text": [ + "['Animations Using clear_output.ipynb',\n", + " 'Cell Magics.ipynb',\n", + " 'Custom Display Logic.ipynb',\n", + " 'Cython Magics.ipynb',\n", + " 'Data Publication API.ipynb',\n", + " 'Frontend-Kernel Model.ipynb',\n", + " 'Importing Notebooks.ipynb',\n", + " 'Octave Magic.ipynb',\n", + " 'Part 1 - Running Code.ipynb',\n", + " 'Part 2 - Basic Output.ipynb',\n", + " 'Part 3 - Plotting with Matplotlib.ipynb',\n", + " 'Part 4 - Markdown Cells.ipynb',\n", + " 'Part 5 - Rich Display System.ipynb',\n", + " 'Progress Bars.ipynb',\n", + " 'R Magics.ipynb',\n", + " 'README.md',\n", + " 'Script Magics.ipynb',\n", + " 'SymPy Examples.ipynb',\n", + " 'Trapezoid Rule.ipynb',\n", + " 'Typesetting Math Using MathJax.ipynb',\n", + " 'animation.m4v',\n", + " 'foo.pyx',\n", + " 'lnum.py',\n", + " 'logo',\n", + " 'nbimp',\n", + " 'python-logo.svg',\n", + " 'test.html']" + ] + } + ], + "prompt_number": 12 + }, + { + "cell_type": "heading", + "level": 2, + "metadata": {}, + "source": [ + "Notebooks in packages" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We also have a notebook inside the `nb` package,\n", + "so let's make sure that works as well." + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "ls nbimp/nbs" + ], + "language": "python", + "metadata": {}, + "outputs": [ + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "__init__.py __init__.pyc other.ipynb\r\n" + ] + } + ], + "prompt_number": 13 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Note that the `__init__.py` is necessary for `nb` to be considered a package,\n", + "just like usual." + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "show_notebook(os.path.join(\"nbimp\", \"nbs\", \"other.ipynb\"))" + ], + "language": "python", + "metadata": {}, + "outputs": [ + { + "html": [ + "

markdown cell

\n", + "
This notebook just defines `bar`
\n", + "

code cell

\n", + "
def bar(x):\n",
+        "    return "bar" * x\n",
+        "
\n" + ], + "metadata": {}, + "output_type": "display_data", + "text": [ + "" + ] + } + ], + "prompt_number": 14 + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "from nbimp.nbs import other\n", + "other.bar(5)" + ], + "language": "python", + "metadata": {}, + "outputs": [ + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "importing IPython notebook from nbimp/nbs/other.ipynb\n" + ] + }, + { + "metadata": {}, + "output_type": "pyout", + "prompt_number": 15, + "text": [ + "'barbarbarbarbar'" + ] + } + ], + "prompt_number": 15 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "So now we have importable notebooks, from both the local directory and inside packages.\n", + "\n", + "I can even put a notebook inside IPython, to further demonstrate that this is working properly:" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "import shutil\n", + "from IPython.utils.path import get_ipython_package_dir\n", + "\n", + "utils = os.path.join(get_ipython_package_dir(), 'utils')\n", + "shutil.copy(os.path.join(\"nbimp\", \"mynotebook.ipynb\"),\n", + " os.path.join(utils, \"inside_ipython.ipynb\")\n", + ")" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 16 + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "and import the notebook from `IPython.utils`" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "from IPython.utils import inside_ipython\n", + "inside_ipython.whatsmyname()" + ], + "language": "python", + "metadata": {}, + "outputs": [ + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "importing IPython notebook from /Users/minrk/dev/ip/mine/IPython/utils/inside_ipython.ipynb\n" + ] + }, + { + "metadata": {}, + "output_type": "pyout", + "prompt_number": 17, + "text": [ + "'IPython.utils.inside_ipython'" + ] + } + ], + "prompt_number": 17 + }, + { + "cell_type": "heading", + "level": 2, + "metadata": {}, + "source": [ + "Even Cython magics" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "With a bit of extra magic for handling the IPython interactive namespace during load,\n", + "even magics like `%%cython` can be used:" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "import Cython_Magics" + ], + "language": "python", + "metadata": {}, + "outputs": [ + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "importing IPython notebook from Cython Magics.ipynb\n", + "1000000 loops, best of 3: 439 ns per loop" + ] + }, + { + "output_type": "stream", + "stream": "stdout", + "text": [ + "\n", + "sin(1)= 0.841470984808\n" + ] + } + ], + "prompt_number": 18 + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "Cython_Magics.black_scholes(100.0, 100.0, 1.0, 0.3, 0.03, 0.0, -1)" + ], + "language": "python", + "metadata": {}, + "outputs": [ + { + "metadata": {}, + "output_type": "pyout", + "prompt_number": 19, + "text": [ + "10.327861752731728" + ] + } + ], + "prompt_number": 19 + } + ], + "metadata": {} + } + ] +} \ No newline at end of file diff --git a/examples/notebooks/nbimp/__init__.py b/examples/notebooks/nbimp/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/examples/notebooks/nbimp/__init__.py diff --git a/examples/notebooks/nbimp/mynotebook.ipynb b/examples/notebooks/nbimp/mynotebook.ipynb new file mode 100644 index 0000000..cdc7d43 --- /dev/null +++ b/examples/notebooks/nbimp/mynotebook.ipynb @@ -0,0 +1,59 @@ +{ + "metadata": { + "name": "" + }, + "nbformat": 3, + "nbformat_minor": 0, + "worksheets": [ + { + "cells": [ + { + "cell_type": "heading", + "level": 1, + "metadata": {}, + "source": [ + "My Notebook" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "def foo():\n", + " return \"foo\"" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 1 + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "def has_ip_syntax():\n", + " listing = !ls\n", + " return listing" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 2 + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "def whatsmyname():\n", + " return __name__" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 4 + } + ], + "metadata": {} + } + ] +} \ No newline at end of file diff --git a/examples/notebooks/nbimp/nbs/__init__.py b/examples/notebooks/nbimp/nbs/__init__.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/examples/notebooks/nbimp/nbs/__init__.py diff --git a/examples/notebooks/nbimp/nbs/other.ipynb b/examples/notebooks/nbimp/nbs/other.ipynb new file mode 100644 index 0000000..33f63f6 --- /dev/null +++ b/examples/notebooks/nbimp/nbs/other.ipynb @@ -0,0 +1,33 @@ +{ + "metadata": { + "name": "" + }, + "nbformat": 3, + "nbformat_minor": 0, + "worksheets": [ + { + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This notebook just defines `bar`" + ] + }, + { + "cell_type": "code", + "collapsed": false, + "input": [ + "def bar(x):\n", + " return \"bar\" * x" + ], + "language": "python", + "metadata": {}, + "outputs": [], + "prompt_number": 2 + } + ], + "metadata": {} + } + ] +} \ No newline at end of file