Merge pull request #6053 from minrk/execute-preprocessor

Add execute preprocessor

closes  #5639

IPython/nbconvert/preprocessors/execute.py

@@ -0,0 +1,165 b''
+"""Module containing a preprocessor that removes the outputs from code cells"""
+
+# Copyright (c) IPython Development Team.
+# Distributed under the terms of the Modified BSD License.
+
+import os
+import sys
+
+try:
+    from queue import Empty  # Py 3
+except ImportError:
+    from Queue import Empty  # Py 2
+
+from IPython.utils.traitlets import List, Unicode
+
+from IPython.nbformat.current import reads, NotebookNode, writes
+from .base import Preprocessor
+from IPython.utils.traitlets import Integer
+
+class ExecutePreprocessor(Preprocessor):
+    """
+    Executes all the cells in a notebook
+    """
+    
+    timeout = Integer(30, config=True,
+        help="The time to wait (in seconds) for output from executions."
+    )
+    # FIXME: to be removed with nbformat v4
+    # map msg_type to v3 output_type
+    msg_type_map = {
+        "error" : "pyerr",
+        "execute_result" : "pyout",
+    }
+    
+    # FIXME: to be removed with nbformat v4
+    # map mime-type to v3 mime-type keys
+    mime_map = {
+        "text/plain" : "text",
+        "text/html" : "html",
+        "image/svg+xml" : "svg",
+        "image/png" : "png",
+        "image/jpeg" : "jpeg",
+        "text/latex" : "latex",
+        "application/json" : "json",
+        "application/javascript" : "javascript",
+    }
+    
+    extra_arguments = List(Unicode)
+    
+    def _create_client(self):
+        from IPython.kernel import KernelManager
+        self.km = KernelManager()
+        self.km.write_connection_file()
+        self.kc = self.km.client()
+        self.kc.start_channels()
+        self.km.start_kernel(extra_arguments=self.extra_arguments, stderr=open(os.devnull, 'w'))
+        self.iopub = self.kc.iopub_channel
+        self.shell = self.kc.shell_channel
+        self.shell.kernel_info()
+        try:
+            self.shell.get_msg(timeout=self.timeout)
+        except Empty:
+            self.log.error("Timeout waiting for kernel_info reply")
+            raise
+        # flush IOPub
+        while True:
+            try:
+                self.iopub.get_msg(block=True, timeout=0.25)
+            except Empty:
+                break
+
+    def _shutdown_client(self):
+        self.kc.stop_channels()
+        self.km.shutdown_kernel()
+        del self.km
+
+    def preprocess(self, nb, resources):
+        self._create_client()
+        nb, resources = super(ExecutePreprocessor, self).preprocess(nb, resources)
+        self._shutdown_client()
+        return nb, resources
+
+    def preprocess_cell(self, cell, resources, cell_index):
+        """
+        Apply a transformation on each code cell. See base.py for details.
+        """
+        if cell.cell_type != 'code':
+            return cell, resources
+        try:
+            outputs = self.run_cell(self.shell, self.iopub, cell)
+        except Exception as e:
+            self.log.error("failed to run cell: " + repr(e))
+            self.log.error(str(cell.input))
+            raise
+        cell.outputs = outputs
+        return cell, resources
+
+    def run_cell(self, shell, iopub, cell):
+        msg_id = shell.execute(cell.input)
+        self.log.debug("Executing cell:\n%s", cell.input)
+        # wait for finish, with timeout
+        while True:
+            try:
+                msg = shell.get_msg(timeout=self.timeout)
+            except Empty:
+                self.log.error("Timeout waiting for execute reply")
+                raise
+            if msg['parent_header'].get('msg_id') == msg_id:
+                break
+            else:
+                # not our reply
+                continue
+        
+        outs = []
+
+        while True:
+            try:
+                msg = iopub.get_msg(timeout=self.timeout)
+            except Empty:
+                self.log.warn("Timeout waiting for IOPub output")
+                break
+            if msg['parent_header'].get('msg_id') != msg_id:
+                # not an output from our execution
+                continue
+
+            msg_type = msg['msg_type']
+            self.log.debug("output: %s", msg_type)
+            content = msg['content']
+            if msg_type == 'status':
+                if content['execution_state'] == 'idle':
+                    break
+                else:
+                    continue
+            elif msg_type in {'execute_input', 'pyin'}:
+                continue
+            elif msg_type == 'clear_output':
+                outs = []
+                continue
+
+            out = NotebookNode(output_type=self.msg_type_map.get(msg_type, msg_type))
+
+            # set the prompt number for the input and the output
+            if 'execution_count' in content:
+                cell['prompt_number'] = content['execution_count']
+                out.prompt_number = content['execution_count']
+
+            if msg_type == 'stream':
+                out.stream = content['name']
+                out.text = content['data']
+            elif msg_type in ('display_data', 'execute_result'):
+                out['metadata'] = content['metadata']
+                for mime, data in content['data'].items():
+                    # map mime-type keys to nbformat v3 keys
+                    # this will be unnecessary in nbformat v4
+                    key = self.mime_map.get(mime, mime)
+                    out[key] = data
+            elif msg_type == 'error':
+                out.ename = content['ename']
+                out.evalue = content['evalue']
+                out.traceback = content['traceback']
+            else:
+                self.log.error("unhandled iopub msg: " + msg_type)
+
+            outs.append(out)
+        return outs

IPython/nbconvert/preprocessors/tests/files/Clear Output.ipynb

@@ -0,0 +1,46 b''
+{
+ "metadata": {
+  "name": ""
+ },
+ "nbformat": 3,
+ "nbformat_minor": 0,
+ "worksheets": [
+  {
+   "cells": [
+    {
+     "cell_type": "code",
+     "collapsed": false,
+     "input": [
+      "from IPython.display import clear_output"
+     ],
+     "language": "python",
+     "metadata": {},
+     "outputs": [],
+     "prompt_number": 1
+    },
+    {
+     "cell_type": "code",
+     "collapsed": false,
+     "input": [
+      "for i in range(10):\n",
+      "    clear_output()\n",
+      "    print(i)"
+     ],
+     "language": "python",
+     "metadata": {},
+     "outputs": [
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        "9\n"
+       ]
+      }
+     ],
+     "prompt_number": 2
+    }
+   ],
+   "metadata": {}
+  }
+ ]
+}
  No newline at end of file

IPython/nbconvert/preprocessors/tests/files/Factorials.ipynb

@@ -0,0 +1,55 b''
+{
+ "metadata": {
+  "name": ""
+ },
+ "nbformat": 3,
+ "nbformat_minor": 0,
+ "worksheets": [
+  {
+   "cells": [
+    {
+     "cell_type": "code",
+     "collapsed": false,
+     "input": [
+      "i, j = 1, 1"
+     ],
+     "language": "python",
+     "metadata": {},
+     "outputs": [],
+     "prompt_number": 1
+    },
+    {
+     "cell_type": "code",
+     "collapsed": false,
+     "input": [
+      "for m in range(10):\n",
+      "    i, j = j, i + j\n",
+      "    print(j)"
+     ],
+     "language": "python",
+     "metadata": {},
+     "outputs": [
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        "2\n",
+        "3\n",
+        "5\n",
+        "8\n",
+        "13\n",
+        "21\n",
+        "34\n",
+        "55\n",
+        "89\n",
+        "144\n"
+       ]
+      }
+     ],
+     "prompt_number": 2
+    }
+   ],
+   "metadata": {}
+  }
+ ]
+}
  No newline at end of file

IPython/nbconvert/preprocessors/tests/files/HelloWorld.ipynb

@@ -0,0 +1,33 b''
+{
+ "metadata": {
+  "name": ""
+ },
+ "nbformat": 3,
+ "nbformat_minor": 0,
+ "worksheets": [
+  {
+   "cells": [
+    {
+     "cell_type": "code",
+     "collapsed": false,
+     "input": [
+      "print(\"Hello World\")"
+     ],
+     "language": "python",
+     "metadata": {},
+     "outputs": [
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        "Hello World\n"
+       ]
+      }
+     ],
+     "prompt_number": 1
+    }
+   ],
+   "metadata": {}
+  }
+ ]
+}
  No newline at end of file

IPython/nbconvert/preprocessors/tests/files/Inline Image.ipynb

@@ -0,0 +1,36 b''
+{
+ "metadata": {
+  "name": ""
+ },
+ "nbformat": 3,
+ "nbformat_minor": 0,
+ "worksheets": [
+  {
+   "cells": [
+    {
+     "cell_type": "code",
+     "collapsed": false,
+     "input": [
+      "from IPython.display import Image"
+     ],
+     "language": "python",
+     "metadata": {},
+     "outputs": [],
+     "prompt_number": 1
+    },
+    {
+     "cell_type": "code",
+     "collapsed": false,
+     "input": [
+      "Image('../input/python.png');"
+     ],
+     "language": "python",
+     "metadata": {},
+     "outputs": [],
+     "prompt_number": 2
+    }
+   ],
+   "metadata": {}
+  }
+ ]
+}
  No newline at end of file

IPython/nbconvert/preprocessors/tests/files/SVG.ipynb

@@ -0,0 +1,53 b''
+{
+ "metadata": {
+  "name": ""
+ },
+ "nbformat": 3,
+ "nbformat_minor": 0,
+ "worksheets": [
+  {
+   "cells": [
+    {
+     "cell_type": "code",
+     "collapsed": false,
+     "input": [
+      "from IPython.display import SVG"
+     ],
+     "language": "python",
+     "metadata": {},
+     "outputs": [],
+     "prompt_number": 1
+    },
+    {
+     "cell_type": "code",
+     "collapsed": false,
+     "input": [
+      "SVG(data='''\n",
+      "<svg height=\"100\" width=\"100\">\n",
+      "    <circle cx=\"50\" cy=\"50\" r=\"40\" stroke=\"black\" stroke-width=\"2\" fill=\"red\" />\n",
+      "</svg>''')"
+     ],
+     "language": "python",
+     "metadata": {},
+     "outputs": [
+      {
+       "metadata": {},
+       "output_type": "pyout",
+       "prompt_number": 2,
+       "svg": [
+        "<svg height=\"100\" width=\"100\">\n",
+        "    <circle cx=\"50\" cy=\"50\" fill=\"red\" r=\"40\" stroke=\"black\" stroke-width=\"2\"/>\n",
+        "</svg>"
+       ],
+       "text": [
+        "<IPython.core.display.SVG at 0x10428e150>"
+       ]
+      }
+     ],
+     "prompt_number": 2
+    }
+   ],
+   "metadata": {}
+  }
+ ]
+}
  No newline at end of file

IPython/nbconvert/preprocessors/tests/files/Skip Exceptions.ipynb

@@ -0,0 +1,55 b''
+{
+ "metadata": {
+  "name": ""
+ },
+ "nbformat": 3,
+ "nbformat_minor": 0,
+ "worksheets": [
+  {
+   "cells": [
+    {
+     "cell_type": "code",
+     "collapsed": false,
+     "input": [
+      "raise Exception(\"message\")"
+     ],
+     "language": "python",
+     "metadata": {},
+     "outputs": [
+      {
+       "ename": "Exception",
+       "evalue": "message",
+       "output_type": "pyerr",
+       "traceback": [
+        "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m\n\u001b[1;31mException\u001b[0m                                 Traceback (most recent call last)",
+        "\u001b[1;32m<ipython-input-1-335814d14fc1>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[1;32mraise\u001b[0m \u001b[0mException\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"message\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
+        "\u001b[1;31mException\u001b[0m: message"
+       ]
+      }
+     ],
+     "prompt_number": 1
+    },
+    {
+     "cell_type": "code",
+     "collapsed": false,
+     "input": [
+      "print('ok')"
+     ],
+     "language": "python",
+     "metadata": {},
+     "outputs": [
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        "ok\n"
+       ]
+      }
+     ],
+     "prompt_number": 2
+    }
+   ],
+   "metadata": {}
+  }
+ ]
+}
  No newline at end of file

IPython/nbconvert/preprocessors/tests/files/Unicode.ipynb

@@ -0,0 +1,33 b''
+{
+ "metadata": {
+  "name": ""
+ },
+ "nbformat": 3,
+ "nbformat_minor": 0,
+ "worksheets": [
+  {
+   "cells": [
+    {
+     "cell_type": "code",
+     "collapsed": false,
+     "input": [
+      "print('\u2603')"
+     ],
+     "language": "python",
+     "metadata": {},
+     "outputs": [
+      {
+       "output_type": "stream",
+       "stream": "stdout",
+       "text": [
+        "\u2603\n"
+       ]
+      }
+     ],
+     "prompt_number": 1
+    }
+   ],
+   "metadata": {}
+  }
+ ]
+}
  No newline at end of file

IPython/nbconvert/preprocessors/tests/test_execute.py

@@ -0,0 +1,82 b''
+"""
+Module with tests for the execute preprocessor.
+"""
+
+# Copyright (c) IPython Development Team.
+# Distributed under the terms of the Modified BSD License.
+
+import copy
+import glob
+import os
+import re
+
+from IPython.nbformat import current as nbformat
+
+from .base import PreprocessorTestsBase
+from ..execute import ExecutePreprocessor
+
+from IPython.nbconvert.filters import strip_ansi
+
+addr_pat = re.compile(r'0x[0-9a-f]{7,9}')
+
+class TestExecute(PreprocessorTestsBase):
+    """Contains test functions for execute.py"""
+
+    @staticmethod
+    def normalize_output(output):
+        """
+        Normalizes outputs for comparison.
+        """
+        output = dict(output)
+        if 'metadata' in output:
+            del output['metadata']
+        if 'text' in output:
+            output['text'] = re.sub(addr_pat, '<HEXADDR>', output['text'])
+        if 'svg' in output:
+            del output['text']
+        if 'traceback' in output:
+            tb = []
+            for line in output['traceback']:
+                tb.append(strip_ansi(line))
+            output['traceback'] = tb
+            
+        return output
+
+
+    def assert_notebooks_equal(self, expected, actual):
+        expected_cells = expected['worksheets'][0]['cells']
+        actual_cells = actual['worksheets'][0]['cells']
+        assert len(expected_cells) == len(actual_cells)
+
+        for expected_cell, actual_cell in zip(expected_cells, actual_cells):
+            expected_outputs = expected_cell.get('outputs', [])
+            actual_outputs = actual_cell.get('outputs', [])
+            normalized_expected_outputs = list(map(self.normalize_output, expected_outputs))
+            normalized_actual_outputs = list(map(self.normalize_output, actual_outputs))
+            assert normalized_expected_outputs == normalized_actual_outputs
+
+
+    def build_preprocessor(self):
+        """Make an instance of a preprocessor"""
+        preprocessor = ExecutePreprocessor()
+        preprocessor.enabled = True
+        return preprocessor
+
+
+    def test_constructor(self):
+        """Can a ExecutePreprocessor be constructed?"""
+        self.build_preprocessor()
+
+
+    def test_run_notebooks(self):
+        """Runs a series of test notebooks and compares them to their actual output"""
+        current_dir = os.path.dirname(__file__)
+        input_files = glob.glob(os.path.join(current_dir, 'files', '*.ipynb'))
+        for filename in input_files:
+            with open(os.path.join(current_dir, 'files', filename)) as f:
+                input_nb = nbformat.read(f, 'ipynb')
+            res = self.build_resources()
+            preprocessor = self.build_preprocessor()
+            output_nb, _ = preprocessor(copy.deepcopy(input_nb), res)
+            self.assert_notebooks_equal(output_nb, input_nb)
+

docs/source/whatsnew/pr/execute-preprocessor

@@ -0,0 +1,4 b''
+New preprocessor for nbconvert that executes all the code cells in a notebook.
+To run a notebook and save its output in a new notebook::
+
+    ipython nbconvert InputNotebook --ExecutePreprocessor.enabled=True --to notebook --output Executed

IPython/nbconvert/exporters/exporter.py

@@ -71,6 +71,7 b' class Exporter(LoggingConfigurable):'
                                   'IPython.nbconvert.preprocessors.RevealHelpPreprocessor',
                                   'IPython.nbconvert.preprocessors.LatexPreprocessor',
                                   'IPython.nbconvert.preprocessors.ClearOutputPreprocessor',
+                                  'IPython.nbconvert.preprocessors.ExecutePreprocessor',
                                   'IPython.nbconvert.preprocessors.HighlightMagicsPreprocessor'],
         config=True,
         help="""List of preprocessors available by default, by name, namespace, 

IPython/nbconvert/preprocessors/__init__.py

@@ -8,6 +8,7 b' from .latex import LatexPreprocessor'
 from .csshtmlheader import CSSHTMLHeaderPreprocessor
 from .highlightmagics import HighlightMagicsPreprocessor
 from .clearoutput import ClearOutputPreprocessor
+from .execute import ExecutePreprocessor
 
 # decorated function Preprocessors
 from .coalescestreams import coalesce_streams

setupbase.py

@@ -185,7 +185,11 b' def find_package_data():'
         'IPython.html.tests' : js_tests,
         'IPython.qt.console' : ['resources/icon/*.svg'],
         'IPython.nbconvert' : nbconvert_templates +
-            ['tests/files/*.*', 'exporters/tests/files/*.*'],
+            [
+                'tests/files/*.*',
+                'exporters/tests/files/*.*',
+                'preprocessors/tests/files/*.*',
+            ],
         'IPython.nbconvert.filters' : ['marked.js'],
         'IPython.nbformat' : ['tests/*.ipynb','v3/v3.withref.json']
     }