From 5fba156d1f5a1db93b36df8bda7e2b2fead16156 2014-06-26 20:21:26 From: Julia Evans Date: 2014-06-26 20:21:26 Subject: [PATCH] Add preprocessor to execute notebooks --- diff --git a/IPython/nbconvert/exporters/exporter.py b/IPython/nbconvert/exporters/exporter.py index 6fd13c7..60fcb23 100644 --- a/IPython/nbconvert/exporters/exporter.py +++ b/IPython/nbconvert/exporters/exporter.py @@ -71,6 +71,7 @@ class Exporter(LoggingConfigurable): 'IPython.nbconvert.preprocessors.RevealHelpPreprocessor', 'IPython.nbconvert.preprocessors.LatexPreprocessor', 'IPython.nbconvert.preprocessors.ClearOutputPreprocessor', + 'IPython.nbconvert.preprocessors.ExecutePreprocessor', 'IPython.nbconvert.preprocessors.HighlightMagicsPreprocessor'], config=True, help="""List of preprocessors available by default, by name, namespace, diff --git a/IPython/nbconvert/preprocessors/__init__.py b/IPython/nbconvert/preprocessors/__init__.py index 35e61e9..3f1a591 100755 --- a/IPython/nbconvert/preprocessors/__init__.py +++ b/IPython/nbconvert/preprocessors/__init__.py @@ -8,6 +8,7 @@ from .latex import LatexPreprocessor from .csshtmlheader import CSSHTMLHeaderPreprocessor from .highlightmagics import HighlightMagicsPreprocessor from .clearoutput import ClearOutputPreprocessor +from .execute import ExecutePreprocessor # decorated function Preprocessors from .coalescestreams import coalesce_streams diff --git a/IPython/nbconvert/preprocessors/execute.py b/IPython/nbconvert/preprocessors/execute.py new file mode 100644 index 0000000..8de6fba --- /dev/null +++ b/IPython/nbconvert/preprocessors/execute.py @@ -0,0 +1,109 @@ +"""Module containing a preprocessor that removes the outputs from code cells""" + +# Copyright (c) IPython Development Team. +# Distributed under the terms of the Modified BSD License. + +#----------------------------------------------------------------------------- +# Imports +#----------------------------------------------------------------------------- + +import os +import sys + +from Queue import Empty +from IPython.kernel import KernelManager +from IPython.nbformat.current import reads, NotebookNode, writes + +from .base import Preprocessor + + +#----------------------------------------------------------------------------- +# Classes +#----------------------------------------------------------------------------- +class ExecutePreprocessor(Preprocessor): + """ + Executes all the cells in a notebook + """ + def __init__(self, *args, **kwargs): + """ + Start an kernel to run the Python code + """ + super(ExecutePreprocessor, self).__init__(*args, **kwargs) + self.km = KernelManager() + # run %pylab inline, because some notebooks assume this + # even though they shouldn't + self.km.start_kernel(extra_arguments=['--pylab=inline'], stderr=open(os.devnull, 'w')) + self.kc = self.km.client() + self.kc.start_channels() + self.iopub = self.kc.iopub_channel + self.shell = self.kc.shell_channel + + self.shell.execute("pass") + self.shell.get_msg() + + def preprocess_cell(self, cell, resources, cell_index): + """ + Apply a transformation on each code cell. See base.py for details. + """ + if cell.cell_type != 'code': + return cell, resources + try: + outputs = self.run_cell(self.shell, self.iopub, cell) + except Exception as e: + print >> sys.stderr, "failed to run cell:", repr(e) + print >> sys.stderr, cell.input + sys.exit(1) + cell.outputs = outputs + return cell, resources + + @staticmethod + def run_cell(shell, iopub, cell): + # print cell.input + shell.execute(cell.input) + # wait for finish, maximum 20s + shell.get_msg(timeout=20) + outs = [] + + while True: + try: + msg = iopub.get_msg(timeout=0.2) + except Empty: + break + msg_type = msg['msg_type'] + if msg_type in ('status', 'pyin'): + continue + elif msg_type == 'clear_output': + outs = [] + continue + + content = msg['content'] + # print msg_type, content + out = NotebookNode(output_type=msg_type) + + if msg_type == 'stream': + out.stream = content['name'] + out.text = content['data'] + elif msg_type in ('display_data', 'pyout'): + out['metadata'] = content['metadata'] + for mime, data in content['data'].iteritems(): + attr = mime.split('/')[-1].lower() + # this gets most right, but fix svg+html, plain + attr = attr.replace('+xml', '').replace('plain', 'text') + setattr(out, attr, data) + if msg_type == 'pyout': + out.prompt_number = content['execution_count'] + elif msg_type == 'pyerr': + out.ename = content['ename'] + out.evalue = content['evalue'] + out.traceback = content['traceback'] + else: + print >> sys.stderr, "unhandled iopub msg:", msg_type + + outs.append(out) + return outs + + + def __del__(self): + self.kc.stop_channels() + self.km.shutdown_kernel() + del self.km diff --git a/IPython/nbconvert/preprocessors/tests/test_execute.py b/IPython/nbconvert/preprocessors/tests/test_execute.py new file mode 100644 index 0000000..8ad3762 --- /dev/null +++ b/IPython/nbconvert/preprocessors/tests/test_execute.py @@ -0,0 +1,45 @@ +""" +Module with tests for the clearoutput preprocessor. +""" + +# Copyright (c) IPython Development Team. +# Distributed under the terms of the Modified BSD License. + +#----------------------------------------------------------------------------- +# Imports +#----------------------------------------------------------------------------- +import copy + +from IPython.nbformat import current as nbformat + +from .base import PreprocessorTestsBase +from ..execute import ExecutePreprocessor + + +#----------------------------------------------------------------------------- +# Class +#----------------------------------------------------------------------------- + +class TestExecute(PreprocessorTestsBase): + """Contains test functions for execute.py""" + + + def build_preprocessor(self): + """Make an instance of a preprocessor""" + preprocessor = ExecutePreprocessor() + preprocessor.enabled = True + return preprocessor + + def test_constructor(self): + """Can a ExecutePreprocessor be constructed?""" + self.build_preprocessor() + + def test_correct_output(self): + """Test that ExecutePreprocessor evaluates a cell to the right thing""" + nb = self.build_notebook() + res = self.build_resources() + nb.worksheets[0].cells[0].input = "print 'hi!'" + preprocessor = self.build_preprocessor() + nb, res = preprocessor(nb, res) + expected_outputs = [{'output_type': 'stream', 'stream': 'stdout', 'text': 'hi!\n'}] + assert nb.worksheets[0].cells[0].outputs == expected_outputs