Add HighlightMagicsPreprocessor

HighlightMagicsPreprocessor is in charge of detecting
cells that use language extensions. It tags the cell
metadata with the language used.

Enable HighlightMagicsPreprocessor by default on
latex and html exporters.

IPython/nbconvert/preprocessors/highlightmagics.py

@@ -0,0 +1,104
+"""This preprocessor detect cells using a different language through
+magic extensions such as `%%R` or `%%octave`. Cell's metadata is marked
+so that the appropriate highlighter can be used in the `highlight`
+filter.
+"""
+
+#-----------------------------------------------------------------------------
+# Copyright (c) 2013, the IPython Development Team.
+#
+# Distributed under the terms of the Modified BSD License.
+#
+# The full license is in the file COPYING.txt, distributed with this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+from __future__ import print_function, absolute_import
+
+import re
+
+# Our own imports
+# Needed to override preprocessor
+from .base import (Preprocessor)
+from IPython.utils.traitlets import Dict
+
+#-----------------------------------------------------------------------------
+# Classes
+#-----------------------------------------------------------------------------
+
+
+class HighlightMagicsPreprocessor(Preprocessor):
+    """
+    Detects and tags code cells that use a different languages than Python.
+    """
+
+    # list of magic language extensions and their associated pygment lexers
+    languages = Dict(
+        default_value={
+            '%%R': 'r',
+            '%%bash': 'bash',
+            '%%octave': 'octave',
+            '%%perl': 'perl',
+            '%%ruby': 'ruby'},
+        config=True,
+        help=("Syntax highlighting for magic's extension languages. "
+         "Each item associates a language magic extension such as %%R, "
+         "with a pygments lexer such as r."))
+
+    def __init__(self, config=None, **kw):
+        """Public constructor"""
+
+        super(HighlightMagicsPreprocessor, self).__init__(config=config, **kw)
+
+        # build a regular expression to catch language extensions and choose
+        # an adequate pygments lexer
+        any_language = "|".join(self.languages.keys())
+        self.re_magic_language = re.compile(
+            r'^\s*({0})\s+'.format(any_language))
+
+    def which_magic_language(self, source):
+        """
+        When a cell uses another language through a magic extension,
+        the other language is returned.
+        If no language magic is detected, this function returns None.
+
+        Parameters
+        ----------
+        source: str
+            Source code of the cell to highlight
+        """
+
+        m = self.re_magic_language.match(source)
+
+        if m:
+            # By construction of the re, the matched language must be in the
+            # languages dictionnary
+            assert(m.group(1) in self.languages)
+            return self.languages[m.group(1)]
+        else:
+            return None
+
+    def preprocess_cell(self, cell, resources, cell_index):
+        """
+        Tags cells using a magic extension language
+
+        Parameters
+        ----------
+        cell : NotebookNode cell
+            Notebook cell being processed
+        resources : dictionary
+            Additional resources used in the conversion process.  Allows
+            preprocessors to pass variables into the Jinja engine.
+        cell_index : int
+            Index of the cell being processed (see base.py)
+        """
+
+        # Only tag code cells
+        if hasattr(cell, "input") and cell.cell_type == "code":
+            magic_language = self.which_magic_language(cell.input)
+            if magic_language:
+                cell['metadata']['magics_language'] = magic_language
+        return cell, resources

IPython/nbconvert/preprocessors/tests/test_highlightmagics.py

@@ -0,0 +1,68
+"""
+Module with tests for the HighlightMagics preprocessor
+"""
+
+#-----------------------------------------------------------------------------
+# Copyright (c) 2013, the IPython Development Team.
+#
+# Distributed under the terms of the Modified BSD License.
+#
+# The full license is in the file COPYING.txt, distributed with this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+from .base import PreprocessorTestsBase
+from ..highlightmagics import HighlightMagicsPreprocessor
+
+
+#-----------------------------------------------------------------------------
+# Class
+#-----------------------------------------------------------------------------
+
+class TestHighlightMagics(PreprocessorTestsBase):
+    """Contains test functions for highlightmagics.py"""
+
+
+    def build_preprocessor(self):
+        """Make an instance of a preprocessor"""
+        preprocessor = HighlightMagicsPreprocessor()
+        preprocessor.enabled = True
+        return preprocessor
+
+    def test_constructor(self):
+        """Can a HighlightMagicsPreprocessor be constructed?"""
+        self.build_preprocessor()
+
+    def test_tagging(self):
+        """Test the HighlightMagicsPreprocessor tagging"""
+        nb = self.build_notebook()
+        res = self.build_resources()
+        preprocessor = self.build_preprocessor()
+        nb.worksheets[0].cells[0].input = """%%R -i x,y -o XYcoef
+            lm.fit <- lm(y~x)
+            par(mfrow=c(2,2))
+            print(summary(lm.fit))
+            plot(lm.fit)
+            XYcoef <- coef(lm.fit)"""
+
+        nb, res = preprocessor(nb, res)
+
+        assert('magics_language' in nb.worksheets[0].cells[0]['metadata'])
+
+        self.assertEqual(nb.worksheets[0].cells[0]['metadata']['magics_language'], 'r')
+
+    def test_no_false_positive(self):
+        """Test that HighlightMagicsPreprocessor does not tag false positives"""
+        nb = self.build_notebook()
+        res = self.build_resources()
+        preprocessor = self.build_preprocessor()
+        nb.worksheets[0].cells[0].input = """# this should not be detected
+                print(\"""
+                %%R -i x, y
+                \""")"""
+        nb, res = preprocessor(nb, res)
+
+        assert('magics_language' not in nb.worksheets[0].cells[0]['metadata'])
  No newline at end of file

IPython/nbconvert/exporters/exporter.py

@@ -68,7 +68,8 class Exporter(LoggingConfigurable):
                                  nbpreprocessors.CSSHTMLHeaderPreprocessor,
                                  nbpreprocessors.RevealHelpPreprocessor,
                                  nbpreprocessors.LatexPreprocessor,
-                                 nbpreprocessors.SphinxPreprocessor],
+                                 nbpreprocessors.SphinxPreprocessor,
+                                 nbpreprocessors.HighlightMagicsPreprocessor],
         config=True,
         help="""List of preprocessors available by default, by name, namespace, 
         instance, or type.""")

IPython/nbconvert/exporters/html.py

@@ -46,7 +46,10 class HTMLExporter(TemplateExporter):
         c = Config({
             'CSSHTMLHeaderPreprocessor':{
                 'enabled':True
-                }          
+                },
+            'HighlightMagicsPreprocessor': {
+                'enabled':True
+                }
             })
         c.merge(super(HTMLExporter,self).default_config)
         return c

IPython/nbconvert/exporters/latex.py

@@ -85,6 +85,9 class LatexExporter(TemplateExporter):
                  },
              'SphinxPreprocessor': {
                     'enabled':True
+                 },
+             'HighlightMagicsPreprocessor': {
+                    'enabled':True
                  }
          })
         c.merge(super(LatexExporter,self).default_config)

IPython/nbconvert/preprocessors/__init__.py

@@ -7,6 +7,7 from .revealhelp import RevealHelpPreprocessor
 from .latex import LatexPreprocessor
 from .sphinx import SphinxPreprocessor
 from .csshtmlheader import CSSHTMLHeaderPreprocessor
+from .highlightmagics import HighlightMagicsPreprocessor
 
 # decorated function Preprocessors
 from .coalescestreams import coalesce_streams