Merge pull request #4187 from pablooliveira/pr-4184

Select highlighter for cell magic languages

IPython/nbconvert/preprocessors/highlightmagics.py

@@ -0,0 +1,113 b''
+"""This preprocessor detect cells using a different language through
+magic extensions such as `%%R` or `%%octave`. Cell's metadata is marked
+so that the appropriate highlighter can be used in the `highlight`
+filter.
+"""
+
+#-----------------------------------------------------------------------------
+# Copyright (c) 2013, the IPython Development Team.
+#
+# Distributed under the terms of the Modified BSD License.
+#
+# The full license is in the file COPYING.txt, distributed with this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+from __future__ import print_function, absolute_import
+
+import re
+
+# Our own imports
+from .base import Preprocessor
+from IPython.utils.traitlets import Dict
+
+#-----------------------------------------------------------------------------
+# Classes
+#-----------------------------------------------------------------------------
+
+
+class HighlightMagicsPreprocessor(Preprocessor):
+    """
+    Detects and tags code cells that use a different languages than Python.
+    """
+
+    # list of magic language extensions and their associated pygment lexers
+    default_languages = Dict(
+        default_value={
+            '%%R': 'r',
+            '%%bash': 'bash',
+            '%%cython': 'cython',
+            '%%javascript': 'javascript',
+            '%%julia': 'julia',
+            '%%latex': 'latex',
+            '%%octave': 'octave',
+            '%%perl': 'perl',
+            '%%ruby': 'ruby',
+            '%%sh': 'sh'})
+
+    # user defined language extensions
+    languages = Dict(
+        config=True,
+        help=("Syntax highlighting for magic's extension languages. "
+         "Each item associates a language magic extension such as %%R, "
+         "with a pygments lexer such as r."))
+
+    def __init__(self, config=None, **kw):
+        """Public constructor"""
+
+        super(HighlightMagicsPreprocessor, self).__init__(config=config, **kw)
+
+        # Update the default languages dict with the user configured ones
+        self.default_languages.update(self.languages)
+
+        # build a regular expression to catch language extensions and choose
+        # an adequate pygments lexer
+        any_language = "|".join(self.default_languages.keys())
+        self.re_magic_language = re.compile(
+            r'^\s*({0})\s+'.format(any_language))
+
+    def which_magic_language(self, source):
+        """
+        When a cell uses another language through a magic extension,
+        the other language is returned.
+        If no language magic is detected, this function returns None.
+
+        Parameters
+        ----------
+        source: str
+            Source code of the cell to highlight
+        """
+
+        m = self.re_magic_language.match(source)
+
+        if m:
+            # By construction of the re, the matched language must be in the
+            # languages dictionary
+            return self.default_languages[m.group(1)]
+        else:
+            return None
+
+    def preprocess_cell(self, cell, resources, cell_index):
+        """
+        Tags cells using a magic extension language
+
+        Parameters
+        ----------
+        cell : NotebookNode cell
+            Notebook cell being processed
+        resources : dictionary
+            Additional resources used in the conversion process.  Allows
+            preprocessors to pass variables into the Jinja engine.
+        cell_index : int
+            Index of the cell being processed (see base.py)
+        """
+
+        # Only tag code cells
+        if hasattr(cell, "input") and cell.cell_type == "code":
+            magic_language = self.which_magic_language(cell.input)
+            if magic_language:
+                cell['metadata']['magics_language'] = magic_language
+        return cell, resources

IPython/nbconvert/preprocessors/tests/test_highlightmagics.py

@@ -0,0 +1,68 b''
+"""
+Module with tests for the HighlightMagics preprocessor
+"""
+
+#-----------------------------------------------------------------------------
+# Copyright (c) 2013, the IPython Development Team.
+#
+# Distributed under the terms of the Modified BSD License.
+#
+# The full license is in the file COPYING.txt, distributed with this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+from .base import PreprocessorTestsBase
+from ..highlightmagics import HighlightMagicsPreprocessor
+
+
+#-----------------------------------------------------------------------------
+# Class
+#-----------------------------------------------------------------------------
+
+class TestHighlightMagics(PreprocessorTestsBase):
+    """Contains test functions for highlightmagics.py"""
+
+
+    def build_preprocessor(self):
+        """Make an instance of a preprocessor"""
+        preprocessor = HighlightMagicsPreprocessor()
+        preprocessor.enabled = True
+        return preprocessor
+
+    def test_constructor(self):
+        """Can a HighlightMagicsPreprocessor be constructed?"""
+        self.build_preprocessor()
+
+    def test_tagging(self):
+        """Test the HighlightMagicsPreprocessor tagging"""
+        nb = self.build_notebook()
+        res = self.build_resources()
+        preprocessor = self.build_preprocessor()
+        nb.worksheets[0].cells[0].input = """%%R -i x,y -o XYcoef
+            lm.fit <- lm(y~x)
+            par(mfrow=c(2,2))
+            print(summary(lm.fit))
+            plot(lm.fit)
+            XYcoef <- coef(lm.fit)"""
+
+        nb, res = preprocessor(nb, res)
+
+        assert('magics_language' in nb.worksheets[0].cells[0]['metadata'])
+
+        self.assertEqual(nb.worksheets[0].cells[0]['metadata']['magics_language'], 'r')
+
+    def test_no_false_positive(self):
+        """Test that HighlightMagicsPreprocessor does not tag false positives"""
+        nb = self.build_notebook()
+        res = self.build_resources()
+        preprocessor = self.build_preprocessor()
+        nb.worksheets[0].cells[0].input = """# this should not be detected
+                print(\"""
+                %%R -i x, y
+                \""")"""
+        nb, res = preprocessor(nb, res)
+
+        assert('magics_language' not in nb.worksheets[0].cells[0]['metadata'])
  No newline at end of file

IPython/nbconvert/exporters/exporter.py

@@ -68,7 +68,8 b' class Exporter(LoggingConfigurable):'
                                  nbpreprocessors.CSSHTMLHeaderPreprocessor,
                                  nbpreprocessors.RevealHelpPreprocessor,
                                  nbpreprocessors.LatexPreprocessor,
-                                 nbpreprocessors.SphinxPreprocessor],
+                                 nbpreprocessors.SphinxPreprocessor,
+                                 nbpreprocessors.HighlightMagicsPreprocessor],
         config=True,
         help="""List of preprocessors available by default, by name, namespace, 
         instance, or type.""")

IPython/nbconvert/exporters/html.py

@@ -46,7 +46,10 b' class HTMLExporter(TemplateExporter):'
         c = Config({
             'CSSHTMLHeaderPreprocessor':{
                 'enabled':True
-                }          
+                },
+            'HighlightMagicsPreprocessor': {
+                'enabled':True
+                }
             })
         c.merge(super(HTMLExporter,self).default_config)
         return c

IPython/nbconvert/exporters/latex.py

@@ -85,6 +85,9 b' class LatexExporter(TemplateExporter):'
                  },
              'SphinxPreprocessor': {
                     'enabled':True
+                 },
+             'HighlightMagicsPreprocessor': {
+                    'enabled':True
                  }
          })
         c.merge(super(LatexExporter,self).default_config)

IPython/nbconvert/exporters/slides.py

@@ -47,6 +47,9 b' class SlidesExporter(TemplateExporter):'
             'RevealHelpPreprocessor':{
                 'enabled':True,
                 },                
+            'HighlightMagicsPreprocessor': {
+                'enabled':True
+                }
             })
         c.merge(super(SlidesExporter,self).default_config)
         return c

IPython/nbconvert/filters/highlight.py

@@ -14,7 +14,7 b' from within Jinja templates.'
 # Imports
 #-----------------------------------------------------------------------------
 
-from  pygments import highlight as pygements_highlight
+from pygments import highlight as pygements_highlight
 from pygments.lexers import get_lexer_by_name
 from pygments.formatters import HtmlFormatter
 from pygments.formatters import LatexFormatter
@@ -37,52 +37,66 b' __all__ = ['
     'highlight2latex'
 ]
 
-
-def highlight2html(source, language='ipython'):
+def highlight2html(source, language='ipython', metadata=None):
     """
     Return a syntax-highlighted version of the input source as html output.
-    
+
     Parameters
     ----------
     source : str
-        Source code to highlight the syntax of.
+        source of the cell to highlight.
     language : str
         Language to highlight the syntax of.
+    metadata : NotebookNode cell metadata
+        metadata of the cell to highlight.
     """
-    
-    return _pygment_highlight(source, HtmlFormatter(), language)
+
+    return _pygment_highlight(source, HtmlFormatter(), language, metadata)
 
 
-def highlight2latex(source, language='ipython'):
+def highlight2latex(source, language='ipython', metadata=None):
     """
     Return a syntax-highlighted version of the input source as latex output.
-    
+
     Parameters
     ----------
     source : str
-        Source code to highlight the syntax of.
+        source of the cell to highlight.
     language : str
         Language to highlight the syntax of.
+    metadata : NotebookNode cell metadata
+        metadata of the cell to highlight.
     """
-    return _pygment_highlight(source, LatexFormatter(), language)
+    return _pygment_highlight(source, LatexFormatter(), language, metadata)
+
 
 
-def _pygment_highlight(source, output_formatter, language='ipython'):
+def _pygment_highlight(source, output_formatter, language='ipython', metadata=None):
     """
     Return a syntax-highlighted version of the input source
-    
+
     Parameters
     ----------
     source : str
-        Source code to highlight the syntax of.
+        source of the cell to highlight.
     output_formatter : Pygments formatter
     language : str
         Language to highlight the syntax of.
+    metadata : NotebookNode cell metadata
+        metadata of the cell to highlight.
     """
-    
+
+    # If the cell uses a magic extension language,
+    # use the magic language instead.
+    if language == 'ipython' \
+        and metadata \
+        and 'magics_language' in metadata:
+
+        language = metadata['magics_language']
+
     if language == 'ipython':
         lexer = IPythonLexer()
     else:
         lexer = get_lexer_by_name(language, stripall=True)
 
-    return pygements_highlight(source, lexer, output_formatter) 
+    return pygements_highlight(source, lexer, output_formatter)

IPython/nbconvert/preprocessors/__init__.py

@@ -7,6 +7,7 b' from .revealhelp import RevealHelpPreprocessor'
 from .latex import LatexPreprocessor
 from .sphinx import SphinxPreprocessor
 from .csshtmlheader import CSSHTMLHeaderPreprocessor
+from .highlightmagics import HighlightMagicsPreprocessor
 
 # decorated function Preprocessors
 from .coalescestreams import coalesce_streams

IPython/nbconvert/templates/html_basic.tpl

@@ -36,7 +36,7 b' In [{{ cell.prompt_number }}]:'
 
 {% block input %}
 <div class="input_area box-flex1">
-{{ cell.input | highlight2html }}
+{{ cell.input | highlight2html(metadata=cell.metadata) }}
 </div>
 {%- endblock input %}
 

IPython/nbconvert/templates/latex/sphinx.tplx

@@ -284,7 +284,7 b' Note: For best display, use latex syntax highlighting. =))'
         \vspace{-25pt}
 
         % Add contents below.
-        ((( cell.input | highlight2latex )))
+        ((( cell.input | highlight2latex(metadata=cell.metadata) )))
 
     ((* elif resources.sphinx.outputstyle == 'notebook' *))
         \vspace{6pt}
@@ -292,7 +292,7 b' Note: For best display, use latex syntax highlighting. =))'
         \vspace{-2.65\baselineskip}
         \begin{ColorVerbatim}
             \vspace{-0.7\baselineskip}
-            ((( cell.input | highlight2latex )))
+            ((( cell.input | highlight2latex(metadata=cell.metadata) )))
             ((* if cell.input == None or cell.input == '' *))
                 \vspace{0.3\baselineskip}
             ((* else *))