highlight: cell metadata to decide how to highlight foreign languages

* Remove magic language detection from highlight filter, and
  replace it by a lookup in the cell metadata.

* The highlight filter now expects a full Cell object instead
  of the Cell input.

* Modify templates so they pass a full cell object.

* Update highlight filter tests

IPython/nbconvert/filters/highlight.py

@@ -30,20 +30,6 b' from IPython.nbconvert.utils.lexers import IPythonLexer'
 
 MULTILINE_OUTPUTS = ['text', 'html', 'svg', 'latex', 'javascript', 'json']
 
-# list of magic language extensions and their associated pygment lexers
-magic_languages = {'%%R': 'r',
-                   '%%bash': 'bash',
-                   '%%octave': 'octave',
-                   '%%perl': 'perl',
-                   '%%ruby': 'ruby'
-                   }
-
-# build a RE to catch language extensions and choose an adequate
-# pygments lexer
-re_languages = "|".join(magic_languages.keys())
-re_magic_language = re.compile(r'^\s*({})\s+'.format(re_languages),
-                               re.MULTILINE)
-
 #-----------------------------------------------------------------------------
 # Utility functions
 #-----------------------------------------------------------------------------
@@ -53,79 +39,60 b' __all__ = ['
     'highlight2latex'
 ]
 
-
-def highlight2html(source, language='ipython'):
+def highlight2html(cell, language='ipython'):
     """
     Return a syntax-highlighted version of the input source as html output.
 
     Parameters
     ----------
-    source : str
-        Source code to highlight the syntax of.
+    cell : NotebookNode cell
+        cell to highlight
     language : str
         Language to highlight the syntax of.
     """
 
-    return _pygment_highlight(source, HtmlFormatter(), language)
+    return _pygment_highlight(cell, HtmlFormatter(), language)
 
 
-def highlight2latex(source, language='ipython'):
+def highlight2latex(cell, language='ipython'):
     """
     Return a syntax-highlighted version of the input source as latex output.
 
     Parameters
     ----------
-    source : str
-        Source code to highlight the syntax of.
+    cell : NotebookNode cell
+        cell to highlight
     language : str
         Language to highlight the syntax of.
     """
-    return _pygment_highlight(source, LatexFormatter(), language)
-
-
-def which_magic_language(source):
-    """
-    When a cell uses another language through a magic extension,
-    the other language is returned.
-    If no language magic is detected, this function returns None.
-
-    Parameters
-    ----------
-    source: str
-        Source code of the cell to highlight
-    """
+    return _pygment_highlight(cell, LatexFormatter(), language)
 
-    m = re_magic_language.search(source)
-
-    if m:
-        # By construction of the re, the matched language must be in the
-        # language dictionnary
-        assert(m.group(1) in magic_languages)
-        return magic_languages[m.group(1)]
-    else:
-        return None
 
 
-def _pygment_highlight(source, output_formatter, language='ipython'):
+def _pygment_highlight(cell, output_formatter, language='ipython'):
     """
     Return a syntax-highlighted version of the input source
 
     Parameters
     ----------
-    source : str
-        Source code to highlight the syntax of.
+    cell : NotebookNode cell
+        cell to highlight
     output_formatter : Pygments formatter
     language : str
         Language to highlight the syntax of.
     """
 
-    magic_language = which_magic_language(source)
-    if magic_language:
-        language = magic_language
+    # If the cell uses a magic extension language,
+    # use the magic language instead.
+    if language == 'ipython' \
+        and 'metadata' in cell \
+        and 'magics_language' in cell['metadata']:
+
+        language = cell['metadata']['magics_language']
 
     if language == 'ipython':
         lexer = IPythonLexer()
     else:
         lexer = get_lexer_by_name(language, stripall=True)
 
-    return pygements_highlight(source, lexer, output_formatter)
+    return pygements_highlight(cell, lexer, output_formatter)

IPython/nbconvert/filters/tests/test_highlight.py

@@ -15,7 +15,7 b' Module with tests for Highlight'
 #-----------------------------------------------------------------------------
 
 from ...tests.base import TestsBase
-from ..highlight import highlight2html, highlight2latex, which_magic_language
+from ..highlight import highlight2html, highlight2latex
 
 
 #-----------------------------------------------------------------------------
@@ -39,59 +39,27 b' class TestHighlight(TestsBase):'
         %%pylab
         plot(x,y, 'r')
         """
-        ]
+        ]   
 
     tokens = [
         ['Hello World Example', 'say', 'text', 'print', 'def'],
         ['pylab', 'plot']]
 
+
     def test_highlight2html(self):
         """highlight2html test"""
         for index, test in enumerate(self.tests):
             self._try_highlight(highlight2html, test, self.tokens[index])
 
+
     def test_highlight2latex(self):
         """highlight2latex test"""
         for index, test in enumerate(self.tests):
             self._try_highlight(highlight2latex, test, self.tokens[index])
 
+
     def _try_highlight(self, method, test, tokens):
         """Try highlighting source, look for key tokens"""
         results = method(test)
         for token in tokens:
             assert token in results
-
-    magic_tests = {
-            'r':
-                """%%R -i x,y -o XYcoef
-                lm.fit <- lm(y~x)
-                par(mfrow=c(2,2))
-                print(summary(lm.fit))
-                plot(lm.fit)
-                XYcoef <- coef(lm.fit)
-                """,
-            'bash':
-                """# the following code is in bash
-                %%bash
-                echo "test" > out
-                """,
-            'ipython':
-                """# this should not be detected
-                print("%%R")
-                """
-            }
-
-    def test_highlight_rmagic(self):
-        """Test %%R magic highlighting"""
-        language = which_magic_language(self.magic_tests['r'])
-        assert language == 'r'
-
-    def test_highlight_bashmagic(self):
-        """Test %%bash magic highlighting"""
-        language = which_magic_language(self.magic_tests['bash'])
-        assert language == 'bash'
-
-    def test_highlight_interferemagic(self):
-        """Test that magic highlighting does not interfere with ipython code"""
-        language = which_magic_language(self.magic_tests['ipython'])
-        assert language == None
  No newline at end of file

IPython/nbconvert/templates/html_basic.tpl

@@ -36,7 +36,7 b' In [{{ cell.prompt_number }}]:'
 
 {% block input %}
 <div class="input_area box-flex1">
-{{ cell.input | highlight2html }}
+{{ cell | highlight2html }}
 </div>
 {%- endblock input %}
 

IPython/nbconvert/templates/latex/sphinx.tplx

@@ -284,7 +284,7 b' Note: For best display, use latex syntax highlighting. =))'
         \vspace{-25pt}
 
         % Add contents below.
-        ((( cell.input | highlight2latex )))
+        ((( cell | highlight2latex )))
 
     ((* elif resources.sphinx.outputstyle == 'notebook' *))
         \vspace{6pt}
@@ -292,7 +292,7 b' Note: For best display, use latex syntax highlighting. =))'
         \vspace{-2.65\baselineskip}
         \begin{ColorVerbatim}
             \vspace{-0.7\baselineskip}
-            ((( cell.input | highlight2latex )))
+            ((( cell | highlight2latex )))
             ((* if cell.input == None or cell.input == '' *))
                 \vspace{0.3\baselineskip}
             ((* else *))