Merge pull request #6720 from takluyver/kernelspec-pygments-lexer

Respect kernelspec metadata for syntax highlighting in nbconvert

docs/source/whatsnew/pr/nbconvert-highlight-config-deprecated.rst

@@ -0,0 +1,3
+- Setting the default highlighting language for nbconvert with the config option
+  ``NbConvertBase.default_language`` is deprecated. Nbconvert now respects
+  metadata stored in the :ref:`kernel spec <kernelspecs>`_.

IPython/html/static/notebook/js/notebook.js

@@ -208,9 +208,9 define([
         
         this.events.on('spec_changed.Kernel', function(event, data) {
             that.set_kernelspec_metadata(data);
-            if (data.codemirror_mode) {
-                that.set_codemirror_mode(data.codemirror_mode);
-            }
+            // Mode 'null' should be plain, unhighlighted text.
+            cm_mode = data.codemirror_mode || data.language || 'null'
+            that.set_codemirror_mode(cm_mode);
         });
 
         var collapse_time = function (time) {

IPython/kernel/kernelspec.py

@@ -38,12 +38,16 class KernelSpec(HasTraits):
     display_name = Unicode()
     language = Unicode()
     codemirror_mode = Any() # can be unicode or dict
+    pygments_lexer = Unicode()
     env = Dict()
     resource_dir = Unicode()
     
     def _codemirror_mode_default(self):
         return self.language
     
+    def _pygments_lexer_default(self):
+        return self.language
+    
     @classmethod
     def from_resource_dir(cls, resource_dir):
         """Create a KernelSpec object by reading kernel.json
@@ -56,12 +60,17 class KernelSpec(HasTraits):
         return cls(resource_dir=resource_dir, **kernel_dict)
     
     def to_dict(self):
-        return dict(argv=self.argv,
+        d = dict(argv=self.argv,
                     env=self.env,
                     display_name=self.display_name,
                     language=self.language,
-                    codemirror_mode=self.codemirror_mode,
                    )
+        if self.codemirror_mode != self.language:
+            d['codemirror_mode'] = self.codemirror_mode
+        if self.pygments_lexer != self.language:
+            d['pygments_lexer'] = self.pygments_lexer
+
+        return d
 
     def to_json(self):
         return json.dumps(self.to_dict())
@@ -114,6 +123,7 class KernelSpecManager(HasTraits):
                'language': 'python',
                'codemirror_mode': {'name': 'ipython',
                                    'version': sys.version_info[0]},
+               'pygments_lexer': 'ipython%d' % (3 if PY3 else 2),
               }
 
     @property

IPython/nbconvert/exporters/html.py

@@ -14,7 +14,7
 
 import os
 
-from IPython.nbconvert import preprocessors
+from IPython.nbconvert.filters.highlight import Highlight2HTML
 from IPython.config import Config
 
 from .templateexporter import TemplateExporter
@@ -57,3 +57,10 class HTMLExporter(TemplateExporter):
             })
         c.merge(super(HTMLExporter,self).default_config)
         return c
+
+    def from_notebook_node(self, nb, resources=None, **kw):
+        kernelspec = nb.metadata.get('kernelspec', {})
+        lexer = kernelspec.get('pygments_lexer', kernelspec.get('language', None))
+        self.register_filter('highlight_code',
+                             Highlight2HTML(pygments_lexer=lexer, parent=self))
+        return super(HTMLExporter, self).from_notebook_node(nb, resources, **kw)

IPython/nbconvert/exporters/latex.py

@@ -19,7 +19,7 import os
 from IPython.utils.traitlets import Unicode
 from IPython.config import Config
 
-from IPython.nbconvert import filters, preprocessors
+from IPython.nbconvert.filters.highlight import Highlight2Latex
 from .templateexporter import TemplateExporter
 
 #-----------------------------------------------------------------------------
@@ -87,3 +87,10 class LatexExporter(TemplateExporter):
          })
         c.merge(super(LatexExporter,self).default_config)
         return c
+
+    def from_notebook_node(self, nb, resources=None, **kw):
+        kernelspec = nb.metadata.get('kernelspec', {})
+        lexer = kernelspec.get('pygments_lexer', kernelspec.get('language', None))
+        self.register_filter('highlight_code',
+                             Highlight2Latex(pygments_lexer=lexer, parent=self))
+        return super(LatexExporter, self).from_notebook_node(nb, resources, **kw)

IPython/nbconvert/filters/highlight.py

@@ -11,6 +11,7 from within Jinja templates.
 # not import time, when it may not be needed.
 
 from IPython.nbconvert.utils.base import NbConvertBase
+from warnings import warn
 
 MULTILINE_OUTPUTS = ['text', 'html', 'svg', 'latex', 'javascript', 'json']
 
@@ -20,6 +21,14 __all__ = [
 ]
 
 class Highlight2HTML(NbConvertBase):
+    def __init__(self, pygments_lexer=None, **kwargs):
+        self.pygments_lexer = pygments_lexer or 'ipython3'
+        super(Highlight2HTML, self).__init__(**kwargs)
+
+    def _default_language_changed(self, name, old, new):
+        warn('Setting default_language in config is deprecated, '
+             'please use kernelspecs instead.')
+        self.pygments_lexer = new
 
     def __call__(self, source, language=None, metadata=None):
         """
@@ -35,8 +44,9 class Highlight2HTML(NbConvertBase):
             metadata of the cell to highlight
         """
         from pygments.formatters import HtmlFormatter
+
         if not language:
-            language=self.default_language
+            language=self.pygments_lexer
 
         return _pygments_highlight(source if len(source) > 0 else ' ',
                                    # needed to help post processors:
@@ -45,6 +55,14 class Highlight2HTML(NbConvertBase):
 
 
 class Highlight2Latex(NbConvertBase):
+    def __init__(self, pygments_lexer=None, **kwargs):
+        self.pygments_lexer = pygments_lexer or 'ipython3'
+        super(Highlight2Latex, self).__init__(**kwargs)
+
+    def _default_language_changed(self, name, old, new):
+        warn('Setting default_language in config is deprecated, '
+             'please use kernelspecs instead.')
+        self.pygments_lexer = new
 
     def __call__(self, source, language=None, metadata=None, strip_verbatim=False):
         """
@@ -63,7 +81,7 class Highlight2Latex(NbConvertBase):
         """
         from pygments.formatters import LatexFormatter
         if not language:
-            language=self.default_language
+            language=self.pygments_lexer
 
         latex = _pygments_highlight(source, LatexFormatter(), language, metadata)
         if strip_verbatim:
@@ -90,7 +108,8 def _pygments_highlight(source, output_formatter, language='ipython', metadata=N
     """
     from pygments import highlight
     from pygments.lexers import get_lexer_by_name
-    from IPython.nbconvert.utils.lexers import IPythonLexer
+    from pygments.util import ClassNotFound
+    from IPython.nbconvert.utils.lexers import IPythonLexer, IPython3Lexer
 
     # If the cell uses a magic extension language,
     # use the magic language instead.
@@ -100,9 +119,17 def _pygments_highlight(source, output_formatter, language='ipython', metadata=N
 
         language = metadata['magics_language']
 
-    if language == 'ipython':
+    if language == 'ipython2':
         lexer = IPythonLexer()
+    elif language == 'ipython3':
+        lexer = IPython3Lexer()
     else:
+        try:
         lexer = get_lexer_by_name(language, stripall=True)
+        except ClassNotFound:
+            warn("No lexer found for language %r. Treating as plain text." % language)
+            from pygments.lexers.special import TextLexer
+            lexer = TextLexer()
+
 
     return highlight(source, lexer, output_formatter)

IPython/nbconvert/filters/tests/test_highlight.py

@@ -25,9 +25,7 import xml
 
 highlight2html = Highlight2HTML()
 highlight2latex = Highlight2Latex()
-c = Config()
-c.Highlight2HTML.default_language='ruby'
-highlight2html_ruby = Highlight2HTML(config=c)
+highlight2html_ruby = Highlight2HTML(pygments_lexer='ruby')
 
 class TestHighlight(TestsBase):
     """Contains test functions for highlight.py"""
@@ -37,8 +35,10 class TestHighlight(TestsBase):
         """
         #Hello World Example
 
+        import foo
+
         def say(text):
-            print(text)
+            foo.bar(text)
 
         end
 
@@ -51,7 +51,7 class TestHighlight(TestsBase):
         ]   
 
     tokens = [
-        ['Hello World Example', 'say', 'text', 'print', 'def'],
+        ['Hello World Example', 'say', 'text', 'import', 'def'],
         ['pylab', 'plot']]
 
 
@@ -72,11 +72,13 class TestHighlight(TestsBase):
         rb =  highlight2html_ruby(self.tests[0])
 
         for lang,tkns in [
-                ( ht, ('def','print') ),
+                ( ht, ('def', )),
                 ( rb, ('def','end'  ) )
                 ]:
+            print(tkns)
+            print(lang)
             root = xml.etree.ElementTree.fromstring(lang)
-            assert self._extract_tokens(root,'k') == set(tkns)
+            self.assertEqual(self._extract_tokens(root,'k'), set(tkns))
 
     def _extract_tokens(self, root, cls):
         return set(map(lambda x:x.text,root.findall(".//*[@class='"+cls+"']")))

IPython/nbconvert/templates/html/basic.tpl

@@ -46,7 +46,7 In [ ]:
 {% block input %}
 <div class="inner_cell">
     <div class="input_area">
-{{ cell.input | highlight2html(language=resources.get('language'), metadata=cell.metadata) }}
+{{ cell.input | highlight_code(metadata=cell.metadata) }}
 </div>
 </div>
 {%- endblock input %}

IPython/nbconvert/templates/latex/style_ipython.tplx

@@ -20,7 +20,7
 %===============================================================================
 
 ((* block input scoped *))
-    ((( add_prompt(cell.input | highlight2latex(strip_verbatim=True), cell, 'In ', 'incolor') )))
+    ((( add_prompt(cell.input | highlight_code(strip_verbatim=True), cell, 'In ', 'incolor') )))
 ((* endblock input *))
 
 

IPython/nbconvert/templates/latex/style_python.tplx

@@ -16,6 +16,6
 
 ((* block input scoped *))
     \begin{Verbatim}[commandchars=\\\{\}]
-((( cell.input | highlight2latex(language=resources.get('language'), strip_verbatim=True) | add_prompts )))
+((( cell.input | highlight_code(strip_verbatim=True) | add_prompts )))
     \end{Verbatim}
 ((* endblock input *))

IPython/nbconvert/utils/base.py

@@ -34,7 +34,8 class NbConvertBase(LoggingConfigurable):
                     """
             )
 
-    default_language = Unicode('ipython', config=True, help='default highlight language')
+    default_language = Unicode('ipython', config=True,
+       help='DEPRECATED default highlight language, please use kernelspecs instead')
 
     def __init__(self, **kw):
         super(NbConvertBase, self).__init__(**kw)

docs/source/development/kernels.rst

@@ -118,8 +118,11 JSON serialised dictionary containing the following keys and values:
   identify alternative kernels that can run some code.
 - **codemirror_mode** (optional): The `codemirror mode <http://codemirror.net/mode/index.html>`_
   to use for code in this language. This can be a string or a dictionary, as
-  passed to codemirror config. The string from *language* will be used if this is
-  not provided.
+  passed to codemirror config. This only needs to be specified if it does not
+  match the value in *language*.
+- **pygments_lexer** (optional): The name of a `Pygments lexer <http://pygments.org/docs/lexers/>`_
+  to use for code in this language, as a string. This only needs to be specified
+  if it does not match the value in *language*.
 - **env** (optional): A dictionary of environment variables to set for the kernel.
   These will be added to the current environment variables before the kernel is
   started.