Add HighlightMagicsPreprocessor

HighlightMagicsPreprocessor is in charge of detecting
cells that use language extensions. It tags the cell
metadata with the language used.

Enable HighlightMagicsPreprocessor by default on
latex and html exporters.

IPython/nbconvert/preprocessors/highlightmagics.py

@@ -0,0 +1,104
+"""This preprocessor detect cells using a different language through
+magic extensions such as `%%R` or `%%octave`. Cell's metadata is marked
+so that the appropriate highlighter can be used in the `highlight`
+filter.
+"""
+
+#-----------------------------------------------------------------------------
+# Copyright (c) 2013, the IPython Development Team.
+#
+# Distributed under the terms of the Modified BSD License.
+#
+# The full license is in the file COPYING.txt, distributed with this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+from __future__ import print_function, absolute_import
+
+import re
+
+# Our own imports
+# Needed to override preprocessor
+from .base import (Preprocessor)
+from IPython.utils.traitlets import Dict
+
+#-----------------------------------------------------------------------------
+# Classes
+#-----------------------------------------------------------------------------
+
+
+class HighlightMagicsPreprocessor(Preprocessor):
+    """
+    Detects and tags code cells that use a different languages than Python.
+    """
+
+    # list of magic language extensions and their associated pygment lexers
+    languages = Dict(
+        default_value={
+            '%%R': 'r',
+            '%%bash': 'bash',
+            '%%octave': 'octave',
+            '%%perl': 'perl',
+            '%%ruby': 'ruby'},
+        config=True,
+        help=("Syntax highlighting for magic's extension languages. "
+         "Each item associates a language magic extension such as %%R, "
+         "with a pygments lexer such as r."))
+
+    def __init__(self, config=None, **kw):
+        """Public constructor"""
+
+        super(HighlightMagicsPreprocessor, self).__init__(config=config, **kw)
+
+        # build a regular expression to catch language extensions and choose
+        # an adequate pygments lexer
+        any_language = "|".join(self.languages.keys())
+        self.re_magic_language = re.compile(
+            r'^\s*({0})\s+'.format(any_language))
+
+    def which_magic_language(self, source):
+        """
+        When a cell uses another language through a magic extension,
+        the other language is returned.
+        If no language magic is detected, this function returns None.
+
+        Parameters
+        ----------
+        source: str
+            Source code of the cell to highlight
+        """
+
+        m = self.re_magic_language.match(source)
+
+        if m:
+            # By construction of the re, the matched language must be in the
+            # languages dictionnary
+            assert(m.group(1) in self.languages)
+            return self.languages[m.group(1)]
+        else:
+            return None
+
+    def preprocess_cell(self, cell, resources, cell_index):
+        """
+        Tags cells using a magic extension language
+
+        Parameters
+        ----------
+        cell : NotebookNode cell
+            Notebook cell being processed
+        resources : dictionary
+            Additional resources used in the conversion process.  Allows
+            preprocessors to pass variables into the Jinja engine.
+        cell_index : int
+            Index of the cell being processed (see base.py)
+        """
+
+        # Only tag code cells
+        if hasattr(cell, "input") and cell.cell_type == "code":
+            magic_language = self.which_magic_language(cell.input)
+            if magic_language:
+                cell['metadata']['magics_language'] = magic_language
+        return cell, resources

IPython/nbconvert/preprocessors/tests/test_highlightmagics.py

@@ -0,0 +1,68
+"""
+Module with tests for the HighlightMagics preprocessor
+"""
+
+#-----------------------------------------------------------------------------
+# Copyright (c) 2013, the IPython Development Team.
+#
+# Distributed under the terms of the Modified BSD License.
+#
+# The full license is in the file COPYING.txt, distributed with this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+from .base import PreprocessorTestsBase
+from ..highlightmagics import HighlightMagicsPreprocessor
+
+
+#-----------------------------------------------------------------------------
+# Class
+#-----------------------------------------------------------------------------
+
+class TestHighlightMagics(PreprocessorTestsBase):
+    """Contains test functions for highlightmagics.py"""
+
+
+    def build_preprocessor(self):
+        """Make an instance of a preprocessor"""
+        preprocessor = HighlightMagicsPreprocessor()
+        preprocessor.enabled = True
+        return preprocessor
+
+    def test_constructor(self):
+        """Can a HighlightMagicsPreprocessor be constructed?"""
+        self.build_preprocessor()
+
+    def test_tagging(self):
+        """Test the HighlightMagicsPreprocessor tagging"""
+        nb = self.build_notebook()
+        res = self.build_resources()
+        preprocessor = self.build_preprocessor()
+        nb.worksheets[0].cells[0].input = """%%R -i x,y -o XYcoef
+            lm.fit <- lm(y~x)
+            par(mfrow=c(2,2))
+            print(summary(lm.fit))
+            plot(lm.fit)
+            XYcoef <- coef(lm.fit)"""
+
+        nb, res = preprocessor(nb, res)
+
+        assert('magics_language' in nb.worksheets[0].cells[0]['metadata'])
+
+        self.assertEqual(nb.worksheets[0].cells[0]['metadata']['magics_language'], 'r')
+
+    def test_no_false_positive(self):
+        """Test that HighlightMagicsPreprocessor does not tag false positives"""
+        nb = self.build_notebook()
+        res = self.build_resources()
+        preprocessor = self.build_preprocessor()
+        nb.worksheets[0].cells[0].input = """# this should not be detected
+                print(\"""
+                %%R -i x, y
+                \""")"""
+        nb, res = preprocessor(nb, res)
+
+        assert('magics_language' not in nb.worksheets[0].cells[0]['metadata'])
  No newline at end of file

IPython/nbconvert/exporters/exporter.py

@@ -1,279 +1,280
 """This module defines Exporter, a highly configurable converter
 that uses Jinja2 to export notebook files into different formats.
 """
 
 #-----------------------------------------------------------------------------
 # Copyright (c) 2013, the IPython Development Team.
 #
 # Distributed under the terms of the Modified BSD License.
 #
 # The full license is in the file COPYING.txt, distributed with this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 from __future__ import print_function, absolute_import
 
 # Stdlib imports
 import io
 import os
 import copy
 import collections
 import datetime
 
 
 # IPython imports
 from IPython.config.configurable import LoggingConfigurable
 from IPython.config import Config
 from IPython.nbformat import current as nbformat
 from IPython.utils.traitlets import MetaHasTraits, Unicode, List
 from IPython.utils.importstring import import_item
 from IPython.utils import text, py3compat
 
 from IPython.nbconvert import preprocessors as nbpreprocessors
 
 
 #-----------------------------------------------------------------------------
 # Class
 #-----------------------------------------------------------------------------
 
 class ResourcesDict(collections.defaultdict):
     def __missing__(self, key):
         return ''
 
 
 class Exporter(LoggingConfigurable):
     """
     Class containing methods that sequentially run a list of preprocessors on a 
     NotebookNode object and then return the modified NotebookNode object and 
     accompanying resources dict.
     """
 
     file_extension = Unicode(
         'txt', config=True,
         help="Extension of the file that should be written to disk"
         )
 
     #Configurability, allows the user to easily add filters and preprocessors.
     preprocessors = List(config=True,
         help="""List of preprocessors, by name or namespace, to enable.""")
 
     _preprocessors = None
 
     default_preprocessors = List([nbpreprocessors.coalesce_streams,
                                  nbpreprocessors.SVG2PDFPreprocessor,
                                  nbpreprocessors.ExtractOutputPreprocessor,
                                  nbpreprocessors.CSSHTMLHeaderPreprocessor,
                                  nbpreprocessors.RevealHelpPreprocessor,
                                  nbpreprocessors.LatexPreprocessor,
-                                 nbpreprocessors.SphinxPreprocessor],
+                                 nbpreprocessors.SphinxPreprocessor,
+                                 nbpreprocessors.HighlightMagicsPreprocessor],
         config=True,
         help="""List of preprocessors available by default, by name, namespace, 
         instance, or type.""")
 
 
     def __init__(self, config=None, **kw):
         """
         Public constructor
 
         Parameters
         ----------
         config : config
             User configuration instance.
         """
         if not config:
             config = self.default_config
 
         super(Exporter, self).__init__(config=config, **kw)
 
         #Init
         self._init_preprocessors()
 
 
     @property
     def default_config(self):
         return Config()
 
     def _config_changed(self, name, old, new):
         """When setting config, make sure to start with our default_config"""
         c = self.default_config
         if new:
             c.merge(new)
         if c != old:
             self.config = c
         super(Exporter, self)._config_changed(name, old, c)
 
 
     def from_notebook_node(self, nb, resources=None, **kw):
         """
         Convert a notebook from a notebook node instance.
 
         Parameters
         ----------
         nb : Notebook node
         resources : dict (**kw)
             of additional resources that can be accessed read/write by
             preprocessors.
         """
         nb_copy = copy.deepcopy(nb)
         resources = self._init_resources(resources)
 
         # Preprocess
         nb_copy, resources = self._preprocess(nb_copy, resources)
 
         return nb_copy, resources
 
 
     def from_filename(self, filename, resources=None, **kw):
         """
         Convert a notebook from a notebook file.
 
         Parameters
         ----------
         filename : str
             Full filename of the notebook file to open and convert.
         """
 
         # Pull the metadata from the filesystem.
         if resources is None:
             resources = ResourcesDict()
         if not 'metadata' in resources or resources['metadata'] == '':
             resources['metadata'] = ResourcesDict()
         basename = os.path.basename(filename)
         notebook_name = basename[:basename.rfind('.')]
         resources['metadata']['name'] = notebook_name
 
         modified_date = datetime.datetime.fromtimestamp(os.path.getmtime(filename))
         resources['metadata']['modified_date'] = modified_date.strftime(text.date_format)
 
         with io.open(filename) as f:
             return self.from_notebook_node(nbformat.read(f, 'json'), resources=resources, **kw)
 
 
     def from_file(self, file_stream, resources=None, **kw):
         """
         Convert a notebook from a notebook file.
 
         Parameters
         ----------
         file_stream : file-like object
             Notebook file-like object to convert.
         """
         return self.from_notebook_node(nbformat.read(file_stream, 'json'), resources=resources, **kw)
 
 
     def register_preprocessor(self, preprocessor, enabled=False):
         """
         Register a preprocessor.
         Preprocessors are classes that act upon the notebook before it is
         passed into the Jinja templating engine.  preprocessors are also
         capable of passing additional information to the Jinja
         templating engine.
 
         Parameters
         ----------
         preprocessor : preprocessor
         """
         if preprocessor is None:
             raise TypeError('preprocessor')
         isclass = isinstance(preprocessor, type)
         constructed = not isclass
 
         #Handle preprocessor's registration based on it's type
         if constructed and isinstance(preprocessor, py3compat.string_types):
             #preprocessor is a string, import the namespace and recursively call
             #this register_preprocessor method
             preprocessor_cls = import_item(preprocessor)
             return self.register_preprocessor(preprocessor_cls, enabled)
 
         if constructed and hasattr(preprocessor, '__call__'):
             #preprocessor is a function, no need to construct it.
             #Register and return the preprocessor.
             if enabled:
                 preprocessor.enabled = True
             self._preprocessors.append(preprocessor)
             return preprocessor
 
         elif isclass and isinstance(preprocessor, MetaHasTraits):
             #preprocessor is configurable.  Make sure to pass in new default for
             #the enabled flag if one was specified.
             self.register_preprocessor(preprocessor(parent=self), enabled)
 
         elif isclass:
             #preprocessor is not configurable, construct it
             self.register_preprocessor(preprocessor(), enabled)
 
         else:
             #preprocessor is an instance of something without a __call__
             #attribute.
             raise TypeError('preprocessor')
 
 
     def _init_preprocessors(self):
         """
         Register all of the preprocessors needed for this exporter, disabled
         unless specified explicitly.
         """
         if self._preprocessors is None:
             self._preprocessors = []
 
         #Load default preprocessors (not necessarly enabled by default).
         if self.default_preprocessors:
             for preprocessor in self.default_preprocessors:
                 self.register_preprocessor(preprocessor)
 
         #Load user preprocessors.  Enable by default.
         if self.preprocessors:
             for preprocessor in self.preprocessors:
                 self.register_preprocessor(preprocessor, enabled=True)
 
 
     def _init_resources(self, resources):
 
         #Make sure the resources dict is of ResourcesDict type.
         if resources is None:
             resources = ResourcesDict()
         if not isinstance(resources, ResourcesDict):
             new_resources = ResourcesDict()
             new_resources.update(resources)
             resources = new_resources
 
         #Make sure the metadata extension exists in resources
         if 'metadata' in resources:
             if not isinstance(resources['metadata'], ResourcesDict):
                 resources['metadata'] = ResourcesDict(resources['metadata'])
         else:
             resources['metadata'] = ResourcesDict()
             if not resources['metadata']['name']:
                 resources['metadata']['name'] = 'Notebook'
 
         #Set the output extension
         resources['output_extension'] = self.file_extension
         return resources
 
 
     def _preprocess(self, nb, resources):
         """
         Preprocess the notebook before passing it into the Jinja engine.
         To preprocess the notebook is to apply all of the
 
         Parameters
         ----------
         nb : notebook node
             notebook that is being exported.
         resources : a dict of additional resources that
             can be accessed read/write by preprocessors
         """
 
         # Do a copy.deepcopy first,
         # we are never safe enough with what the preprocessors could do.
         nbc =  copy.deepcopy(nb)
         resc = copy.deepcopy(resources)
 
         #Run each preprocessor on the notebook.  Carry the output along
         #to each preprocessor
         for preprocessor in self._preprocessors:
             nbc, resc = preprocessor(nbc, resc)
         return nbc, resc

IPython/nbconvert/exporters/html.py

@@ -1,52 +1,55
 """
 Exporter that exports Basic HTML.
 """
 
 #-----------------------------------------------------------------------------
 # Copyright (c) 2013, the IPython Development Team.
 #
 # Distributed under the terms of the Modified BSD License.
 #
 # The full license is in the file COPYING.txt, distributed with this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 from IPython.utils.traitlets import Unicode, List
 
 from IPython.nbconvert import preprocessors
 from IPython.config import Config
 
 from .templateexporter import TemplateExporter
 
 #-----------------------------------------------------------------------------
 # Classes
 #-----------------------------------------------------------------------------
 
 class HTMLExporter(TemplateExporter):
     """
     Exports a basic HTML document.  This exporter assists with the export of
     HTML.  Inherit from it if you are writing your own HTML template and need
     custom preprocessors/filters.  If you don't need custom preprocessors/
     filters, just change the 'template_file' config option.  
     """
     
     file_extension = Unicode(
         'html', config=True, 
         help="Extension of the file that should be written to disk"
         )
 
     default_template = Unicode('full', config=True, help="""Flavor of the data 
         format to use.  I.E. 'full' or 'basic'""")
 
     @property
     def default_config(self):
         c = Config({
             'CSSHTMLHeaderPreprocessor':{
                 'enabled':True
-                }          
+                },
+            'HighlightMagicsPreprocessor': {
+                'enabled':True
+                }
             })
         c.merge(super(HTMLExporter,self).default_config)
         return c

IPython/nbconvert/exporters/latex.py

@@ -1,91 +1,94
 """
 Exporter that allows Latex Jinja templates to work.  Contains logic to
 appropriately prepare IPYNB files for export to LaTeX.  Including but 
 not limited to escaping LaTeX, fixing math region tags, using special
 tags to circumvent Jinja/Latex syntax conflicts.
 """
 #-----------------------------------------------------------------------------
 # Copyright (c) 2013, the IPython Development Team.
 #
 # Distributed under the terms of the Modified BSD License.
 #
 # The full license is in the file COPYING.txt, distributed with this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 # Stdlib imports
 import os
 
 # IPython imports
 from IPython.utils.traitlets import Unicode, List
 from IPython.config import Config
 
 from IPython.nbconvert import filters, preprocessors
 from .templateexporter import TemplateExporter
 
 #-----------------------------------------------------------------------------
 # Classes and functions
 #-----------------------------------------------------------------------------
 
 class LatexExporter(TemplateExporter):
     """
     Exports to a Latex template.  Inherit from this class if your template is
     LaTeX based and you need custom tranformers/filters.  Inherit from it if 
     you are writing your own HTML template and need custom tranformers/filters.  
     If you don't need custom tranformers/filters, just change the 
     'template_file' config option.  Place your template in the special "/latex" 
     subfolder of the "../templates" folder.
     """
     
     file_extension = Unicode(
         'tex', config=True, 
         help="Extension of the file that should be written to disk")
 
     default_template = Unicode('article', config=True, help="""Template of the 
         data format to use.  I.E. 'full' or 'basic'""")
 
     #Latex constants
     default_template_path = Unicode(
         os.path.join("..", "templates", "latex"), config=True,
         help="Path where the template files are located.")
 
     template_skeleton_path = Unicode(
         os.path.join("..", "templates", "latex", "skeleton"), config=True,
         help="Path where the template skeleton files are located.") 
 
     #Special Jinja2 syntax that will not conflict when exporting latex.
     jinja_comment_block_start = Unicode("((=", config=True)
     jinja_comment_block_end = Unicode("=))", config=True)
     jinja_variable_block_start = Unicode("(((", config=True)
     jinja_variable_block_end = Unicode(")))", config=True)
     jinja_logic_block_start = Unicode("((*", config=True)
     jinja_logic_block_end = Unicode("*))", config=True)
     
     #Extension that the template files use.    
     template_extension = Unicode(".tplx", config=True)
 
 
     @property
     def default_config(self):
         c = Config({
             'NbConvertBase': {
                 'display_data_priority' : ['latex', 'pdf', 'png', 'jpg', 'svg', 'jpeg', 'text']
                 },
              'ExtractOutputPreprocessor': {
                     'enabled':True
                  },
              'SVG2PDFPreprocessor': {
                     'enabled':True
                  },
              'LatexPreprocessor': {
                     'enabled':True
                  },
              'SphinxPreprocessor': {
                     'enabled':True
+                 },
+             'HighlightMagicsPreprocessor': {
+                    'enabled':True
                  }
          })
         c.merge(super(LatexExporter,self).default_config)
         return c

IPython/nbconvert/preprocessors/__init__.py

@@ -1,12 +1,13
 # Class base Preprocessors
 from .base import Preprocessor
 from .convertfigures import ConvertFiguresPreprocessor
 from .svg2pdf import SVG2PDFPreprocessor
 from .extractoutput import ExtractOutputPreprocessor
 from .revealhelp import RevealHelpPreprocessor
 from .latex import LatexPreprocessor
 from .sphinx import SphinxPreprocessor
 from .csshtmlheader import CSSHTMLHeaderPreprocessor
+from .highlightmagics import HighlightMagicsPreprocessor
 
 # decorated function Preprocessors
 from .coalescestreams import coalesce_streams