Merge pull request #4090 from ellisonbg/citation

Add LaTeX citation handling to nbconvert

IPython/nbconvert/filters/citation.py

@@ -0,0 +1,72
+"""Citation handling for LaTeX output."""
+
+#-----------------------------------------------------------------------------
+# Copyright (c) 2013, the IPython Development Team.
+#
+# Distributed under the terms of the Modified BSD License.
+#
+# The full license is in the file COPYING.txt, distributed with this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Code
+#-----------------------------------------------------------------------------
+
+
+__all__ = ['citation2latex']
+
+
+def citation2latex(s):
+    """Parse citations in Markdown cells.
+    
+    This looks for HTML tags having a data attribute names `data-cite`
+    and replaces it by the call to LaTeX cite command. The tranformation
+    looks like this:
+    
+    `<cite data-cite="granger">(Granger, 2013)</cite>`
+    
+    Becomes
+    
+    `\\cite{granger}`
+    
+    Any HTML tag can be used, which allows the citations to be formatted
+    in HTML in any manner.
+    """
+    try:
+        from lxml import html
+    except ImportError:
+        return s
+
+    tree = html.fragment_fromstring(s, create_parent='div')
+    _process_node_cite(tree)
+    s = html.tostring(tree)
+    if s.endswith('</div>'):
+        s = s[:-6]
+    if s.startswith('<div>'):
+        s = s[5:]
+    return s
+
+
+def _process_node_cite(node):
+    """Do the citation replacement as we walk the lxml tree."""
+    
+    def _get(o, name):
+        value = getattr(o, name, None)
+        return '' if value is None else value
+    
+    if 'data-cite' in node.attrib:
+        cite = '\cite{%(ref)s}' % {'ref': node.attrib['data-cite']}
+        prev = node.getprevious()
+        if prev is not None:
+            prev.tail = _get(prev, 'tail') + cite + _get(node, 'tail')
+        else:
+            parent = node.getparent()
+            if parent is not None:
+                parent.text = _get(parent, 'text') + cite + _get(node, 'tail')
+        try:
+            node.getparent().remove(node)
+        except AttributeError:
+            pass
+    else:
+        for child in node:
+            _process_node_cite(child)

IPython/nbconvert/filters/tests/test_citation.py

@@ -0,0 +1,58
+#-----------------------------------------------------------------------------
+# Copyright (c) 2013, the IPython Development Team.
+#
+# Distributed under the terms of the Modified BSD License.
+#
+# The full license is in the file COPYING.txt, distributed with this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+
+from ..citation import citation2latex
+
+#-----------------------------------------------------------------------------
+# Tests
+#-----------------------------------------------------------------------------
+
+test_md = """
+# My Heading
+
+Lorem ipsum dolor sit amet, consectetur adipiscing elit. Phasellus ac magna non augue
+porttitor scelerisque ac id diam <cite data-cite="granger">Granger</cite>. Mauris elit
+velit, lobortis sed interdum at, vestibulum vitae libero <strong data-cite="fperez">Perez</strong>.
+Lorem ipsum dolor sit amet, consectetur adipiscing elit
+<em data-cite="takluyver">Thomas</em>. Quisque iaculis ligula ut ipsum mattis viverra.
+
+<p>Here is a plain paragraph that should be unaffected.</p>
+
+* One <cite data-cite="jdfreder">Jonathan</cite>.
+* Two <cite data-cite="carreau">Matthias</cite>.
+* Three <cite data-cite="ivanov">Paul</cite>.
+"""
+
+test_md_parsed = """
+# My Heading
+
+Lorem ipsum dolor sit amet, consectetur adipiscing elit. Phasellus ac magna non augue
+porttitor scelerisque ac id diam \cite{granger}. Mauris elit
+velit, lobortis sed interdum at, vestibulum vitae libero \cite{fperez}.
+Lorem ipsum dolor sit amet, consectetur adipiscing elit
+\cite{takluyver}. Quisque iaculis ligula ut ipsum mattis viverra.
+
+<p>Here is a plain paragraph that should be unaffected.</p>
+
+* One \cite{jdfreder}.
+* Two \cite{carreau}.
+* Three \cite{ivanov}.
+"""
+
+def test_citation2latex():
+    """Are citations parsed properly?"""
+    try:
+        import lxml
+    except ImportError:
+        assert test_md == citation2latex(test_md) 
+    else:
+        assert test_md_parsed == citation2latex(test_md)

IPython/nbconvert/exporters/exporter.py

@@ -68,6 +68,7 default_filters = {
         'strip_math_space': filters.strip_math_space,
         'wrap_text': filters.wrap_text,
         'escape_latex': filters.escape_latex,
+        'citation2latex': filters.citation2latex
 }
 
 #-----------------------------------------------------------------------------

IPython/nbconvert/filters/__init__.py

@@ -3,4 +3,5 from .datatypefilter import *
 from .highlight import *
 from .latex import *
 from .markdown import *
-from .strings import *
  No newline at end of file
+from .strings import *
+from .citation import *
  No newline at end of file

IPython/nbconvert/postprocessors/pdf.py

@@ -26,35 +26,107 from .base import PostProcessorBase
 class PDFPostProcessor(PostProcessorBase):
     """Writer designed to write to PDF files"""
 
-    iteration_count = Integer(3, config=True, help="""
+    latex_count = Integer(3, config=True, help="""
         How many times pdflatex will be called.
         """)
 
-    command = List(["pdflatex", "{filename}"], config=True, help="""
+    latex_command = List(["pdflatex", "{filename}"], config=True, help="""
         Shell command used to compile PDF.""")
 
+    bib_command = List(["bibtex", "{filename}"], config=True, help="""
+        Shell command used to run bibtex.""")
+
     verbose = Bool(False, config=True, help="""
         Whether or not to display the output of the compile call.
         """)
 
-    def postprocess(self, input):
-            """
-            Consume and write Jinja output a PDF.  
-            See files.py for more...
-            """        
-            command = [c.format(filename=input) for c in self.command]
-            self.log.info("Building PDF: %s", command)
-            with open(os.devnull, 'rb') as null:
-                stdout = subprocess.PIPE if not self.verbose else None
-                for index in range(self.iteration_count):
-                    p = subprocess.Popen(command, stdout=stdout, stdin=null)
-                    out, err = p.communicate()
-                    if p.returncode:
-                        if self.verbose:
-                            # verbose means I didn't capture stdout with PIPE,
-                            # so it's already been displayed and `out` is None.
-                            out = u''
-                        else:
-                            out = out.decode('utf-8', 'replace')
-                        self.log.critical(u"PDF conversion failed: %s\n%s", command, out)
-                        return
+    temp_file_exts = List(['.aux', '.bbl', '.blg', '.idx', '.log', '.out'], 
+        config=True, help="""
+        Filename extensions of temp files to remove after running.
+        """)
+
+    def run_command(self, command_list, filename, count, log_function):
+        """Run command_list count times.
+        
+        Parameters
+        ----------
+        command_list : list
+            A list of args to provide to Popen. Each element of this
+            list will be interpolated with the filename to convert.
+        filename : unicode
+            The name of the file to convert.
+        count : int
+            How many times to run the command.
+        
+        Returns
+        -------
+        continue : bool
+            A boolean indicating if the command was successful (True)
+            or failed (False).
+        """
+        command = [c.format(filename=filename) for c in command_list]
+        times = 'time' if count == 1 else 'times'
+        self.log.info("Running %s %i %s: %s", command_list[0], count, times, command)
+        with open(os.devnull, 'rb') as null:
+            stdout = subprocess.PIPE if not self.verbose else None
+            for index in range(count):
+                p = subprocess.Popen(command, stdout=stdout, stdin=null)
+                out, err = p.communicate()
+                if p.returncode:
+                    if self.verbose:
+                        # verbose means I didn't capture stdout with PIPE,
+                        # so it's already been displayed and `out` is None.
+                        out = u''
+                    else:
+                        out = out.decode('utf-8', 'replace')
+                    log_function(command, out)
+                    return False # failure
+        return True # success
+
+    def run_latex(self, filename):
+        """Run pdflatex self.latex_count times."""
+
+        def log_error(command, out):
+            self.log.critical(u"%s failed: %s\n%s", command[0], command, out)
+
+        return self.run_command(self.latex_command, filename, 
+            self.latex_count, log_error)
+
+    def run_bib(self, filename):
+        """Run bibtex self.latex_count times."""
+        filename = os.path.splitext(filename)[0]
+
+        def log_error(command, out):
+            self.log.warn('%s had problems, most likely because there were no citations',
+                command[0])
+            self.log.debug(u"%s output: %s\n%s", command[0], command, out)
+
+        return self.run_command(self.bib_command, filename, 1, log_error)
+
+    def clean_temp_files(self, filename):
+        """Remove temporary files created by pdflatex/bibtext."""
+        self.log.info("Removing temporary LaTeX files")
+        filename = os.path.splitext(filename)[0]
+        for ext in self.temp_file_exts:
+            try:
+                os.remove(filename+ext)
+            except OSError:
+                pass
+
+    def postprocess(self, filename):
+        """Build a PDF by running pdflatex and bibtex"""
+        self.log.info("Building PDF")
+        cont = self.run_latex(filename)
+        if cont:
+            cont = self.run_bib(filename)
+        else:
+            self.clean_temp_files(filename)
+            return
+        if cont:
+            cont = self.run_latex(filename)
+        self.clean_temp_files(filename)
+        filename = os.path.splitext(filename)[0]
+        if os.path.isfile(filename+'.pdf'):
+            self.log.info('PDF successfully created')
+        return
+

IPython/nbconvert/postprocessors/tests/test_pdf.py

@@ -62,3 +62,7 class TestPDF(TestsBase):
 
             # Check that the PDF was created.
             assert os.path.isfile('a.pdf')
+            
+            # Make sure that temp files are cleaned up
+            for ext in processor.temp_file_exts:
+                assert not os.path.isfile('a'+ext)

IPython/nbconvert/templates/latex/latex_basic.tplx

@@ -103,11 +103,11 it introduces a new line
 ((* endblock stream *))
 
 ((* block markdowncell scoped *))
-((( cell.source | markdown2latex )))
+((( cell.source | citation2latex | markdown2latex )))
 ((* endblock markdowncell *))
 
 ((* block headingcell scoped -*))
-((( ('#' * cell.level + cell.source) | replace('\n', ' ') | markdown2latex )))
+((( ('#' * cell.level + cell.source) | replace('\n', ' ') | citation2latex | markdown2latex )))
 ((* endblock headingcell *))
 
 ((* block rawcell scoped *))
@@ -127,6 +127,10 unknown type ((( cell.type )))
 ((( super() )))
 
 ((* block bodyEnd *))
+
+((* block bibliography *))
+((* endblock bibliography *))
+
 \end{document}
 ((* endblock bodyEnd *))
 ((* endblock body *))

IPython/nbconvert/templates/latex/sphinx.tplx

@@ -192,6 +192,9 Note: For best display, use latex syntax highlighting. =))
         \renewcommand{\indexname}{Index}
         \printindex
 
+        ((* block bibliography *))
+        ((* endblock bibliography *))
+
     % End of document
     \end{document}
 ((* endblock bodyEnd *))
@@ -229,7 +232,7 Note: For best display, use latex syntax highlighting. =))
     in IPYNB file titles) do not make their way into latex.  Sometimes this
     causes latex to barf. =))
     ((*- endif -*))
-    {((( cell.source | markdown2latex )))}
+    {((( cell.source | citation2latex | markdown2latex )))}
 ((*- endblock headingcell *))
 
 %==============================================================================
@@ -239,7 +242,7 Note: For best display, use latex syntax highlighting. =))
 %          called since we know we want latex output.
 %==============================================================================
 ((*- block markdowncell scoped-*))
-((( cell.source | markdown2latex )))
+((( cell.source | citation2latex | markdown2latex )))
 ((*- endblock markdowncell -*))
 
 %==============================================================================

docs/source/interactive/nbconvert.rst

@@ -125,6 +125,21 and using the command::
 
 .. _notebook_format:
 
+LaTeX citations
+---------------
+
+``nbconvert`` now has support for LaTeX citations. With this capability you
+can:
+
+* Manage citations using BibTeX.
+* Cite those citations in Markdown cells using HTML data attributes.
+* Have ``nbconvert`` generate proper LaTeX citations and run BibTeX.
+
+For an example of how this works, please see the citations example in
+the nbconvert-examples_ repository.
+
+.. _nbconvert-examples: https://github.com/ipython/nbconvert-examples
+
 Notebook JSON file format
 -------------------------