Merge pull request #4505 from mrmagooey/nbconvert-latex-markdown-images2

files url handling in nbconvert

closes  #4191

IPython/nbconvert/filters/latex.py

@@ -1,63 +1,63 b''
 """Latex filters.
 
 Module of useful filters for processing Latex within Jinja latex templates.
 """
 #-----------------------------------------------------------------------------
 # Copyright (c) 2013, the IPython Development Team.
 #
 # Distributed under the terms of the Modified BSD License.
 #
 # The full license is in the file COPYING.txt, distributed with this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 import re
 
 #-----------------------------------------------------------------------------
 # Globals and constants
 #-----------------------------------------------------------------------------
 
 LATEX_RE_SUBS = (
     (re.compile(r'\.\.\.+'), r'\\ldots'),
 )
 
 # Latex substitutions for escaping latex.
 # see: http://stackoverflow.com/questions/16259923/how-can-i-escape-latex-special-characters-inside-django-templates
 
 LATEX_SUBS = {
     '&':  r'\&',
     '%':  r'\%',
     '$':  r'\$',
     '#':  r'\#',
     '_':  r'\_',
     '{':  r'\{',
     '}':  r'\}',
     '~':  r'\textasciitilde{}',
     '^':  r'\^{}',
     '\\': r'\textbackslash{}',
 }
 
 
 #-----------------------------------------------------------------------------
 # Functions
 #-----------------------------------------------------------------------------
 
 __all__ = ['escape_latex']
 
 def escape_latex(text):
     """
     Escape characters that may conflict with latex.
 
     Parameters
     ----------
     text : str
         Text containing characters that may conflict with Latex
     """
     text = ''.join(LATEX_SUBS.get(c, c) for c in text)
     for pattern, replacement in LATEX_RE_SUBS:
         text = pattern.sub(replacement, text)
 
     return text
-
+    

IPython/nbconvert/filters/strings.py

@@ -1,204 +1,207 b''
 # coding: utf-8
 """String filters.
 
 Contains a collection of useful string manipulation filters for use in Jinja
 templates.
 """
 #-----------------------------------------------------------------------------
 # Copyright (c) 2013, the IPython Development Team.
 #
 # Distributed under the terms of the Modified BSD License.
 #
 # The full license is in the file COPYING.txt, distributed with this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 import os
 import re
 import textwrap
 try:
     from urllib.parse import quote  # Py 3
 except ImportError:
     from urllib2 import quote  # Py 2
 from xml.etree import ElementTree
 
 from IPython.core.interactiveshell import InteractiveShell
 from IPython.utils import py3compat
 
 #-----------------------------------------------------------------------------
 # Functions
 #-----------------------------------------------------------------------------
 
 __all__ = [
     'wrap_text',
     'html2text',
     'add_anchor',
     'strip_dollars',
     'strip_files_prefix',
     'comment_lines',
     'get_lines',
     'ipython2python',
     'posix_path',
     'path2url',
     'add_prompts'
 ]
 
 
 def wrap_text(text, width=100):
     """ 
     Intelligently wrap text.
     Wrap text without breaking words if possible.
     
     Parameters
     ----------
     text : str
         Text to wrap.
     width : int, optional
         Number of characters to wrap to, default 100.
     """
 
     split_text = text.split('\n')
     wrp = map(lambda x:textwrap.wrap(x,width), split_text)
     wrpd = map('\n'.join, wrp)
     return '\n'.join(wrpd)
 
 
 def html2text(element):
     """extract inner text from html
     
     Analog of jQuery's $(element).text()
     """
     if isinstance(element, py3compat.string_types):
         element = ElementTree.fromstring(element)
     
     text = element.text or ""
     for child in element:
         text += html2text(child)
     text += (element.tail or "")
     return text
 
 
 def add_anchor(html):
     """Add an anchor-link to an html header tag
     
     For use in heading cells
     """
     h = ElementTree.fromstring(py3compat.cast_bytes_py2(html, encoding='utf-8'))
     link = html2text(h).replace(' ', '-')
     h.set('id', link)
     a = ElementTree.Element("a", {"class" : "anchor-link", "href" : "#" + link})
     a.text = u'¶'
     h.append(a)
 
     # Known issue of Python3.x, ElementTree.tostring() returns a byte string
     # instead of a text string.  See issue http://bugs.python.org/issue10942
     # Workaround is to make sure the bytes are casted to a string.
     return py3compat.decode(ElementTree.tostring(h), 'utf-8')
 
 
 def add_prompts(code, first='>>> ', cont='... '):
     """Add prompts to code snippets"""
     new_code = []
     code_list = code.split('\n')
     new_code.append(first + code_list[0])
     for line in code_list[1:]:
         new_code.append(cont + line)
     return '\n'.join(new_code)
 
     
 def strip_dollars(text):
     """
     Remove all dollar symbols from text
     
     Parameters
     ----------
     text : str
         Text to remove dollars from
     """
 
     return text.strip('$')
 
 
-files_url_pattern = re.compile(r'(src|href)\=([\'"]?)files/')
+files_url_pattern = re.compile(r'(src|href)\=([\'"]?)/?files/')
+markdown_url_pattern = re.compile(r'(!?)\[(?P<caption>.*?)\]\(/?files/(?P<location>.*?)\)')
 
 def strip_files_prefix(text):
     """
-    Fix all fake URLs that start with `files/`,
-    stripping out the `files/` prefix.
+    Fix all fake URLs that start with `files/`, stripping out the `files/` prefix.
+    Applies to both urls (for html) and relative paths (for markdown paths).
     
     Parameters
     ----------
     text : str
         Text in which to replace 'src="files/real...' with 'src="real...'
     """
-    return files_url_pattern.sub(r"\1=\2", text)
+    cleaned_text = files_url_pattern.sub(r"\1=\2", text)
+    cleaned_text = markdown_url_pattern.sub(r'\1[\2](\3)', cleaned_text)
+    return cleaned_text
 
 
 def comment_lines(text, prefix='# '):
     """
     Build a Python comment line from input text.
     
     Parameters
     ----------
     text : str
         Text to comment out.
     prefix : str
         Character to append to the start of each line.
     """
     
     #Replace line breaks with line breaks and comment symbols.
     #Also add a comment symbol at the beginning to comment out
     #the first line.
     return prefix + ('\n'+prefix).join(text.split('\n')) 
 
 
 def get_lines(text, start=None,end=None):
     """
     Split the input text into separate lines and then return the 
     lines that the caller is interested in.
     
     Parameters
     ----------
     text : str
         Text to parse lines from.
     start : int, optional
         First line to grab from.
     end : int, optional
         Last line to grab from.
     """
     
     # Split the input into lines.
     lines = text.split("\n")
     
     # Return the right lines.
     return "\n".join(lines[start:end]) #re-join
 
 def ipython2python(code):
     """Transform IPython syntax to pure Python syntax
 
     Parameters
     ----------
 
     code : str
         IPython code, to be transformed to pure Python
     """
     shell = InteractiveShell.instance()
     return shell.input_transformer_manager.transform_cell(code)
 
 def posix_path(path):
     """Turn a path into posix-style path/to/etc
     
     Mainly for use in latex on Windows,
     where native Windows paths are not allowed.
     """
     if os.path.sep != '/':
         return path.replace(os.path.sep, '/')
     return path
 
 def path2url(path):
     """Turn a file path into a URL"""
     parts = path.split(os.path.sep)
     return '/'.join(quote(part) for part in parts)

IPython/nbconvert/filters/tests/test_latex.py

@@ -1,45 +1,43 b''
 """
 Module with tests for Latex
 """
 
 #-----------------------------------------------------------------------------
 # Copyright (c) 2013, the IPython Development Team.
 #
 # Distributed under the terms of the Modified BSD License.
 #
 # The full license is in the file COPYING.txt, distributed with this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 from ...tests.base import TestsBase
 from ..latex import escape_latex
 
 
 #-----------------------------------------------------------------------------
 # Class
 #-----------------------------------------------------------------------------
 
 class TestLatex(TestsBase):
 
 
     def test_escape_latex(self):
         """escape_latex test"""
         tests = [
             (r'How are \you doing today?', r'How are \textbackslash{}you doing today?'),
             (r'\escapechar=`\A\catcode`\|=0 |string|foo', r'\textbackslash{}escapechar=`\textbackslash{}A\textbackslash{}catcode`\textbackslash{}|=0 |string|foo'),
             (r'# $ % & ~ _ ^ \ { }', r'\# \$ \% \& \textasciitilde{} \_ \^{} \textbackslash{} \{ \}'),
             ('...', r'\ldots'),
             ('','')]
 
         for test in tests:
             self._try_escape_latex(test[0], test[1])
 
 
     def _try_escape_latex(self, test, result):
         """Try to remove latex from string"""
         self.assertEqual(escape_latex(test), result)
-
-

IPython/nbconvert/filters/tests/test_strings.py

@@ -1,135 +1,148 b''
 """
 Module with tests for Strings
 """
 
 #-----------------------------------------------------------------------------
 # Copyright (c) 2013, the IPython Development Team.
 #
 # Distributed under the terms of the Modified BSD License.
 #
 # The full license is in the file COPYING.txt, distributed with this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 import os
 
 from ...tests.base import TestsBase
 from ..strings import (wrap_text, html2text, add_anchor, strip_dollars, 
     strip_files_prefix, get_lines, comment_lines, ipython2python, posix_path,
     add_prompts
 )
 
 
 #-----------------------------------------------------------------------------
 # Class
 #-----------------------------------------------------------------------------
 
 class TestStrings(TestsBase):
 
     def test_wrap_text(self):
         """wrap_text test"""
         test_text = """
         Tush! never tell me; I take it much unkindly
         That thou, Iago, who hast had my purse
         As if the strings were thine, shouldst know of this.
         """
         for length in [30,5,1]:
             self._confirm_wrap_text(test_text, length)
     
 
     def _confirm_wrap_text(self, text, length):
         for line in wrap_text(text, length).split('\n'):
             assert len(line) <= length
 
 
     def test_html2text(self):
         """html2text test"""
         #TODO: More tests
         self.assertEqual(html2text('<name>joe</name>'), 'joe')
         
 
     def test_add_anchor(self):
         """add_anchor test"""
         #TODO: More tests
         results = add_anchor('<b>Hello World!</b>')
         assert 'Hello World!' in results
         assert 'id="' in results
         assert 'class="anchor-link"' in results
         assert '<b' in results
         assert '</b>' in results
 
         
     def test_strip_dollars(self):
         """strip_dollars test"""
         tests = [
             ('', ''), 
             ('$$', ''), 
             ('$H$', 'H'), 
             ('$He', 'He'), 
             ('H$el', 'H$el'), 
             ('Hell$', 'Hell'),
             ('Hello', 'Hello'),
             ('W$o$rld', 'W$o$rld')]
         for test in tests:
             self._try_strip_dollars(test[0], test[1])
 
 
     def _try_strip_dollars(self, test, result):
         self.assertEqual(strip_dollars(test), result)
 
 
     def test_strip_files_prefix(self):
         """strip_files_prefix test"""
         tests = [
             ('', ''), 
             ('/files', '/files'), 
             ('test="/files"', 'test="/files"'), 
             ('My files are in `files/`', 'My files are in `files/`'),
-            ('<a href="files/test.html">files/test.html</a>', '<a href="test.html">files/test.html</a>')]
+            ('<a href="files/test.html">files/test.html</a>', '<a href="test.html">files/test.html</a>'),
+            ('<a href="/files/test.html">files/test.html</a>', '<a href="test.html">files/test.html</a>'),
+            ("<a href='files/test.html'>files/test.html</a>", "<a href='test.html'>files/test.html</a>"),
+            ('<img src="files/url/location.gif">', '<img src="url/location.gif">'),
+            ('<img src="/files/url/location.gif">', '<img src="url/location.gif">'),
+            ('hello![caption]', 'hello![caption]'),
+            ('hello![caption](/url/location.gif)', 'hello![caption](/url/location.gif)'),
+            ('hello![caption](url/location.gif)', 'hello![caption](url/location.gif)'),
+            ('hello![caption](url/location.gif)', 'hello![caption](url/location.gif)'),
+            ('hello![caption](files/url/location.gif)', 'hello![caption](url/location.gif)'),
+            ('hello![caption](/files/url/location.gif)', 'hello![caption](url/location.gif)'),
+            ('hello [text](/files/url/location.gif)', 'hello [text](url/location.gif)'),
+            ('hello [text space](files/url/location.gif)', 'hello [text space](url/location.gif)'),
+        ]
         for test in tests:
             self._try_files_prefix(test[0], test[1])
 
 
     def _try_files_prefix(self, test, result):
         self.assertEqual(strip_files_prefix(test), result)
         
 
     def test_comment_lines(self):
         """comment_lines test"""
         for line in comment_lines('hello\nworld\n!').split('\n'):
             assert line.startswith('# ')
         for line in comment_lines('hello\nworld\n!', 'beep').split('\n'):
             assert line.startswith('beep')
         
 
     def test_get_lines(self):
         """get_lines test"""
         text = "hello\nworld\n!"
         self.assertEqual(get_lines(text, start=1), "world\n!")
         self.assertEqual(get_lines(text, end=2), "hello\nworld")
         self.assertEqual(get_lines(text, start=2, end=5), "!")
         self.assertEqual(get_lines(text, start=-2), "world\n!")
         
 
     def test_ipython2python(self):
         """ipython2python test"""
         #TODO: More tests
         results = ipython2python(u'%%pylab\nprint("Hello-World")').replace("u'", "'")
         self.fuzzy_compare(results, u"get_ipython().run_cell_magic('pylab', '', 'print(\"Hello-World\")')", 
             ignore_spaces=True, ignore_newlines=True)
     
     def test_posix_path(self):
         """posix_path test"""
         path_list = ['foo', 'bar']
         expected = '/'.join(path_list)
         native = os.path.join(*path_list)
         filtered = posix_path(native)
         self.assertEqual(filtered, expected)
     
     def test_add_prompts(self):
         """add_prompts test"""
         text1 = """for i in range(10):\n  i += 1\n  print i"""
         text2 = """>>> for i in range(10):\n...   i += 1\n...   print i"""
         self.assertEqual(text2, add_prompts(text1))

IPython/nbconvert/templates/latex/latex_base.tplx

@@ -1,223 +1,223 b''
 ((= Latex base template (must inherit)
 This template builds upon the abstract template, adding common latex output
 functions.  Figures, data_text,
 This template does not define a docclass, the inheriting class must define this.=))
 
 ((*- extends 'display_priority.tplx' -*))
 
 %===============================================================================
 % Abstract overrides
 %===============================================================================
 
 ((* block header *))
     ((* block docclass *))((* endblock docclass *))
     
     ((* block packages *))
     \usepackage{graphicx} % Used to insert images
     \usepackage{adjustbox} % Used to constrain images to a maximum size 
     \usepackage{color} % Allow colors to be defined
     \usepackage{enumerate} % Needed for markdown enumerations to work
     \usepackage{geometry} % Used to adjust the document margins
     \usepackage{amsmath} % Equations
     \usepackage{amssymb} % Equations
     \usepackage[utf8]{inputenc} % Allow utf-8 characters in the tex document
     \usepackage[mathletters]{ucs} % Extended unicode (utf-8) support
     \usepackage{fancyvrb} % verbatim replacement that allows latex
     \usepackage{grffile} % extends the file name processing of package graphics 
                          % to support a larger range 
     % The hyperref package gives us a pdf with properly built
     % internal navigation ('pdf bookmarks' for the table of contents,
     % internal cross-reference links, web links for URLs, etc.)
     \usepackage{hyperref}
     \usepackage{longtable} % longtable support required by pandoc >1.10
     ((* endblock packages *))
 
     ((* block definitions *))
     \definecolor{orange}{cmyk}{0,0.4,0.8,0.2}
     \definecolor{darkorange}{rgb}{.71,0.21,0.01}
     \definecolor{darkgreen}{rgb}{.12,.54,.11}
     \definecolor{myteal}{rgb}{.26, .44, .56}
     \definecolor{gray}{gray}{0.45}
     \definecolor{lightgray}{gray}{.95}
     \definecolor{mediumgray}{gray}{.8}
     \definecolor{inputbackground}{rgb}{.95, .95, .85}
     \definecolor{outputbackground}{rgb}{.95, .95, .95}
     \definecolor{traceback}{rgb}{1, .95, .95}
     % ansi colors
     \definecolor{red}{rgb}{.6,0,0}
     \definecolor{green}{rgb}{0,.65,0}
     \definecolor{brown}{rgb}{0.6,0.6,0}
     \definecolor{blue}{rgb}{0,.145,.698}
     \definecolor{purple}{rgb}{.698,.145,.698}
     \definecolor{cyan}{rgb}{0,.698,.698}
     \definecolor{lightgray}{gray}{0.5}
     
     % bright ansi colors
     \definecolor{darkgray}{gray}{0.25}
     \definecolor{lightred}{rgb}{1.0,0.39,0.28}
     \definecolor{lightgreen}{rgb}{0.48,0.99,0.0}
     \definecolor{lightblue}{rgb}{0.53,0.81,0.92}
     \definecolor{lightpurple}{rgb}{0.87,0.63,0.87}
     \definecolor{lightcyan}{rgb}{0.5,1.0,0.83}
     
     % commands and environments needed by pandoc snippets
     % extracted from the output of `pandoc -s`
     
     \DefineShortVerb[commandchars=\\\{\}]{\|}
     \DefineVerbatimEnvironment{Highlighting}{Verbatim}{commandchars=\\\{\}}
     % Add ',fontsize=\small' for more characters per line
     \newenvironment{Shaded}{}{}
     \newcommand{\KeywordTok}[1]{\textcolor[rgb]{0.00,0.44,0.13}{\textbf{{#1}}}}
     \newcommand{\DataTypeTok}[1]{\textcolor[rgb]{0.56,0.13,0.00}{{#1}}}
     \newcommand{\DecValTok}[1]{\textcolor[rgb]{0.25,0.63,0.44}{{#1}}}
     \newcommand{\BaseNTok}[1]{\textcolor[rgb]{0.25,0.63,0.44}{{#1}}}
     \newcommand{\FloatTok}[1]{\textcolor[rgb]{0.25,0.63,0.44}{{#1}}}
     \newcommand{\CharTok}[1]{\textcolor[rgb]{0.25,0.44,0.63}{{#1}}}
     \newcommand{\StringTok}[1]{\textcolor[rgb]{0.25,0.44,0.63}{{#1}}}
     \newcommand{\CommentTok}[1]{\textcolor[rgb]{0.38,0.63,0.69}{\textit{{#1}}}}
     \newcommand{\OtherTok}[1]{\textcolor[rgb]{0.00,0.44,0.13}{{#1}}}
     \newcommand{\AlertTok}[1]{\textcolor[rgb]{1.00,0.00,0.00}{\textbf{{#1}}}}
     \newcommand{\FunctionTok}[1]{\textcolor[rgb]{0.02,0.16,0.49}{{#1}}}
     \newcommand{\RegionMarkerTok}[1]{{#1}}
     \newcommand{\ErrorTok}[1]{\textcolor[rgb]{1.00,0.00,0.00}{\textbf{{#1}}}}
     \newcommand{\NormalTok}[1]{{#1}}
     
     % Define a nice break command that doesn't care if a line doesn't already
     % exist.
     \def\br{\hspace*{\fill} \\* }
     % Math Jax compatability definitions
     \def\gt{>}
     \def\lt{<}
     % Document parameters
     ((* block title *))\title{((( resources.metadata.name | escape_latex )))}((* endblock title *))
     ((* block date *))((* endblock date *))
     ((* block author *))((* endblock author *))
     ((* endblock definitions *))
 
     ((* block commands *))
     % Prevent overflowing lines due to hard-to-break entities
     \sloppy 
     % Setup hyperref package
     \hypersetup{
       breaklinks=true,  % so long urls are correctly broken across lines
       colorlinks=true,
       urlcolor=blue,
       linkcolor=darkorange,
       citecolor=darkgreen,
       }
     % Slightly bigger margins than the latex defaults
     ((* block margins *))
     \geometry{verbose,tmargin=1in,bmargin=1in,lmargin=1in,rmargin=1in}
     ((* endblock margins *))
     ((* endblock commands *))
 ((* endblock header *))
 
 ((* block body *))
     \begin{document}
     
     ((* block predoc *))
     ((* block maketitle *))\maketitle((* endblock maketitle *))
     ((* block abstract *))((* endblock abstract *))
     ((* endblock predoc *))
 
     ((( super() )))
 
     % Add a bibliography block to the postdoc
     ((* block postdoc *))
     ((* block bibliography *))((* endblock bibliography *))
     ((* endblock postdoc *))
     \end{document}
 ((* endblock body *))
 
 %===============================================================================
 % Support blocks
 %===============================================================================
 
 % Displaying simple data text
 ((* block data_text *))
     \begin{verbatim}
 ((( output.text )))
     \end{verbatim}
 ((* endblock data_text *))
 
 % Display python error text as-is
 ((* block pyerr *))
     \begin{Verbatim}[commandchars=\\\{\}]
 ((( super() )))
     \end{Verbatim}
 ((* endblock pyerr *))
 ((* block traceback_line *))
     ((( line | indent | strip_ansi | escape_latex )))
 ((* endblock traceback_line *))
 
 % Display stream ouput with coloring
 ((* block stream *))
     \begin{Verbatim}[commandchars=\\\{\}]
 ((( output.text | escape_latex | ansi2latex )))
     \end{Verbatim}
 ((* endblock stream *))
 
 % Display latex
 ((* block data_latex -*))
     ((*- if output.latex.startswith('$'): -*)) 
         ((= Replace $ symbols with more explicit, equation block. =))
         \begin{equation*}
         ((( output.latex | strip_dollars )))
         \end{equation*}
     ((*- else -*))
         ((( output.latex )))
     ((*- endif *))
 ((* endblock data_latex *))
 
 % Default mechanism for rendering figures
 ((*- block data_png -*))((( draw_figure(output.png_filename) )))((*- endblock -*))
 ((*- block data_jpg -*))((( draw_figure(output.jpeg_filename) )))((*- endblock -*))
 ((*- block data_svg -*))((( draw_figure(output.svg_filename) )))((*- endblock -*))
 ((*- block data_pdf -*))((( draw_figure(output.pdf_filename) )))((*- endblock -*))
 
 % Draw a figure using the graphicx package.
 ((* macro draw_figure(filename) -*))
 ((* set filename = filename | posix_path *))
 ((*- block figure scoped -*))
     \begin{center}
     \adjustimage{max size={0.9\linewidth}{0.9\paperheight}}{((( filename )))}
     \end{center}
     { \hspace*{\fill} \\}
 ((*- endblock figure -*))
 ((*- endmacro *))
 
 % Draw heading cell.  Explicitly map different cell levels.
 ((* block headingcell scoped *))
 
     ((* if cell.level == 1 -*))
         ((* block h1 -*))\section((* endblock h1 -*))
     ((* elif cell.level == 2 -*))
         ((* block h2 -*))\subsection((* endblock h2 -*))
     ((* elif cell.level == 3 -*))
         ((* block h3 -*))\subsubsection((* endblock h3 -*))
     ((* elif cell.level == 4 -*))
         ((* block h4 -*))\paragraph((* endblock h4 -*))
     ((* elif cell.level == 5 -*))
         ((* block h5 -*))\subparagraph((* endblock h5 -*))
     ((* elif cell.level == 6 -*))
         ((* block h6 -*))\\*\textit((* endblock h6 -*))
     ((*- endif -*))
-    {((( cell.source | replace('\n', ' ') | citation2latex | markdown2latex )))}
+    {((( cell.source | replace('\n', ' ') | citation2latex | strip_files_prefix | markdown2latex )))}
 
 ((* endblock headingcell *))
 
 % Redirect pyout to display data priority.
 ((* block pyout scoped *))
     ((* block data_priority scoped *))
     ((( super() )))
     ((* endblock *))
 ((* endblock pyout *))
 
 % Render markdown
 ((* block markdowncell scoped *))
-    ((( cell.source | citation2latex | markdown2latex )))
+    ((( cell.source | citation2latex | strip_files_prefix | markdown2latex )))
 ((* endblock markdowncell *))
 
 % Don't display unknown types
 ((* block unknowncell scoped *))
 ((* endblock unknowncell *))