Merge pull request #6588 from benjaminabel/master

Handle numbers in heading cells when converting to Latex

.travis.yml

@@ -1,29 +1,31 b''
 # http://travis-ci.org/#!/ipython/ipython
 language: python
 python:
     - 3.4
     - 2.7
     - 3.3
 env:
     - GROUP=js
     - GROUP=
 before_install:
     # workaround for https://github.com/travis-ci/travis-cookbooks/issues/155
     - sudo rm -rf /dev/shm && sudo ln -s /run/shm /dev/shm
     # Pierre Carrier's PPA for PhantomJS and CasperJS
     - time sudo add-apt-repository -y ppa:pcarrier/ppa
+    # Needed to get recent version of pandoc in ubntu 12.04
+    - time sudo add-apt-repository -y ppa:marutter/c2d4u
     - time sudo apt-get update
     - time sudo apt-get install pandoc casperjs libzmq3-dev
     # pin tornado < 4 for js tests while phantom is on super old webkit
     - if [[ $GROUP == 'js' ]]; then pip install 'tornado<4'; fi
     - time pip install -f https://nipy.bic.berkeley.edu/wheelhouse/travis jinja2 sphinx pygments tornado requests!=2.4.2 mock pyzmq jsonschema jsonpointer mistune
     - if [[ $GROUP == 'js' ]]; then python -m IPython.external.mathjax; fi
 install:
     - time python setup.py install -q
 script:
     - cd /tmp && iptest $GROUP
 
 matrix:
     exclude:
         - python: 3.3
           env: GROUP=js

IPython/nbconvert/exporters/templateexporter.py

@@ -1,320 +1,321 b''
 """This module defines TemplateExporter, a highly configurable converter
 that uses Jinja2 to export notebook files into different formats.
 """
 
 #-----------------------------------------------------------------------------
 # Copyright (c) 2013, the IPython Development Team.
 #
 # Distributed under the terms of the Modified BSD License.
 #
 # The full license is in the file COPYING.txt, distributed with this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 from __future__ import print_function, absolute_import
 
 # Stdlib imports
 import os
 
 # other libs/dependencies are imported at runtime
 # to move ImportErrors to runtime when the requirement is actually needed
 
 # IPython imports
 from IPython.utils.traitlets import MetaHasTraits, Unicode, List, Dict, Any
 from IPython.utils.importstring import import_item
 from IPython.utils import py3compat, text
 
 from IPython.nbconvert import filters
 from .exporter import Exporter
 
 #-----------------------------------------------------------------------------
 # Globals and constants
 #-----------------------------------------------------------------------------
 
 #Jinja2 extensions to load.
 JINJA_EXTENSIONS = ['jinja2.ext.loopcontrols']
 
 default_filters = {
         'indent': text.indent,
         'markdown2html': filters.markdown2html,
         'ansi2html': filters.ansi2html,
         'filter_data_type': filters.DataTypeFilter,
         'get_lines': filters.get_lines,
         'highlight2html': filters.Highlight2HTML,
         'highlight2latex': filters.Highlight2Latex,
         'ipython2python': filters.ipython2python,
         'posix_path': filters.posix_path,
         'markdown2latex': filters.markdown2latex,
         'markdown2rst': filters.markdown2rst,
         'comment_lines': filters.comment_lines,
         'strip_ansi': filters.strip_ansi,
         'strip_dollars': filters.strip_dollars,
         'strip_files_prefix': filters.strip_files_prefix,
         'html2text' : filters.html2text,
         'add_anchor': filters.add_anchor,
         'ansi2latex': filters.ansi2latex,
         'wrap_text': filters.wrap_text,
         'escape_latex': filters.escape_latex,
         'citation2latex': filters.citation2latex,
         'path2url': filters.path2url,
         'add_prompts': filters.add_prompts,
         'ascii_only': filters.ascii_only,
+        'prevent_list_blocks': filters.prevent_list_blocks,
 }
 
 #-----------------------------------------------------------------------------
 # Class
 #-----------------------------------------------------------------------------
 
 class TemplateExporter(Exporter):
     """
     Exports notebooks into other file formats.  Uses Jinja 2 templating engine
     to output new formats.  Inherit from this class if you are creating a new
     template type along with new filters/preprocessors.  If the filters/
     preprocessors provided by default suffice, there is no need to inherit from
     this class.  Instead, override the template_file and file_extension
     traits via a config file.
 
     {filters}
     """
     
     # finish the docstring
     __doc__ = __doc__.format(filters = '- '+'\n    - '.join(default_filters.keys()))
 
 
     template_file = Unicode(u'default',
             config=True,
             help="Name of the template file to use")
     def _template_file_changed(self, name, old, new):
         if new == 'default':
             self.template_file = self.default_template
         else:
             self.template_file = new
         self.template = None
         self._load_template()
     
     default_template = Unicode(u'')
     template = Any()
     environment = Any()
 
     template_path = List(['.'], config=True)
     def _template_path_changed(self, name, old, new):
         self._load_template()
 
     default_template_path = Unicode(
         os.path.join("..", "templates"), 
         help="Path where the template files are located.")
 
     template_skeleton_path = Unicode(
         os.path.join("..", "templates", "skeleton"), 
         help="Path where the template skeleton files are located.") 
 
     #Jinja block definitions
     jinja_comment_block_start = Unicode("", config=True)
     jinja_comment_block_end = Unicode("", config=True)
     jinja_variable_block_start = Unicode("", config=True)
     jinja_variable_block_end = Unicode("", config=True)
     jinja_logic_block_start = Unicode("", config=True)
     jinja_logic_block_end = Unicode("", config=True)
     
     #Extension that the template files use.    
     template_extension = Unicode(".tpl", config=True)
 
     filters = Dict(config=True,
         help="""Dictionary of filters, by name and namespace, to add to the Jinja
         environment.""")
 
     raw_mimetypes = List(config=True,
         help="""formats of raw cells to be included in this Exporter's output."""
     )
     def _raw_mimetypes_default(self):
         return [self.output_mimetype, '']
 
 
     def __init__(self, config=None, extra_loaders=None, **kw):
         """
         Public constructor
     
         Parameters
         ----------
         config : config
             User configuration instance.
         extra_loaders : list[of Jinja Loaders]
             ordered list of Jinja loader to find templates. Will be tried in order
             before the default FileSystem ones.
         template : str (optional, kw arg)
             Template to use when exporting.
         """
         super(TemplateExporter, self).__init__(config=config, **kw)
 
         #Init
         self._init_template()
         self._init_environment(extra_loaders=extra_loaders)
         self._init_filters()
 
 
     def _load_template(self):
         """Load the Jinja template object from the template file
         
         This is a no-op if the template attribute is already defined,
         or the Jinja environment is not setup yet.
         
         This is triggered by various trait changes that would change the template.
         """
         from jinja2 import TemplateNotFound
         
         if self.template is not None:
             return
         # called too early, do nothing
         if self.environment is None:
             return
         # Try different template names during conversion.  First try to load the
         # template by name with extension added, then try loading the template
         # as if the name is explicitly specified, then try the name as a 
         # 'flavor', and lastly just try to load the template by module name.
         try_names = []
         if self.template_file:
             try_names.extend([
                 self.template_file + self.template_extension,
                 self.template_file,
             ])
         for try_name in try_names:
             self.log.debug("Attempting to load template %s", try_name)
             try:
                 self.template = self.environment.get_template(try_name)
             except (TemplateNotFound, IOError):
                 pass
             except Exception as e:
                 self.log.warn("Unexpected exception loading template: %s", try_name, exc_info=True)
             else:
                 self.log.info("Loaded template %s", try_name)
                 break
 
     def from_notebook_node(self, nb, resources=None, **kw):
         """
         Convert a notebook from a notebook node instance.
     
         Parameters
         ----------
         nb : :class:`~IPython.nbformat.current.NotebookNode`
           Notebook node
         resources : dict
           Additional resources that can be accessed read/write by
           preprocessors and filters.
         """
         nb_copy, resources = super(TemplateExporter, self).from_notebook_node(nb, resources, **kw)
         resources.setdefault('raw_mimetypes', self.raw_mimetypes)
 
         self._load_template()
 
         if self.template is not None:
             output = self.template.render(nb=nb_copy, resources=resources)
         else:
             raise IOError('template file "%s" could not be found' % self.template_file)
         return output, resources
 
 
     def register_filter(self, name, jinja_filter):
         """
         Register a filter.
         A filter is a function that accepts and acts on one string.  
         The filters are accesible within the Jinja templating engine.
     
         Parameters
         ----------
         name : str
             name to give the filter in the Jinja engine
         filter : filter
         """
         if jinja_filter is None:
             raise TypeError('filter')
         isclass = isinstance(jinja_filter, type)
         constructed = not isclass
 
         #Handle filter's registration based on it's type
         if constructed and isinstance(jinja_filter, py3compat.string_types):
             #filter is a string, import the namespace and recursively call
             #this register_filter method
             filter_cls = import_item(jinja_filter)
             return self.register_filter(name, filter_cls)
         
         if constructed and hasattr(jinja_filter, '__call__'):
             #filter is a function, no need to construct it.
             self.environment.filters[name] = jinja_filter
             return jinja_filter
 
         elif isclass and isinstance(jinja_filter, MetaHasTraits):
             #filter is configurable.  Make sure to pass in new default for 
             #the enabled flag if one was specified.
             filter_instance = jinja_filter(parent=self)
             self.register_filter(name, filter_instance )
 
         elif isclass:
             #filter is not configurable, construct it
             filter_instance = jinja_filter()
             self.register_filter(name, filter_instance)
 
         else:
             #filter is an instance of something without a __call__ 
             #attribute.  
             raise TypeError('filter')
 
         
     def _init_template(self):
         """
         Make sure a template name is specified.  If one isn't specified, try to
         build one from the information we know.
         """
         self._template_file_changed('template_file', self.template_file, self.template_file)
         
 
     def _init_environment(self, extra_loaders=None):
         """
         Create the Jinja templating environment.
         """
         from jinja2 import Environment, ChoiceLoader, FileSystemLoader
         here = os.path.dirname(os.path.realpath(__file__))
         loaders = []
         if extra_loaders:
             loaders.extend(extra_loaders)
 
         paths = self.template_path
         paths.extend([os.path.join(here, self.default_template_path),
                       os.path.join(here, self.template_skeleton_path)])
         loaders.append(FileSystemLoader(paths))
 
         self.environment = Environment(
             loader= ChoiceLoader(loaders),
             extensions=JINJA_EXTENSIONS
             )
         
         #Set special Jinja2 syntax that will not conflict with latex.
         if self.jinja_logic_block_start:
             self.environment.block_start_string = self.jinja_logic_block_start
         if self.jinja_logic_block_end:
             self.environment.block_end_string = self.jinja_logic_block_end
         if self.jinja_variable_block_start:
             self.environment.variable_start_string = self.jinja_variable_block_start
         if self.jinja_variable_block_end:
             self.environment.variable_end_string = self.jinja_variable_block_end
         if self.jinja_comment_block_start:
             self.environment.comment_start_string = self.jinja_comment_block_start
         if self.jinja_comment_block_end:
             self.environment.comment_end_string = self.jinja_comment_block_end
 
     
     def _init_filters(self):
         """
         Register all of the filters required for the exporter.
         """
         
         #Add default filters to the Jinja2 environment
         for key, value in default_filters.items():
             self.register_filter(key, value)
 
         #Load user filters.  Overwrite existing filters if need be.
         if self.filters:
             for key, user_filter in self.filters.items():
                 self.register_filter(key, user_filter)

IPython/nbconvert/filters/markdown.py

@@ -1,227 +1,231 b''
 """Markdown filters
 
 This file contains a collection of utility filters for dealing with 
 markdown within Jinja templates.
 """
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
 from __future__ import print_function
 
 # Stdlib imports
 import os
 import subprocess
 from io import TextIOWrapper, BytesIO
 import re
 
 import mistune
 from pygments import highlight
 from pygments.lexers import get_lexer_by_name
 from pygments.formatters import HtmlFormatter
 from pygments.util import ClassNotFound
 
 # IPython imports
 from IPython.nbconvert.utils.pandoc import pandoc
 from IPython.nbconvert.utils.exceptions import ConversionException
 from IPython.utils.decorators import undoc
 from IPython.utils.process import get_output_error_code
 from IPython.utils.py3compat import cast_bytes
 from IPython.utils.version import check_version
 
 
 marked = os.path.join(os.path.dirname(__file__), "marked.js")
 _node = None
 
 __all__ = [
     'markdown2html',
     'markdown2html_pandoc',
     'markdown2html_marked',
     'markdown2html_mistune',
     'markdown2latex',
     'markdown2rst',
 ]
 
 class NodeJSMissing(ConversionException):
     """Exception raised when node.js is missing."""
     pass
 
-def markdown2latex(source, extra_args=None):
+def markdown2latex(source, markup='markdown', extra_args=None):
     """Convert a markdown string to LaTeX via pandoc.
 
     This function will raise an error if pandoc is not installed.
     Any error messages generated by pandoc are printed to stderr.
 
     Parameters
     ----------
     source : string
       Input string, assumed to be valid markdown.
+    markup : string
+      Markup used by pandoc's reader
+      default : pandoc extended markdown
+      (see http://johnmacfarlane.net/pandoc/README.html#pandocs-markdown)
 
     Returns
     -------
     out : string
       Output as returned by pandoc.
     """
-    return pandoc(source, 'markdown', 'latex', extra_args=extra_args)
+    return pandoc(source, markup, 'latex', extra_args=extra_args)
 
 
 @undoc
 class MathBlockGrammar(mistune.BlockGrammar):
     block_math = re.compile("^\$\$(.*?)\$\$", re.DOTALL)
     latex_environment = re.compile(r"^\\begin\{([a-z]*\*?)\}(.*?)\\end\{\1\}",
                                                 re.DOTALL)
 
 @undoc
 class MathBlockLexer(mistune.BlockLexer):
     default_features = ['block_math', 'latex_environment'] + mistune.BlockLexer.default_features
 
     def __init__(self, rules=None, **kwargs):
         if rules is None:
             rules = MathBlockGrammar()
         super(MathBlockLexer, self).__init__(rules, **kwargs)
 
     def parse_block_math(self, m):
         """Parse a $$math$$ block"""
         self.tokens.append({
             'type': 'block_math',
             'text': m.group(1)
         })
 
     def parse_latex_environment(self, m):
         self.tokens.append({
             'type': 'latex_environment',
             'name': m.group(1),
             'text': m.group(2)
         })
 
 @undoc
 class MathInlineGrammar(mistune.InlineGrammar):
     math = re.compile("^\$(.+?)\$")
     text = re.compile(r'^[\s\S]+?(?=[\\<!\[_*`~$]|https?://| {2,}\n|$)')
 
 @undoc
 class MathInlineLexer(mistune.InlineLexer):
     default_features = ['math'] + mistune.InlineLexer.default_features
 
     def __init__(self, renderer, rules=None, **kwargs):
         if rules is None:
             rules = MathInlineGrammar()
         super(MathInlineLexer, self).__init__(renderer, rules, **kwargs)
 
     def output_math(self, m):
         return self.renderer.inline_math(m.group(1))
 
 @undoc
 class MarkdownWithMath(mistune.Markdown):
     def __init__(self, renderer, **kwargs):
         if 'inline' not in kwargs:
             kwargs['inline'] = MathInlineLexer
         if 'block' not in kwargs:
             kwargs['block'] = MathBlockLexer
         super(MarkdownWithMath, self).__init__(renderer, **kwargs)
 
     def parse_block_math(self):
         return self.renderer.block_math(self.token['text'])
 
     def parse_latex_environment(self):
         return self.renderer.latex_environment(self.token['name'], self.token['text'])
 
 @undoc
 class IPythonRenderer(mistune.Renderer):
     def block_code(self, code, lang):
         if lang:
             try:
                 lexer = get_lexer_by_name(lang, stripall=True)
             except ClassNotFound:
                 code = lang + '\n' + code
                 lang = None
 
         if not lang:
             return '\n<pre><code>%s</code></pre>\n' % \
                 mistune.escape(code)
 
         formatter = HtmlFormatter()
         return highlight(code, lexer, formatter)
 
     # Pass math through unaltered - mathjax does the rendering in the browser
     def block_math(self, text):
         return '$$%s$$' % text
 
     def latex_environment(self, name, text):
         return r'\begin{%s}%s\end{%s}' % (name, text, name)
 
     def inline_math(self, text):
         return '$%s$' % text
 
 def markdown2html_mistune(source):
     """Convert a markdown string to HTML using mistune"""
     return MarkdownWithMath(renderer=IPythonRenderer()).render(source)
 
 def markdown2html_pandoc(source, extra_args=None):
     """Convert a markdown string to HTML via pandoc"""
     extra_args = extra_args or ['--mathjax']
     return pandoc(source, 'markdown', 'html', extra_args=extra_args)
 
 def _find_nodejs():
     global _node
     if _node is None:
         # prefer md2html via marked if node.js >= 0.9.12 is available
         # node is called nodejs on debian, so try that first
         _node = 'nodejs'
         if not _verify_node(_node):
             _node = 'node'
     return _node
 
 def markdown2html_marked(source, encoding='utf-8'):
     """Convert a markdown string to HTML via marked"""
     command = [_find_nodejs(), marked]
     try:
         p = subprocess.Popen(command,
                              stdin=subprocess.PIPE, stdout=subprocess.PIPE
         )
     except OSError as e:
         raise NodeJSMissing(
             "The command '%s' returned an error: %s.\n" % (" ".join(command), e) +
             "Please check that Node.js is installed."
         )
     out, _ = p.communicate(cast_bytes(source, encoding))
     out = TextIOWrapper(BytesIO(out), encoding, 'replace').read()
     return out.rstrip('\n')
 
 # The mistune renderer is the default, because it's simple to depend on it
 markdown2html = markdown2html_mistune
 
 def markdown2rst(source, extra_args=None):
     """Convert a markdown string to ReST via pandoc.
 
     This function will raise an error if pandoc is not installed.
     Any error messages generated by pandoc are printed to stderr.
 
     Parameters
     ----------
     source : string
       Input string, assumed to be valid markdown.
 
     Returns
     -------
     out : string
       Output as returned by pandoc.
     """
     return pandoc(source, 'markdown', 'rst', extra_args=extra_args)
 
 def _verify_node(cmd):
     """Verify that the node command exists and is at least the minimum supported
     version of node.
 
     Parameters
     ----------
     cmd : string
         Node command to verify (i.e 'node')."""
     try:
         out, err, return_code = get_output_error_code([cmd, '--version'])
     except OSError:
         # Command not found
         return False
     if return_code:
         # Command error
         return False
     return check_version(out.lstrip('v'), '0.9.12')

IPython/nbconvert/filters/strings.py

@@ -1,221 +1,232 b''
 # coding: utf-8
 """String filters.
 
 Contains a collection of useful string manipulation filters for use in Jinja
 templates.
 """
 #-----------------------------------------------------------------------------
 # Copyright (c) 2013, the IPython Development Team.
 #
 # Distributed under the terms of the Modified BSD License.
 #
 # The full license is in the file COPYING.txt, distributed with this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 import os
 import re
 import textwrap
 try:
     from urllib.parse import quote  # Py 3
 except ImportError:
     from urllib2 import quote  # Py 2
 from xml.etree import ElementTree
 
 from IPython.core.interactiveshell import InteractiveShell
 from IPython.utils import py3compat
 
 #-----------------------------------------------------------------------------
 # Functions
 #-----------------------------------------------------------------------------
 
 __all__ = [
     'wrap_text',
     'html2text',
     'add_anchor',
     'strip_dollars',
     'strip_files_prefix',
     'comment_lines',
     'get_lines',
     'ipython2python',
     'posix_path',
     'path2url',
     'add_prompts',
     'ascii_only',
+    'prevent_list_blocks',
 ]
 
 
 def wrap_text(text, width=100):
     """ 
     Intelligently wrap text.
     Wrap text without breaking words if possible.
     
     Parameters
     ----------
     text : str
         Text to wrap.
     width : int, optional
         Number of characters to wrap to, default 100.
     """
 
     split_text = text.split('\n')
     wrp = map(lambda x:textwrap.wrap(x,width), split_text)
     wrpd = map('\n'.join, wrp)
     return '\n'.join(wrpd)
 
 
 def html2text(element):
     """extract inner text from html
     
     Analog of jQuery's $(element).text()
     """
     if isinstance(element, py3compat.string_types):
         try:
             element = ElementTree.fromstring(element)
         except Exception:
             # failed to parse, just return it unmodified
             return element
     
     text = element.text or ""
     for child in element:
         text += html2text(child)
     text += (element.tail or "")
     return text
 
 
 def add_anchor(html):
     """Add an anchor-link to an html header tag
     
     For use in heading cells
     """
     try:
         h = ElementTree.fromstring(py3compat.cast_bytes_py2(html, encoding='utf-8'))
     except Exception:
         # failed to parse, just return it unmodified
         return html
     link = html2text(h).replace(' ', '-')
     h.set('id', link)
     a = ElementTree.Element("a", {"class" : "anchor-link", "href" : "#" + link})
     a.text = u'¶'
     h.append(a)
 
     # Known issue of Python3.x, ElementTree.tostring() returns a byte string
     # instead of a text string.  See issue http://bugs.python.org/issue10942
     # Workaround is to make sure the bytes are casted to a string.
     return py3compat.decode(ElementTree.tostring(h), 'utf-8')
 
 
 def add_prompts(code, first='>>> ', cont='... '):
     """Add prompts to code snippets"""
     new_code = []
     code_list = code.split('\n')
     new_code.append(first + code_list[0])
     for line in code_list[1:]:
         new_code.append(cont + line)
     return '\n'.join(new_code)
 
     
 def strip_dollars(text):
     """
     Remove all dollar symbols from text
     
     Parameters
     ----------
     text : str
         Text to remove dollars from
     """
 
     return text.strip('$')
 
 
 files_url_pattern = re.compile(r'(src|href)\=([\'"]?)/?files/')
 markdown_url_pattern = re.compile(r'(!?)\[(?P<caption>.*?)\]\(/?files/(?P<location>.*?)\)')
 
 def strip_files_prefix(text):
     """
     Fix all fake URLs that start with `files/`, stripping out the `files/` prefix.
     Applies to both urls (for html) and relative paths (for markdown paths).
     
     Parameters
     ----------
     text : str
         Text in which to replace 'src="files/real...' with 'src="real...'
     """
     cleaned_text = files_url_pattern.sub(r"\1=\2", text)
     cleaned_text = markdown_url_pattern.sub(r'\1[\2](\3)', cleaned_text)
     return cleaned_text
 
 
 def comment_lines(text, prefix='# '):
     """
     Build a Python comment line from input text.
     
     Parameters
     ----------
     text : str
         Text to comment out.
     prefix : str
         Character to append to the start of each line.
     """
     
     #Replace line breaks with line breaks and comment symbols.
     #Also add a comment symbol at the beginning to comment out
     #the first line.
     return prefix + ('\n'+prefix).join(text.split('\n')) 
 
 
 def get_lines(text, start=None,end=None):
     """
     Split the input text into separate lines and then return the 
     lines that the caller is interested in.
     
     Parameters
     ----------
     text : str
         Text to parse lines from.
     start : int, optional
         First line to grab from.
     end : int, optional
         Last line to grab from.
     """
     
     # Split the input into lines.
     lines = text.split("\n")
     
     # Return the right lines.
     return "\n".join(lines[start:end]) #re-join
 
 def ipython2python(code):
     """Transform IPython syntax to pure Python syntax
 
     Parameters
     ----------
 
     code : str
         IPython code, to be transformed to pure Python
     """
     shell = InteractiveShell.instance()
     return shell.input_transformer_manager.transform_cell(code)
 
 def posix_path(path):
     """Turn a path into posix-style path/to/etc
     
     Mainly for use in latex on Windows,
     where native Windows paths are not allowed.
     """
     if os.path.sep != '/':
         return path.replace(os.path.sep, '/')
     return path
 
 def path2url(path):
     """Turn a file path into a URL"""
     parts = path.split(os.path.sep)
     return '/'.join(quote(part) for part in parts)
 
 def ascii_only(s):
     """ensure a string is ascii"""
     s = py3compat.cast_unicode(s)
     return s.encode('ascii', 'replace').decode('ascii')
+
+def prevent_list_blocks(s):
+    """
+    Prevent presence of enumerate or itemize blocks in latex headings cells
+    """
+    out = re.sub('(^\s*\d*)\.', '\\1\.', s)
+    out = re.sub('(^\s*)\-', '\\1\-', out)
+    out = re.sub('(^\s*)\+', '\\1\+', out)
+    out = re.sub('(^\s*)\*', '\\1\*', out)
+    return out

IPython/nbconvert/filters/tests/test_markdown.py

@@ -1,115 +1,129 b''
 """Tests for conversions from markdown to other formats"""
 
 # Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
 
 from copy import copy
 
 from IPython.utils.py3compat import string_types
 from IPython.testing import decorators as dec
 
 from ...tests.base import TestsBase
 from ..markdown import markdown2latex, markdown2html, markdown2rst
 
 from jinja2 import Environment
 
 class TestMarkdown(TestsBase):
 
     tests = [
         '*test',
         '**test',
         '*test*',
         '_test_',
         '__test__',
         '__*test*__',
         '**test**',
         '#test',
         '##test',
         'test\n----',
         'test [link](https://google.com/)']
 
     tokens = [
         '*test',
         '**test',
         'test',
         'test',
         'test',
         'test',
         'test',
         'test',
         'test',
         'test',
         ('test', 'https://google.com/')]
 
 
     @dec.onlyif_cmds_exist('pandoc')
     def test_markdown2latex(self):
         """markdown2latex test"""
         for index, test in enumerate(self.tests):
             self._try_markdown(markdown2latex, test, self.tokens[index])
 
     @dec.onlyif_cmds_exist('pandoc')
+    def test_markdown2latex_markup(self):
+        """markdown2latex with markup kwarg test"""
+        # This string should be passed through unaltered with pandoc's
+        # markdown_strict reader
+        s = '1) arabic number with parenthesis'
+        self.assertEqual(markdown2latex(s, markup='markdown_strict'), s)
+        # This string should be passed through unaltered with pandoc's
+        # markdown_strict+tex_math_dollars reader
+        s = '$\\alpha$ latex math'
+        self.assertEqual(
+            markdown2latex(s, markup='markdown_strict+tex_math_dollars'),
+            s)
+
+    @dec.onlyif_cmds_exist('pandoc')
     def test_pandoc_extra_args(self):
         # pass --no-wrap
         s = '\n'.join([
-            "#latex {{long_line | md2l(['--no-wrap'])}}",
+            "#latex {{long_line | md2l('markdown', ['--no-wrap'])}}",
             "#rst {{long_line | md2r(['--columns', '5'])}}",
         ])
         long_line = ' '.join(['long'] * 30)
         env = Environment()
         env.filters.update({
             'md2l': markdown2latex,
             'md2r': markdown2rst,
         })
         tpl = env.from_string(s)
         rendered = tpl.render(long_line=long_line)
         _, latex, rst = rendered.split('#')
         
         self.assertEqual(latex.strip(), 'latex %s' % long_line)
         self.assertEqual(rst.strip(), 'rst %s' % long_line.replace(' ', '\n'))
 
     def test_markdown2html(self):
         """markdown2html test"""
         for index, test in enumerate(self.tests):
             self._try_markdown(markdown2html, test, self.tokens[index])
 
     def test_markdown2html_math(self):
         # Mathematical expressions should be passed through unaltered
         cases = [("\\begin{equation*}\n"
                   "\\left( \\sum_{k=1}^n a_k b_k \\right)^2 \\leq \\left( \\sum_{k=1}^n a_k^2 \\right) \\left( \\sum_{k=1}^n b_k^2 \\right)\n"
                   "\\end{equation*}"),
                  ("$$\n"
                   "a = 1 *3* 5\n"
                   "$$"),
                   "$ a = 1 *3* 5 $",
                 ]
         for case in cases:
             self.assertIn(case, markdown2html(case))
 
     def test_markdown2html_math_paragraph(self):
         # https://github.com/ipython/ipython/issues/6724
         a = """Water that is stored in $t$, $s_t$, must equal the storage content of the previous stage,
 $s_{t-1}$, plus a stochastic inflow, $I_t$, minus what is being released in $t$, $r_t$.
 With $s_0$ defined as the initial storage content in $t=1$, we have"""
         self.assertIn(a, markdown2html(a))
 
     @dec.onlyif_cmds_exist('pandoc')
     def test_markdown2rst(self):
         """markdown2rst test"""
 
         #Modify token array for rst, escape asterik
         tokens = copy(self.tokens)
         tokens[0] = r'\*test'
         tokens[1] = r'\*\*test'
 
         for index, test in enumerate(self.tests):
             self._try_markdown(markdown2rst, test, tokens[index])
 
 
     def _try_markdown(self, method, test, tokens):
         results = method(test)
         if isinstance(tokens, string_types):
             assert tokens in results
         else:
             for token in tokens:
                 assert token in results

IPython/nbconvert/filters/tests/test_strings.py

@@ -1,153 +1,165 b''
 """
 Module with tests for Strings
 """
 
 #-----------------------------------------------------------------------------
 # Copyright (c) 2013, the IPython Development Team.
 #
 # Distributed under the terms of the Modified BSD License.
 #
 # The full license is in the file COPYING.txt, distributed with this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 import os
 
 from ...tests.base import TestsBase
 from ..strings import (wrap_text, html2text, add_anchor, strip_dollars, 
     strip_files_prefix, get_lines, comment_lines, ipython2python, posix_path,
-    add_prompts
+    add_prompts, prevent_list_blocks
 )
 
 
 #-----------------------------------------------------------------------------
 # Class
 #-----------------------------------------------------------------------------
 
 class TestStrings(TestsBase):
 
     def test_wrap_text(self):
         """wrap_text test"""
         test_text = """
         Tush! never tell me; I take it much unkindly
         That thou, Iago, who hast had my purse
         As if the strings were thine, shouldst know of this.
         """
         for length in [30,5,1]:
             self._confirm_wrap_text(test_text, length)
     
 
     def _confirm_wrap_text(self, text, length):
         for line in wrap_text(text, length).split('\n'):
             assert len(line) <= length
 
 
     def test_html2text(self):
         """html2text test"""
         #TODO: More tests
         self.assertEqual(html2text('<name>joe</name>'), 'joe')
         
 
     def test_add_anchor(self):
         """add_anchor test"""
         #TODO: More tests
         results = add_anchor('<b>Hello World!</b>')
         assert 'Hello World!' in results
         assert 'id="' in results
         assert 'class="anchor-link"' in results
         assert '<b' in results
         assert '</b>' in results
 
     def test_add_anchor_fail(self):
         """add_anchor does nothing when it fails"""
         html = '<h1>Hello <br>World!</h1>'
         results = add_anchor(html)
         self.assertEqual(html, results)
 
     def test_strip_dollars(self):
         """strip_dollars test"""
         tests = [
             ('', ''), 
             ('$$', ''), 
             ('$H$', 'H'), 
             ('$He', 'He'), 
             ('H$el', 'H$el'), 
             ('Hell$', 'Hell'),
             ('Hello', 'Hello'),
             ('W$o$rld', 'W$o$rld')]
         for test in tests:
             self._try_strip_dollars(test[0], test[1])
 
 
     def _try_strip_dollars(self, test, result):
         self.assertEqual(strip_dollars(test), result)
 
 
     def test_strip_files_prefix(self):
         """strip_files_prefix test"""
         tests = [
             ('', ''), 
             ('/files', '/files'), 
             ('test="/files"', 'test="/files"'), 
             ('My files are in `files/`', 'My files are in `files/`'),
             ('<a href="files/test.html">files/test.html</a>', '<a href="test.html">files/test.html</a>'),
             ('<a href="/files/test.html">files/test.html</a>', '<a href="test.html">files/test.html</a>'),
             ("<a href='files/test.html'>files/test.html</a>", "<a href='test.html'>files/test.html</a>"),
             ('<img src="files/url/location.gif">', '<img src="url/location.gif">'),
             ('<img src="/files/url/location.gif">', '<img src="url/location.gif">'),
             ('hello![caption]', 'hello![caption]'),
             ('hello![caption](/url/location.gif)', 'hello![caption](/url/location.gif)'),
             ('hello![caption](url/location.gif)', 'hello![caption](url/location.gif)'),
             ('hello![caption](url/location.gif)', 'hello![caption](url/location.gif)'),
             ('hello![caption](files/url/location.gif)', 'hello![caption](url/location.gif)'),
             ('hello![caption](/files/url/location.gif)', 'hello![caption](url/location.gif)'),
             ('hello [text](/files/url/location.gif)', 'hello [text](url/location.gif)'),
             ('hello [text space](files/url/location.gif)', 'hello [text space](url/location.gif)'),
         ]
         for test in tests:
             self._try_files_prefix(test[0], test[1])
 
 
     def _try_files_prefix(self, test, result):
         self.assertEqual(strip_files_prefix(test), result)
         
 
     def test_comment_lines(self):
         """comment_lines test"""
         for line in comment_lines('hello\nworld\n!').split('\n'):
             assert line.startswith('# ')
         for line in comment_lines('hello\nworld\n!', 'beep').split('\n'):
             assert line.startswith('beep')
         
 
     def test_get_lines(self):
         """get_lines test"""
         text = "hello\nworld\n!"
         self.assertEqual(get_lines(text, start=1), "world\n!")
         self.assertEqual(get_lines(text, end=2), "hello\nworld")
         self.assertEqual(get_lines(text, start=2, end=5), "!")
         self.assertEqual(get_lines(text, start=-2), "world\n!")
         
 
     def test_ipython2python(self):
         """ipython2python test"""
         #TODO: More tests
         results = ipython2python(u'%%pylab\nprint("Hello-World")').replace("u'", "'")
         self.fuzzy_compare(results, u"get_ipython().run_cell_magic('pylab', '', 'print(\"Hello-World\")')", 
             ignore_spaces=True, ignore_newlines=True)
     
     def test_posix_path(self):
         """posix_path test"""
         path_list = ['foo', 'bar']
         expected = '/'.join(path_list)
         native = os.path.join(*path_list)
         filtered = posix_path(native)
         self.assertEqual(filtered, expected)
     
     def test_add_prompts(self):
         """add_prompts test"""
         text1 = """for i in range(10):\n  i += 1\n  print i"""
         text2 = """>>> for i in range(10):\n...   i += 1\n...   print i"""
         self.assertEqual(text2, add_prompts(text1))
+
+    def test_prevent_list_blocks(self):
+        """prevent_list_blocks test"""
+        tests = [
+            ('1. arabic point', '1\\. arabic point'),
+            ('* bullet asterisk', '\\* bullet asterisk'),
+            ('+ bullet Plus Sign', '\\+ bullet Plus Sign'),
+            ('- bullet Hyphen-Minus', '\\- bullet Hyphen-Minus'),
+            ('  1. spaces + arabic point', '  1\\. spaces + arabic point'),
+        ]
+        for test in tests:
+            self.assertEqual(prevent_list_blocks(test[0]), test[1])

IPython/nbconvert/templates/latex/base.tplx

@@ -1,222 +1,223 b''
 ((= Latex base template (must inherit)
 This template builds upon the abstract template, adding common latex output
 functions.  Figures, data_text,
 This template does not define a docclass, the inheriting class must define this.=))
 
 ((*- extends 'display_priority.tplx' -*))
 
 %===============================================================================
 % Abstract overrides
 %===============================================================================
 
 ((* block header *))
     ((* block docclass *))((* endblock docclass *))
     
     ((* block packages *))
     \usepackage{graphicx} % Used to insert images
     \usepackage{adjustbox} % Used to constrain images to a maximum size 
     \usepackage{color} % Allow colors to be defined
     \usepackage{enumerate} % Needed for markdown enumerations to work
     \usepackage{geometry} % Used to adjust the document margins
     \usepackage{amsmath} % Equations
     \usepackage{amssymb} % Equations
     \usepackage[mathletters]{ucs} % Extended unicode (utf-8) support
     \usepackage[utf8x]{inputenc} % Allow utf-8 characters in the tex document
     \usepackage{fancyvrb} % verbatim replacement that allows latex
     \usepackage{grffile} % extends the file name processing of package graphics 
                          % to support a larger range 
     % The hyperref package gives us a pdf with properly built
     % internal navigation ('pdf bookmarks' for the table of contents,
     % internal cross-reference links, web links for URLs, etc.)
     \usepackage{hyperref}
     \usepackage{longtable} % longtable support required by pandoc >1.10
     \usepackage{booktabs}  % table support for pandoc > 1.12.2
     ((* endblock packages *))
 
     ((* block definitions *))
     \definecolor{orange}{cmyk}{0,0.4,0.8,0.2}
     \definecolor{darkorange}{rgb}{.71,0.21,0.01}
     \definecolor{darkgreen}{rgb}{.12,.54,.11}
     \definecolor{myteal}{rgb}{.26, .44, .56}
     \definecolor{gray}{gray}{0.45}
     \definecolor{lightgray}{gray}{.95}
     \definecolor{mediumgray}{gray}{.8}
     \definecolor{inputbackground}{rgb}{.95, .95, .85}
     \definecolor{outputbackground}{rgb}{.95, .95, .95}
     \definecolor{traceback}{rgb}{1, .95, .95}
     % ansi colors
     \definecolor{red}{rgb}{.6,0,0}
     \definecolor{green}{rgb}{0,.65,0}
     \definecolor{brown}{rgb}{0.6,0.6,0}
     \definecolor{blue}{rgb}{0,.145,.698}
     \definecolor{purple}{rgb}{.698,.145,.698}
     \definecolor{cyan}{rgb}{0,.698,.698}
     \definecolor{lightgray}{gray}{0.5}
     
     % bright ansi colors
     \definecolor{darkgray}{gray}{0.25}
     \definecolor{lightred}{rgb}{1.0,0.39,0.28}
     \definecolor{lightgreen}{rgb}{0.48,0.99,0.0}
     \definecolor{lightblue}{rgb}{0.53,0.81,0.92}
     \definecolor{lightpurple}{rgb}{0.87,0.63,0.87}
     \definecolor{lightcyan}{rgb}{0.5,1.0,0.83}
     
     % commands and environments needed by pandoc snippets
     % extracted from the output of `pandoc -s`
     \DefineVerbatimEnvironment{Highlighting}{Verbatim}{commandchars=\\\{\}}
     % Add ',fontsize=\small' for more characters per line
     \newenvironment{Shaded}{}{}
     \newcommand{\KeywordTok}[1]{\textcolor[rgb]{0.00,0.44,0.13}{\textbf{{#1}}}}
     \newcommand{\DataTypeTok}[1]{\textcolor[rgb]{0.56,0.13,0.00}{{#1}}}
     \newcommand{\DecValTok}[1]{\textcolor[rgb]{0.25,0.63,0.44}{{#1}}}
     \newcommand{\BaseNTok}[1]{\textcolor[rgb]{0.25,0.63,0.44}{{#1}}}
     \newcommand{\FloatTok}[1]{\textcolor[rgb]{0.25,0.63,0.44}{{#1}}}
     \newcommand{\CharTok}[1]{\textcolor[rgb]{0.25,0.44,0.63}{{#1}}}
     \newcommand{\StringTok}[1]{\textcolor[rgb]{0.25,0.44,0.63}{{#1}}}
     \newcommand{\CommentTok}[1]{\textcolor[rgb]{0.38,0.63,0.69}{\textit{{#1}}}}
     \newcommand{\OtherTok}[1]{\textcolor[rgb]{0.00,0.44,0.13}{{#1}}}
     \newcommand{\AlertTok}[1]{\textcolor[rgb]{1.00,0.00,0.00}{\textbf{{#1}}}}
     \newcommand{\FunctionTok}[1]{\textcolor[rgb]{0.02,0.16,0.49}{{#1}}}
     \newcommand{\RegionMarkerTok}[1]{{#1}}
     \newcommand{\ErrorTok}[1]{\textcolor[rgb]{1.00,0.00,0.00}{\textbf{{#1}}}}
     \newcommand{\NormalTok}[1]{{#1}}
     
     % Define a nice break command that doesn't care if a line doesn't already
     % exist.
     \def\br{\hspace*{\fill} \\* }
     % Math Jax compatability definitions
     \def\gt{>}
     \def\lt{<}
     % Document parameters
     ((* block title *))\title{((( resources.metadata.name | ascii_only | escape_latex )))}((* endblock title *))
     ((* block date *))((* endblock date *))
     ((* block author *))((* endblock author *))
     ((* endblock definitions *))
 
     ((* block commands *))
     % Prevent overflowing lines due to hard-to-break entities
     \sloppy 
     % Setup hyperref package
     \hypersetup{
       breaklinks=true,  % so long urls are correctly broken across lines
       colorlinks=true,
       urlcolor=blue,
       linkcolor=darkorange,
       citecolor=darkgreen,
       }
     % Slightly bigger margins than the latex defaults
     ((* block margins *))
     \geometry{verbose,tmargin=1in,bmargin=1in,lmargin=1in,rmargin=1in}
     ((* endblock margins *))
     ((* endblock commands *))
 ((* endblock header *))
 
 ((* block body *))
     \begin{document}
     
     ((* block predoc *))
     ((* block maketitle *))\maketitle((* endblock maketitle *))
     ((* block abstract *))((* endblock abstract *))
     ((* endblock predoc *))
 
     ((( super() )))
 
     % Add a bibliography block to the postdoc
     ((* block postdoc *))
     ((* block bibliography *))((* endblock bibliography *))
     ((* endblock postdoc *))
     \end{document}
 ((* endblock body *))
 
 %===============================================================================
 % Support blocks
 %===============================================================================
 
 % Displaying simple data text
 ((* block data_text *))
     \begin{verbatim}
 ((( output.text )))
     \end{verbatim}
 ((* endblock data_text *))
 
 % Display python error text as-is
 ((* block pyerr *))
     \begin{Verbatim}[commandchars=\\\{\}]
 ((( super() )))
     \end{Verbatim}
 ((* endblock pyerr *))
 ((* block traceback_line *))
     ((( line | indent | strip_ansi | escape_latex )))
 ((* endblock traceback_line *))
 
 % Display stream ouput with coloring
 ((* block stream *))
     \begin{Verbatim}[commandchars=\\\{\}]
 ((( output.text | escape_latex | ansi2latex )))
     \end{Verbatim}
 ((* endblock stream *))
 
 % Display latex
 ((* block data_latex -*))
     ((*- if output.latex.startswith('$'): -*))
         ((= Replace $ symbols with more explicit, equation block. =))
         \begin{equation*}\adjustbox{max width=\hsize}{$
         ((( output.latex | strip_dollars )))
         $}\end{equation*}
     ((*- else -*))
         ((( output.latex )))
     ((*- endif *))
 ((* endblock data_latex *))
 
 % Default mechanism for rendering figures
 ((*- block data_png -*))((( draw_figure(output.png_filename) )))((*- endblock -*))
 ((*- block data_jpg -*))((( draw_figure(output.jpeg_filename) )))((*- endblock -*))
 ((*- block data_svg -*))((( draw_figure(output.svg_filename) )))((*- endblock -*))
 ((*- block data_pdf -*))((( draw_figure(output['application/pdf_filename']) )))((*- endblock -*))
 
 % Draw a figure using the graphicx package.
 ((* macro draw_figure(filename) -*))
 ((* set filename = filename | posix_path *))
 ((*- block figure scoped -*))
     \begin{center}
     \adjustimage{max size={0.9\linewidth}{0.9\paperheight}}{((( filename )))}
     \end{center}
     { \hspace*{\fill} \\}
 ((*- endblock figure -*))
 ((*- endmacro *))
 
 % Draw heading cell.  Explicitly map different cell levels.
 ((* block headingcell scoped *))
 
     ((* if cell.level == 1 -*))
         ((* block h1 -*))\section((* endblock h1 -*))
     ((* elif cell.level == 2 -*))
         ((* block h2 -*))\subsection((* endblock h2 -*))
     ((* elif cell.level == 3 -*))
         ((* block h3 -*))\subsubsection((* endblock h3 -*))
     ((* elif cell.level == 4 -*))
         ((* block h4 -*))\paragraph((* endblock h4 -*))
     ((* elif cell.level == 5 -*))
         ((* block h5 -*))\subparagraph((* endblock h5 -*))
     ((* elif cell.level == 6 -*))
         ((* block h6 -*))\\*\textit((* endblock h6 -*))
     ((*- endif -*))
-    {((( cell.source | replace('\n', ' ') | citation2latex | strip_files_prefix | markdown2latex )))}
+    {((( cell.source | replace('\n', ' ') | citation2latex | strip_files_prefix | prevent_list_blocks | markdown2latex(markup='markdown_strict+tex_math_dollars') )))}
+    
 
 ((* endblock headingcell *))
 
 % Redirect pyout to display data priority.
 ((* block pyout scoped *))
     ((* block data_priority scoped *))
     ((( super() )))
     ((* endblock *))
 ((* endblock pyout *))
 
 % Render markdown
 ((* block markdowncell scoped *))
     ((( cell.source | citation2latex | strip_files_prefix | markdown2latex )))
 ((* endblock markdowncell *))
 
 % Don't display unknown types
 ((* block unknowncell scoped *))
 ((* endblock unknowncell *))