converters/template.py

"""Base classes for the notebook conversion pipeline.

This module defines Converter, from which all objects designed to implement
a conversion of IPython notebooks to some other format should inherit.
"""
#-----------------------------------------------------------------------------
# Copyright (c) 2012, the IPython Development Team.
#
# Distributed under the terms of the Modified BSD License.
#
# The full license is in the file COPYING.txt, distributed with this software.
#-----------------------------------------------------------------------------

#-----------------------------------------------------------------------------
# Imports
#-----------------------------------------------------------------------------

from __future__ import print_function, absolute_import

# Stdlib imports
import io
import os
import re
from IPython.utils import path

from jinja2 import Environment, FileSystemLoader
env = Environment(
        loader=FileSystemLoader([
            './templates/',
            './templates/skeleton/',
            ]),
        extensions=['jinja2.ext.loopcontrols']
        )

texenv = Environment(
        loader=FileSystemLoader([
            './templates/tex/',
            './templates/skeleton/tex/',
            ]),
        extensions=['jinja2.ext.loopcontrols']
        )

# IPython imports
from IPython.nbformat import current as nbformat
from IPython.config.configurable import Configurable
from IPython.utils.traitlets import ( Unicode, Any, List, Bool)

# Our own imports
from IPython.utils.text import indent
from .utils import remove_ansi
from markdown import markdown
from .utils import highlight, ansi2html
from .utils import markdown2latex
#-----------------------------------------------------------------------------
# Class declarations
#-----------------------------------------------------------------------------
def rm_fake(strng):
    return strng.replace('/files/', '')

class ConversionException(Exception):
    pass


def python_comment(string):
    return '# '+'\n# '.join(string.split('\n'))



def header_body():
    """Return the body of the header as a list of strings."""

    from pygments.formatters import HtmlFormatter

    header = []
    static = os.path.join(path.get_ipython_package_dir(),
    'frontend', 'html', 'notebook', 'static',
    )
    here = os.path.split(os.path.realpath(__file__))[0]
    css = os.path.join(static, 'css')
    for sheet in [
        # do we need jquery and prettify?
        # os.path.join(static, 'jquery', 'css', 'themes', 'base',
        # 'jquery-ui.min.css'),
        # os.path.join(static, 'prettify', 'prettify.css'),
        os.path.join(css, 'boilerplate.css'),
        os.path.join(css, 'fbm.css'),
        os.path.join(css, 'notebook.css'),
        os.path.join(css, 'renderedhtml.css'),
        # our overrides:
        os.path.join(here, '..', 'css', 'static_html.css'),
    ]:

        with io.open(sheet, encoding='utf-8') as f:
            s = f.read()
            header.append(s)

    pygments_css = HtmlFormatter().get_style_defs('.highlight')
    header.append(pygments_css)
    return header

# todo, make the key part configurable.
def _new_figure(data, fmt, count):
    """Create a new figure file in the given format.

    Returns a path relative to the input file.
    """
    figname = '_fig_%02i.%s' % (count, fmt)

    # Binary files are base64-encoded, SVG is already XML
    if fmt in ('png', 'jpg', 'pdf'):
        data = data.decode('base64')

    return figname,data




inlining = {}
inlining['css'] = header_body()

LATEX_SUBS = (
    (re.compile(r'\\'), r'\\textbackslash'),
    (re.compile(r'([{}_#%&$])'), r'\\\1'),
    (re.compile(r'~'), r'\~{}'),
    (re.compile(r'\^'), r'\^{}'),
    (re.compile(r'"'), r"''"),
    (re.compile(r'\.\.\.+'), r'\\ldots'),
)

def escape_tex(value):
    newval = value
    for pattern, replacement in LATEX_SUBS:
        newval = pattern.sub(replacement, newval)
    return newval

texenv.block_start_string = '((*'
texenv.block_end_string = '*))'
texenv.variable_start_string = '((('
texenv.variable_end_string = ')))'
texenv.comment_start_string = '((='
texenv.comment_end_string = '=))'
texenv.filters['escape_tex'] = escape_tex

def cell_preprocessor(function):
    """ wrap a function to be executed on all cells of a notebook

    wrapped function  parameters :
    cell  : the cell
    other : external resources
    index : index of the cell
    """
    def wrappedfunc(nb,other):
        for worksheet in nb.worksheets :
            for index, cell in enumerate(worksheet.cells):
                worksheet.cells[index],other= function(cell,other,index)
        return nb,other
    return wrappedfunc



@cell_preprocessor
def haspyout_transformer(cell, other, count):
    """
    Add a haspyout flag to cell that have it
    
    Easier for templating, where you can't know in advance
    wether to write the out prompt

    """
    cell.type = cell.cell_type
    cell.haspyout = False
    for out in cell.get('outputs', []):
        if out.output_type == 'pyout':
            cell.haspyout = True
            break
    return cell,other


@cell_preprocessor
def extract_figure_transformer(cell,other,count):
    for i,out in enumerate(cell.get('outputs', [])):
        for type in ['html', 'pdf', 'svg', 'latex', 'png', 'jpg', 'jpeg']:
            if out.hasattr(type):
                figname,data = _new_figure(out[type], type,count)
                cell.outputs[i][type] = figname
                out['key_'+type] = figname
                other[figname] = data
                count = count+1
    return cell,other


class ConverterTemplate(Configurable):
    """ A Jinja2 base converter templates"""

    display_data_priority = List(['html', 'pdf', 'svg', 'latex', 'png', 'jpg', 'jpeg' , 'text'],
            config=True,
              help= """
                    An ordered list of prefered output type, the firs encounterd will usually be 
                    used when converting discarding the others.
                    """
            )
    
    pre_transformer_order = List([],
            config=True,
              help= """ An ordered list of pretransformer to apply to the ipynb file befor running through templates
                    """
            )

    extract_figures = Bool(False,
            config=True,
              help= """
                    wether to remove figure data from ipynb and store them in auxiliary
                    dictionnary
                    """
            )

    tex_environement = Bool(False,
            config=True,
            help=""" is this a tex environment or not """)

    template_file = Unicode('',
            config=True,
            help=""" whetever """ )
    #-------------------------------------------------------------------------
    # Instance-level attributes that are set in the constructor for this
    # class.
    #-------------------------------------------------------------------------
    infile = Any()


    infile_dir = Unicode()

    def filter_data_type(self,output):
        for fmt in self.display_data_priority:
            if fmt in output:
                return [fmt]

    def __init__(self, preprocessors=[], config=None, **kw):
        """
        tplfile : jinja template file to process.

        config: the Configurable confg object to pass around

        preprocessors: list of function to run on ipynb json data before conversion
        to extract/inline file,

        """
        super(ConverterTemplate, self).__init__(config=config, **kw)
        self.env = texenv  if self.tex_environement else env
        self.ext = '.tplx' if self.tex_environement else '.tpl'
        self.nb = None
        self.preprocessors = preprocessors
        self.preprocessors.append(haspyout_transformer)
        if self.extract_figures:
            self.preprocessors.append(extract_figure_transformer)

        self.env.filters['filter_data_type'] = self.filter_data_type
        self.env.filters['pycomment'] = python_comment
        self.env.filters['indent'] = indent
        self.env.filters['rm_fake'] = rm_fake
        self.env.filters['rm_ansi'] = remove_ansi
        self.env.filters['markdown'] = markdown
        self.env.filters['highlight'] = highlight
        self.env.filters['ansi2html'] = ansi2html
        self.env.filters['markdown2latex'] = markdown2latex

        self.template = self.env.get_template(self.template_file+self.ext)



    def process(self):
        """
        preprocess the notebook json for easier use with the templates.
        will call all the `preprocessor`s in order before returning it.
        """
        nb = self.nb

        # dict of 'resources' that could be made by the preprocessors
        # like key/value data to extract files from ipynb like in latex conversion
        resources = {}

        for preprocessor in self.preprocessors:
            nb,resources = preprocessor(nb,resources)

        return nb, resources

    def convert(self):
        """ convert the ipynb file

        return both the converted ipynb file and a dict containing potential
        other resources
        """
        nb,resources = self.process()
        return self.template.render(nb=nb, inlining=inlining), resources


    def read(self, filename):
        "read and parse notebook into NotebookNode called self.nb"
        with io.open(filename) as f:
            self.nb = nbformat.read(f, 'json')