create configurable preprocessors

converters/template.py

@@ -1,220 +1,227 b''
 """Base classes for the notebook conversion pipeline.
 
 This module defines Converter, from which all objects designed to implement
 a conversion of IPython notebooks to some other format should inherit.
 """
 #-----------------------------------------------------------------------------
 # Copyright (c) 2012, the IPython Development Team.
 #
 # Distributed under the terms of the Modified BSD License.
 #
 # The full license is in the file COPYING.txt, distributed with this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 from __future__ import print_function, absolute_import
-from .transformers import extract_figure_transformer
 import converters.transformers as trans
 from converters.jinja_filters import (python_comment, indent,
         rm_fake, remove_ansi, markdown, highlight,
         ansi2html, markdown2latex, escape_tex)
 
 
+
 # Stdlib imports
 import io
 import os
 from IPython.utils import path
+from IPython.utils.traitlets import MetaHasTraits
 
 from jinja2 import Environment, FileSystemLoader
 env = Environment(
         loader=FileSystemLoader([
             './templates/',
             './templates/skeleton/',
             ]),
         extensions=['jinja2.ext.loopcontrols']
         )
 
 texenv = Environment(
         loader=FileSystemLoader([
             './templates/tex/',
             './templates/skeleton/tex/',
             ]),
         extensions=['jinja2.ext.loopcontrols']
         )
 
 # IPython imports
 from IPython.nbformat import current as nbformat
 from IPython.config.configurable import Configurable
 from IPython.utils.traitlets import ( Unicode, Any, List, Bool)
 
 #-----------------------------------------------------------------------------
 # Class declarations
 #-----------------------------------------------------------------------------
 class ConversionException(Exception):
     pass
 
 
 def header_body():
     """Return the body of the header as a list of strings."""
 
     from pygments.formatters import HtmlFormatter
 
     header = []
     static = os.path.join(path.get_ipython_package_dir(),
     'frontend', 'html', 'notebook', 'static',
     )
     here = os.path.split(os.path.realpath(__file__))[0]
     css = os.path.join(static, 'css')
     for sheet in [
         # do we need jquery and prettify?
         # os.path.join(static, 'jquery', 'css', 'themes', 'base',
         # 'jquery-ui.min.css'),
         # os.path.join(static, 'prettify', 'prettify.css'),
         os.path.join(css, 'boilerplate.css'),
         os.path.join(css, 'fbm.css'),
         os.path.join(css, 'notebook.css'),
         os.path.join(css, 'renderedhtml.css'),
         # our overrides:
         os.path.join(here, '..', 'css', 'static_html.css'),
     ]:
 
         with io.open(sheet, encoding='utf-8') as f:
             s = f.read()
             header.append(s)
 
     pygments_css = HtmlFormatter().get_style_defs('.highlight')
     header.append(pygments_css)
     return header
 
 
 
 
 
 inlining = {}
 inlining['css'] = header_body()
 
 
 
 texenv.block_start_string = '((*'
 texenv.block_end_string = '*))'
 texenv.variable_start_string = '((('
 texenv.variable_end_string = ')))'
 texenv.comment_start_string = '((='
 texenv.comment_end_string = '=))'
 texenv.filters['escape_tex'] = escape_tex
 
 
 class ConverterTemplate(Configurable):
     """ A Jinja2 base converter templates"""
 
     display_data_priority = List(['html', 'pdf', 'svg', 'latex', 'png', 'jpg', 'jpeg' , 'text'],
             config=True,
               help= """
                     An ordered list of prefered output type, the first
                     encounterd will usually be used when converting discarding
                     the others.
                     """
             )
-    
-    pre_transformer_order = List(['haspyout_transformer'],
+
+    pre_transformer_order = List(['haspyout_transformer', 'Foobar'],
             config=True,
               help= """
                     An ordered list of pre transformer to apply to the ipynb
                     file befor running through templates
                     """
             )
 
     extract_figures = Bool(False,
             config=True,
               help= """
                     wether to remove figure data from ipynb and store them in auxiliary
                     dictionnary
                     """
             )
 
     tex_environement = Bool(False,
             config=True,
             help=""" is this a tex environment or not """)
 
     template_file = Unicode('',
             config=True,
             help=""" whetever """ )
     #-------------------------------------------------------------------------
     # Instance-level attributes that are set in the constructor for this
     # class.
     #-------------------------------------------------------------------------
     infile = Any()
 
 
     infile_dir = Unicode()
 
     def filter_data_type(self, output):
         for fmt in self.display_data_priority:
             if fmt in output:
                 return [fmt]
 
     def __init__(self, preprocessors=[], config=None, **kw):
         """
         config: the Configurable confg object to pass around
 
         preprocessors: list of function to run on ipynb json data before conversion
         to extract/inline file,
 
         """
         super(ConverterTemplate, self).__init__(config=config, **kw)
         self.env = texenv  if self.tex_environement else env
         self.ext = '.tplx' if self.tex_environement else '.tpl'
         self.nb = None
         self.preprocessors = preprocessors
+
         for name in self.pre_transformer_order:
-            self.preprocessors.append(getattr(trans, name))
+            tr = getattr(trans, name)
+            if isinstance(tr, MetaHasTraits):
+                tr = tr(config=config)
+            self.preprocessors.append(tr)
         if self.extract_figures:
-            self.preprocessors.append(extract_figure_transformer)
+            self.preprocessors.append(trans.ExtractFigureTransformer(config=config))
 
         self.env.filters['filter_data_type'] = self.filter_data_type
         self.env.filters['pycomment'] = python_comment
         self.env.filters['indent'] = indent
         self.env.filters['rm_fake'] = rm_fake
         self.env.filters['rm_ansi'] = remove_ansi
         self.env.filters['markdown'] = markdown
         self.env.filters['highlight'] = highlight
         self.env.filters['ansi2html'] = ansi2html
         self.env.filters['markdown2latex'] = markdown2latex
 
         self.template = self.env.get_template(self.template_file+self.ext)
 
 
     def process(self):
         """
         preprocess the notebook json for easier use with the templates.
         will call all the `preprocessor`s in order before returning it.
         """
         nb = self.nb
 
         # dict of 'resources' that could be made by the preprocessors
         # like key/value data to extract files from ipynb like in latex conversion
         resources = {}
 
         for preprocessor in self.preprocessors:
             nb, resources = preprocessor(nb, resources)
 
         return nb, resources
 
     def convert(self):
         """ convert the ipynb file
 
         return both the converted ipynb file and a dict containing potential
         other resources
         """
         nb, resources = self.process()
         return self.template.render(nb=nb, inlining=inlining), resources
 
 
     def read(self, filename):
         "read and parse notebook into NotebookNode called self.nb"
         with io.open(filename) as f:
             self.nb = nbformat.read(f, 'json')
 
+
+

converters/transformers.py

@@ -1,64 +1,125 b''
 """
 
 """
 
+from __future__ import print_function
+
+from IPython.config.configurable import Configurable
+from IPython.utils.traitlets import Unicode, Bool, Dict
+
+class ConfigurableTransformers(Configurable):
+    """ A configurable transformer """
+
+    def __init__(self, config=None, **kw):
+        super(ConfigurableTransformers, self).__init__(config=config, **kw)
+
+    def __call__(self, nb, other):
+        try :
+            for worksheet in nb.worksheets :
+                for index, cell in enumerate(worksheet.cells):
+                    worksheet.cells[index], other = self.cell_transform(cell, other, index)
+            return nb, other
+        except NotImplementedError as error :
+            raise NotImplementedError('should be implemented by subclass')
+
+    def cell_transform(self, cell, other, index):
+        """
+        Overwrite if you want to apply a transformation on each cell
+        """
+        raise NotImplementedError('should be implemented by subclass')
+
+
+class Foobar(ConfigurableTransformers):
+    message = Unicode('-- nothing', config=True)
+
+
+    def cell_transform(self, cell, other, index):
+        return cell, other
+
+
 def cell_preprocessor(function):
     """ wrap a function to be executed on all cells of a notebook
 
     wrapped function  parameters :
     cell  : the cell
     other : external resources
     index : index of the cell
     """
     def wrappedfunc(nb, other):
         for worksheet in nb.worksheets :
             for index, cell in enumerate(worksheet.cells):
                 worksheet.cells[index], other = function(cell, other, index)
         return nb, other
     return wrappedfunc
 
 
 @cell_preprocessor
 def haspyout_transformer(cell, other, count):
     """
     Add a haspyout flag to cell that have it
-    
+
     Easier for templating, where you can't know in advance
     wether to write the out prompt
 
     """
     cell.type = cell.cell_type
     cell.haspyout = False
     for out in cell.get('outputs', []):
         if out.output_type == 'pyout':
             cell.haspyout = True
             break
     return cell, other
 
 
 # todo, make the key part configurable.
-def _new_figure(data, fmt, count):
-    """Create a new figure file in the given format.
 
-    Returns a path relative to the input file.
-    """
-    figname = '_fig_%02i.%s' % (count, fmt)
+class ExtractFigureTransformer(ConfigurableTransformers):
+    enabled = Bool(False,
+            config=True,
+            help=""" If set to false, this transformer will be no-op """
+            )
 
-    # Binary files are base64-encoded, SVG is already XML
-    if fmt in ('png', 'jpg', 'pdf'):
-        data = data.decode('base64')
+    extra_ext_map =  Dict({},
+            config=True,
+            help="""extra map to override extension based on type.
+            Usefull for latex where svg will be converted to pdf before inclusion
+            """
+            )
 
-    return figname, data
 
-@cell_preprocessor
-def extract_figure_transformer(cell, other, count):
-    for i, out in enumerate(cell.get('outputs', [])):
-        for type in ['html', 'pdf', 'svg', 'latex', 'png', 'jpg', 'jpeg']:
-            if out.hasattr(type):
-                figname, data = _new_figure(out[type], type, count)
-                cell.outputs[i][type] = figname
-                out['key_'+type] = figname
-                other[figname] = data
-                count = count+1
-    return cell, other
+    #to do change this to .format {} syntax
+    key_tpl = Unicode('_fig_%02i.%s', config=True)
+
+    def _get_ext(self, ext):
+        if ext in self.extra_ext_map :
+            return self.extra_ext_map[ext]
+        return ext
+
+    def _new_figure(self, data, fmt, count):
+        """Create a new figure file in the given format.
+
+        Returns a path relative to the input file.
+        """
+        figname = self.key_tpl % (count, self._get_ext(fmt))
+        key     = self.key_tpl % (count, fmt)
+
+        # Binary files are base64-encoded, SVG is already XML
+        if fmt in ('png', 'jpg', 'pdf'):
+            data = data.decode('base64')
+
+        return figname, key, data
+
+
+    def cell_transform(self, cell, other, count):
+        if not self.enabled:
+            return cell, other
+        for i, out in enumerate(cell.get('outputs', [])):
+            for type in ['html', 'pdf', 'svg', 'latex', 'png', 'jpg', 'jpeg']:
+                if out.hasattr(type):
+                    figname, key, data = self._new_figure(out[type], type, count)
+                    cell.outputs[i][type] = figname
+                    out['key_'+type] = figname
+                    other[key] = data
+                    count = count+1
+        return cell, other
 

profile/full_html.nbcv

@@ -1,5 +1,8 b''
 c = get_config()
 
+
 c.ConverterTemplate.extract_figures=False
 c.ConverterTemplate.template_file='fullhtml'
 c.ConverterTemplate.tex_environement=False
+
+c.ExtractFigureTransformer.enabled = False

profile/latex_base.nbcv

@@ -1,5 +1,7 b''
 c = get_config()
 
-c.ConverterTemplate.extract_figures=False
+c.ConverterTemplate.extract_figures=True
 c.ConverterTemplate.template_file='latex_base'
 c.ConverterTemplate.tex_environement=True
+
+c.ExtractFigureTransformer.extra_ext_map={'svg':'pdf'}

runme.py

@@ -1,94 +1,94 b''
 #!/usr/bin/env python
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 from __future__ import print_function
 import sys
 import io
 import os
 
 from converters.template import *
 from converters.template import ConverterTemplate
 from converters.html import ConverterHTML
 # From IPython
 
 # All the stuff needed for the configurable things
 from IPython.config.application import Application
 from IPython.config.loader import ConfigFileNotFound
 from IPython.utils.traitlets import List, Unicode, Type, Bool, Dict, CaselessStrEnum
 
+from converters.transformers import (ConfigurableTransformers,Foobar,ExtractFigureTransformer)
+
 
 class NbconvertApp(Application):
 
 
     def __init__(self, **kwargs):
         super(NbconvertApp, self).__init__(**kwargs)
         self.classes.insert(0,ConverterTemplate)
+        # register class here to have help with help all
+        self.classes.insert(0,ExtractFigureTransformer)
+        self.classes.insert(0,Foobar)
         # ensure those are registerd
 
     def load_config_file(self, profile_name):
         try:
             Application.load_config_file(
                 self,
                 profile_name+'.nbcv',
                 path=[os.path.join(os.getcwdu(),'profile')]
             )
         except ConfigFileNotFound:
             self.log.warn("Config file for profile '%s' not found, giving up ",profile_name)
             exit(1)
 
 
     def initialize(self, argv=None):
         self.parse_command_line(argv)
         cl_config = self.config
         profile_file = sys.argv[1]
         self.load_config_file(profile_file)
         self.update_config(cl_config)
 
 
 
     def run(self):
         """Convert a notebook to html in one step"""
         template_file = (self.extra_args or [None])[0]
         ipynb_file = (self.extra_args or [None])[1]
 
         template_file = sys.argv[1]
 
-        if template_file.startswith('latex'):
-            tex_environement=True
-        else:
-            tex_environement=False
-
         C = ConverterTemplate(tplfile=sys.argv[1],
                 config=self.config)
         C.read(ipynb_file)
 
         output,resources = C.convert()
 
         print(output.encode('utf-8'))
 
         keys = resources.keys()
         if keys :
             print('''
 ====================== Keys in Resources ==================================
 ''')
             print(resources.keys())
             print("""
 ===========================================================================
 you are responsible from writing those data do a file in the right place if
 they need to be.
 ===========================================================================
                   """)
 
 def main():
     """Convert a notebook to html in one step"""
     app = NbconvertApp.instance()
     app.initialize()
     app.start()
     app.run()
 #-----------------------------------------------------------------------------
 # Script main
 #-----------------------------------------------------------------------------
 
 if __name__ == '__main__':
     main()

templates/tex/latex_base.tplx

@@ -1,207 +1,214 b''
 ((*- extends 'display_priority.tplx' -*))
 
 ((* block in_prompt *))((* endblock in_prompt *))
 
 ((* block output_prompt *))((* endblock output_prompt *))
 
 ((* block codecell *))\begin{codecell}((( super() )))
 \end{codecell}
 ((* endblock *))
 
 ((* block input *))
 \begin{codeinput}
 \begin{lstlisting}
 ((( cell.input )))
 \end{lstlisting}
 \end{codeinput}
 ((* endblock input *))
 
 
 ((= Those Two are for error displaying
 even if the first one seem to do nothing, 
 it introduces a new line
 
 =))
 ((* block pyerr *))((( super() )))
 ((* endblock pyerr *))
 
 ((* block traceback_line *))
 ((( line |indent| rm_ansi )))((* endblock traceback_line *))
 ((= .... =))
 
 
 ((*- block output_group -*))
 \begin{codeoutput}
 ((( super() )))
 \end{codeoutput}((* endblock *))
 
 ((*- block data_png -*))
 \begin{center}
-\includegraphics[width=0.7\textwidth]{(((output.png)))}
+\includegraphics[width=0.7\textwidth]{(((output.key_png)))}
+\par
+\end{center}
+((*- endblock -*))
+
+((*- block data_svg -*))
+\begin{center}
+\includegraphics[width=0.7\textwidth]{(((output.key_svg)))}
 \par
 \end{center}
 ((*- endblock -*))
 
 ((* block pyout *))
 ((( output.text)))
 ((* endblock pyout *))
 
 ((* block data_text *))
 \begin{verbatim}
 ((( output.text )))
 \end{verbatim}
 ((* endblock *))
 
 ((* block stream *))
 \begin{verbatim}
 ((( output.text)))
 \end{verbatim}
 ((* endblock stream *))
 
 
 
 
 ((* block markdowncell scoped *))((( cell.source | markdown2latex )))
 ((* endblock markdowncell *))
 
 ((* block headingcell scoped *))
 \section{((( cell.source)))}
 ((* endblock headingcell *))
 
 ((* block rawcell scoped *))
 ((( cell.source | pycomment )))
 ((* endblock rawcell *))
 
 ((* block unknowncell scoped *))
 unknown type  (((cell.type)))
 ((* endblock unknowncell *))
 
 
 
 ((* block body *))\begin{document}
 ((( super() )))
 \end{document}
 ((* endblock*))
 
 ((* block header *))
 %% This file was auto-generated by IPython.
 %% Conversion from the original notebook file:
 %% tests/ipynbref/Gun_Data.orig.ipynb
 %%
 \documentclass[11pt,english]{article}
 
 %% This is the automatic preamble used by IPython.  Note that it does *not*
 %% include a documentclass declaration, that is added at runtime to the overall
 %% document.
 
 \usepackage{amsmath}
 \usepackage{amssymb}
 \usepackage{graphicx}
 \usepackage{ucs}
 \usepackage[utf8x]{inputenc}
 
 % needed for markdown enumerations to work
 \usepackage{enumerate}
 
 % Slightly bigger margins than the latex defaults
 \usepackage{geometry}
 \geometry{verbose,tmargin=3cm,bmargin=3cm,lmargin=2.5cm,rmargin=2.5cm}
 
 % Define a few colors for use in code, links and cell shading
 \usepackage{color}
 \definecolor{orange}{cmyk}{0,0.4,0.8,0.2}
 \definecolor{darkorange}{rgb}{.71,0.21,0.01}
 \definecolor{darkgreen}{rgb}{.12,.54,.11}
 \definecolor{myteal}{rgb}{.26, .44, .56}
 \definecolor{gray}{gray}{0.45}
 \definecolor{lightgray}{gray}{.95}
 \definecolor{mediumgray}{gray}{.8}
 \definecolor{inputbackground}{rgb}{.95, .95, .85}
 \definecolor{outputbackground}{rgb}{.95, .95, .95}
 \definecolor{traceback}{rgb}{1, .95, .95}
 
 % Framed environments for code cells (inputs, outputs, errors, ...).  The
 % various uses of \unskip (or not) at the end were fine-tuned by hand, so don't
 % randomly change them unless you're sure of the effect it will have.
 \usepackage{framed}
 
 % remove extraneous vertical space in boxes
 \setlength\fboxsep{0pt}
 
 % codecell is the whole input+output set of blocks that a Code cell can
 % generate.
 
 % TODO: unfortunately, it seems that using a framed codecell environment breaks
 % the ability of the frames inside of it to be broken across pages.  This
 % causes at least the problem of having lots of empty space at the bottom of
 % pages as new frames are moved to the next page, and if a single frame is too
 % long to fit on a page, will completely stop latex from compiling the
 % document.  So unless we figure out a solution to this, we'll have to instead
 % leave the codecell env. as empty.  I'm keeping the original codecell
 % definition here (a thin vertical bar) for reference, in case we find a
 % solution to the page break issue.
 
 %% \newenvironment{codecell}{%
 %%     \def\FrameCommand{\color{mediumgray} \vrule width 1pt \hspace{5pt}}%
 %%    \MakeFramed{\vspace{-0.5em}}}
 %%  {\unskip\endMakeFramed}
 
 % For now, make this a no-op...
 \newenvironment{codecell}{}
 
  \newenvironment{codeinput}{%
    \def\FrameCommand{\colorbox{inputbackground}}%
    \MakeFramed{\advance\hsize-\width \FrameRestore}}
  {\unskip\endMakeFramed}
 
 \newenvironment{codeoutput}{%
    \def\FrameCommand{\colorbox{outputbackground}}%
    \vspace{-1.4em}
    \MakeFramed{\advance\hsize-\width \FrameRestore}}
  {\unskip\medskip\endMakeFramed}
 
 \newenvironment{traceback}{%
    \def\FrameCommand{\colorbox{traceback}}%
    \MakeFramed{\advance\hsize-\width \FrameRestore}}
  {\endMakeFramed}
 
 % Use and configure listings package for nicely formatted code
 \usepackage{listingsutf8}
 \lstset{
   language=python,
   inputencoding=utf8x,
   extendedchars=\true,
   aboveskip=\smallskipamount,
   belowskip=\smallskipamount,
   xleftmargin=2mm,
   breaklines=true,
   basicstyle=\small \ttfamily,
   showstringspaces=false,
   keywordstyle=\color{blue}\bfseries,
   commentstyle=\color{myteal},
   stringstyle=\color{darkgreen},
   identifierstyle=\color{darkorange},
   columns=fullflexible,  % tighter character kerning, like verb
 }
 
 % The hyperref package gives us a pdf with properly built
 % internal navigation ('pdf bookmarks' for the table of contents,
 % internal cross-reference links, web links for URLs, etc.)
 \usepackage{hyperref}
 \hypersetup{
   breaklinks=true,  % so long urls are correctly broken across lines
   colorlinks=true,
   urlcolor=blue,
   linkcolor=darkorange,
   citecolor=darkgreen,
   }
 
 % hardcode size of all verbatim environments to be a bit smaller
 \makeatletter 
 \g@addto@macro\@verbatim\small\topsep=0.5em\partopsep=0pt
 \makeatother 
 
 % Prevent overflowing lines due to urls and other hard-to-break entities.
 \sloppy
 
 ((* endblock *))