create rst converter and fix some bugs

converters/template.py

@@ -1,240 +1,241 b''
 """Base classes for the notebook conversion pipeline.
 
 This module defines Converter, from which all objects designed to implement
 a conversion of IPython notebooks to some other format should inherit.
 """
 #-----------------------------------------------------------------------------
 # Copyright (c) 2012, the IPython Development Team.
 #
 # Distributed under the terms of the Modified BSD License.
 #
 # The full license is in the file COPYING.txt, distributed with this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 from __future__ import print_function, absolute_import
 import converters.transformers as trans
 from converters.jinja_filters import (python_comment, indent,
         rm_fake, remove_ansi, markdown, highlight,
         ansi2html, markdown2latex, escape_tex)
 
 from converters.utils import  markdown2rst
 
 
 
 # Stdlib imports
 import io
 import os
 from IPython.utils import path
 from IPython.utils.traitlets import MetaHasTraits
 
 from jinja2 import Environment, FileSystemLoader
 env = Environment(
         loader=FileSystemLoader([
             './templates/',
             './templates/skeleton/',
             ]),
         extensions=['jinja2.ext.loopcontrols']
         )
 
 texenv = Environment(
         loader=FileSystemLoader([
             './templates/tex/',
             './templates/skeleton/tex/',
             ]),
         extensions=['jinja2.ext.loopcontrols']
         )
 
 # IPython imports
 from IPython.nbformat import current as nbformat
 from IPython.config.configurable import Configurable
 from IPython.utils.traitlets import ( Unicode, Any, List, Bool)
 
 #-----------------------------------------------------------------------------
 # Class declarations
 #-----------------------------------------------------------------------------
 class ConversionException(Exception):
     pass
 
 #todo move this out
 def header_body():
     """Return the body of the header as a list of strings."""
 
     from pygments.formatters import HtmlFormatter
 
     header = []
     static = os.path.join(path.get_ipython_package_dir(),
     'frontend', 'html', 'notebook', 'static',
     )
     here = os.path.split(os.path.realpath(__file__))[0]
     css = os.path.join(static, 'css')
     for sheet in [
         # do we need jquery and prettify?
         # os.path.join(static, 'jquery', 'css', 'themes', 'base',
         # 'jquery-ui.min.css'),
         # os.path.join(static, 'prettify', 'prettify.css'),
         os.path.join(css, 'boilerplate.css'),
         os.path.join(css, 'fbm.css'),
         os.path.join(css, 'notebook.css'),
         os.path.join(css, 'renderedhtml.css'),
         # our overrides:
         os.path.join(here, '..', 'css', 'static_html.css'),
     ]:
 
         with io.open(sheet, encoding='utf-8') as f:
             s = f.read()
             header.append(s)
 
     pygments_css = HtmlFormatter().get_style_defs('.highlight')
     header.append(pygments_css)
     return header
 
 
 
 
 
 inlining = {}
 inlining['css'] = header_body()
 
 
 
 texenv.block_start_string = '((*'
 texenv.block_end_string = '*))'
 texenv.variable_start_string = '((('
 texenv.variable_end_string = ')))'
 texenv.comment_start_string = '((='
 texenv.comment_end_string = '=))'
 texenv.filters['escape_tex'] = escape_tex
 
 
 class ConverterTemplate(Configurable):
     """ A Jinja2 base converter templates
 
     Preprocess the ipynb files, feed it throug jinja templates,
     and spit an converted files and a data object with other data
 
     shoudl be mostly configurable
     """
 
     display_data_priority = List(['html', 'pdf', 'svg', 'latex', 'png', 'jpg', 'jpeg' , 'text'],
             config=True,
               help= """
                     An ordered list of prefered output type, the first
                     encounterd will usually be used when converting discarding
                     the others.
                     """
             )
 
     pre_transformer_order = List(['haspyout_transformer'],
             config=True,
               help= """
                     An ordered list of pre transformer to apply to the ipynb
                     file befor running through templates
                     """
             )
 
     extract_figures = Bool(False,
             config=True,
               help= """
                     wether to remove figure data from ipynb and store them in auxiliary
                     dictionnary
                     """
             )
 
     tex_environement = Bool(False,
             config=True,
             help=""" is this a tex environment or not """)
 
     template_file = Unicode('',
             config=True,
             help=""" Name of the template file to use """ )
     #-------------------------------------------------------------------------
     # Instance-level attributes that are set in the constructor for this
     # class.
     #-------------------------------------------------------------------------
     infile = Any()
 
 
     infile_dir = Unicode()
 
     #todo: move to filter
     def filter_data_type(self, output):
         """ return the first availlable format in priority """
         for fmt in self.display_data_priority:
             if fmt in output:
                 return [fmt]
+        raise Exception("did not found any format I can extract in output, shoudl at lest have one")
 
     preprocessors = []
 
     def __init__(self, preprocessors={}, jinja_filters={}, config=None, **kw):
         """
         config: the Configurable confg object to pass around
 
         preprocessors: dict of **availlable** key/value function to run on ipynb json data before conversion
         to extract/inline file,
 
         """
         super(ConverterTemplate, self).__init__(config=config, **kw)
         self.env = texenv  if self.tex_environement else env
         self.ext = '.tplx' if self.tex_environement else '.tpl'
 
         for name in self.pre_transformer_order:
             transformer = getattr(preprocessors, name, getattr(trans, name, None))
             if isinstance(transformer, MetaHasTraits):
                 transformer = transformer(config=config)
             self.preprocessors.append(transformer)
 
         ## for compat, remove later
         if self.extract_figures:
             self.preprocessors.append(trans.ExtractFigureTransformer(config=config))
 
         ##
         self.env.filters['filter_data_type'] = self.filter_data_type
         self.env.filters['pycomment'] = python_comment
         self.env.filters['indent'] = indent
         self.env.filters['rm_fake'] = rm_fake
         self.env.filters['rm_ansi'] = remove_ansi
         self.env.filters['markdown'] = markdown
         self.env.filters['highlight'] = highlight
         self.env.filters['ansi2html'] = ansi2html
         self.env.filters['markdown2latex'] = markdown2latex
         self.env.filters['markdown2rst'] = markdown2rst
         for k, v in jinja_filters.iteritems():
             self.env.filters[k] = v
 
         self.template = self.env.get_template(self.template_file+self.ext)
 
 
     def process(self, nb):
         """
         preprocess the notebook json for easier use with the templates.
         will call all the `preprocessor`s in order before returning it.
         """
 
         # dict of 'resources' that could be made by the preprocessors
         # like key/value data to extract files from ipynb like in latex conversion
         resources = {}
 
         for preprocessor in self.preprocessors:
             nb, resources = preprocessor(nb, resources)
 
         return nb, resources
 
     def convert(self, nb):
         """ convert the ipynb file
 
         return both the converted ipynb file and a dict containing potential
         other resources
         """
         nb, resources = self.process(nb)
         return self.template.render(nb=nb, inlining=inlining), resources
 
 
     def from_filename(self, filename):
         "read and parse notebook into NotebookNode called self.nb"
         with io.open(filename) as f:
             return self.convert(nbformat.read(f, 'json'))
 

converters/transformers.py

@@ -1,135 +1,134 b''
 """
 
 """
 
 from __future__ import print_function
 
 from IPython.config.configurable import Configurable
 from IPython.utils.traitlets import Unicode, Bool, Dict, List
 
 class ConfigurableTransformers(Configurable):
     """ A configurable transformer """
 
     def __init__(self, config=None, **kw):
         super(ConfigurableTransformers, self).__init__(config=config, **kw)
 
     def __call__(self, nb, other):
         try :
             for worksheet in nb.worksheets :
                 for index, cell in enumerate(worksheet.cells):
                     worksheet.cells[index], other = self.cell_transform(cell, other, index)
             return nb, other
         except NotImplementedError as error :
             raise NotImplementedError('should be implemented by subclass')
 
     def cell_transform(self, cell, other, index):
         """
         Overwrite if you want to apply a transformation on each cell
         """
         raise NotImplementedError('should be implemented by subclass')
 
 
 class Foobar(ConfigurableTransformers):
     message = Unicode('-- nothing', config=True)
 
 
     def cell_transform(self, cell, other, index):
         return cell, other
 
 
 def cell_preprocessor(function):
     """ wrap a function to be executed on all cells of a notebook
 
     wrapped function  parameters :
     cell  : the cell
     other : external resources
     index : index of the cell
     """
     def wrappedfunc(nb, other):
         for worksheet in nb.worksheets :
             for index, cell in enumerate(worksheet.cells):
                 worksheet.cells[index], other = function(cell, other, index)
         return nb, other
     return wrappedfunc
 
 
 @cell_preprocessor
 def haspyout_transformer(cell, other, count):
     """
     Add a haspyout flag to cell that have it
 
     Easier for templating, where you can't know in advance
     wether to write the out prompt
 
     """
     cell.type = cell.cell_type
     cell.haspyout = False
     for out in cell.get('outputs', []):
         if out.output_type == 'pyout':
             cell.haspyout = True
             break
     return cell, other
 
 
 # todo, make the key part configurable.
 
 class ExtractFigureTransformer(ConfigurableTransformers):
     enabled = Bool(False,
             config=True,
             help=""" If set to false, this transformer will be no-op """
             )
 
     extra_ext_map =  Dict({},
             config=True,
             help="""extra map to override extension based on type.
             Usefull for latex where svg will be converted to pdf before inclusion
             """
             )
     display_data_priority = List(['html', 'pdf', 'svg', 'latex', 'png', 'jpg', 'jpeg' , 'text'],
             config=True,
               help= """
                     An ordered list of prefered output type, the first
                     encounterd will usually be used when converting discarding
                     the others.
                     """
             )
 
 
     #to do change this to .format {} syntax
     key_tpl = Unicode('_fig_%02i.%s', config=True)
 
     def _get_ext(self, ext):
         if ext in self.extra_ext_map :
             return self.extra_ext_map[ext]
         return ext
 
     def _new_figure(self, data, fmt, count):
         """Create a new figure file in the given format.
 
         Returns a path relative to the input file.
         """
         figname = self.key_tpl % (count, self._get_ext(fmt))
         key     = self.key_tpl % (count, fmt)
 
         # Binary files are base64-encoded, SVG is already XML
         if fmt in ('png', 'jpg', 'pdf'):
             data = data.decode('base64')
 
         return figname, key, data
 
 
     def cell_transform(self, cell, other, count):
         if not self.enabled:
             return cell, other
         if other.get('figures',None) is None :
             other['figures']={}
         for i, out in enumerate(cell.get('outputs', [])):
             for type in self.display_data_priority:
                 if out.hasattr(type):
                     figname, key, data = self._new_figure(out[type], type, count)
-                    cell.outputs[i][type] = figname
                     out['key_'+type] = figname
                     other['figures'][key] = data
                     count = count+1
         return cell, other
 

profile/rst.nbcv

@@ -1,7 +1,13 b''
 c = get_config()
 
-c.ConverterTemplate.extract_figures=False
+c.ConverterTemplate.extract_figures=True
 c.ConverterTemplate.template_file='rst'
 c.ConverterTemplate.tex_environement=False
 
 c.NbconvertApp.fileext='rst'
+c.ExtractFigureTransformer.enabled=True
+
+c.ExtractFigureTransformer.display_data_priority=['svg', 'png', 'latex',  'jpg', 'jpeg','text']
+c.ConverterTemplate.display_data_priority=['svg', 'png', 'latex',  'jpg', 'jpeg','text']
+
+

templates/rst.tpl

@@ -1,69 +1,70 b''
-{%- extends 'null.tpl' -%}
-
+{%- extends 'display_priority.tpl' -%}
 {% block in_prompt -%}
 In[{{cell.prompt_number if cell.prompt_number else ' '}}]:
 
 .. code:: python
 
 {% endblock in_prompt %}
 
-{% block output_prompt %}
-Out[{{cell.prompt_number}}]:{% endblock output_prompt %}
+{% block output_prompt %}{% if cell.haspyout -%}
+Out[{{cell.prompt_number}}]:{% endif %}{% endblock output_prompt %}
 
 {% block input %}{{ cell.input | indent}}
-{% endblock input %}
-
 
-{# Those Two are for error displaying
-even if the first one seem to do nothing, 
-it introduces a new line
+{% endblock input %}
 
-#}
 {% block pyerr %}{{ super() }}
 {% endblock pyerr %}
 
 {% block traceback_line %}
 {{ line |indent| rm_ansi }}{% endblock traceback_line %}
-{# .... #}
-
 
 {% block pyout %}
 .. parsed-literal::
 
 {{ output.text| indent }}
 {% endblock pyout %}
 
 {% block stream %}
 .. parsed-literal::
 
 {{ output.text| indent }}
 {% endblock stream %}
 
 
 
 
-{% block display_data scoped %}
-# image file:
-{% endblock display_data %}
+{% block data_svg %}.. image:: {{output.key_svg}}
+
+{% endblock data_svg %}
 
-{% block markdowncell scoped %}
-{{ cell.source | markdown2rst }}
+{% block data_png %}.. image:: {{output.key_png}}
+
+{% endblock data_png %}
+
+{% block data_text scoped %}.. parsed-literal::
+
+{{output.text | indent}}
+
+{% endblock data_text %}
+
+{% block markdowncell scoped %}{{ cell.source | markdown2rst }}
 {% endblock markdowncell %}
 
 {% block headingcell scoped %}
 {%- set len = cell.source|length -%}
 {{ cell.source}}
 {% if cell.level == 1 %}
 {{- '=' * len }}
 {%- elif cell.level == 2 %}
 {{- '-' * len }}
 {% endif %}
+
 {% endblock headingcell %}
 
-{% block rawcell scoped %}
-{{ cell.source  }}
+{% block rawcell scoped %}{{ cell.source  }}
 {% endblock rawcell %}
 
 {% block unknowncell scoped %}
 unknown type  {{cell.type}}
 {% endblock unknowncell %}