allow configurable filters

converters/jinja_filters.py

@@ -1,42 +1,88 b''
 #-----------------------------------------------------------------------------
 # Copyright (c) 2012, the IPython Development Team.
 #
 # Distributed under the terms of the Modified BSD License.
 #
 # The full license is in the file COPYING.txt, distributed with this software.
 #-----------------------------------------------------------------------------
 
 from __future__ import absolute_import
 
 # Stdlib imports
 import re
 
 from IPython.utils.text import indent
 from markdown import markdown
 from .utils import remove_ansi
 from .utils import highlight, ansi2html
 from .utils import markdown2latex
+
+from IPython.config.configurable import Configurable
+from IPython.utils.traitlets import Unicode, Bool, Dict, List
+
 #-----------------------------------------------------------------------------
 # Class declarations
 #-----------------------------------------------------------------------------
+
+class GlobalConfigurable(Configurable):
+    """Global configurable class for shared config
+
+    Usefull for display data priority that might be use by many trasnformers
+    """
+    
+    display_data_priority = List(['html', 'pdf', 'svg', 'latex', 'png', 'jpg', 'jpeg' , 'text'],
+            config=True,
+              help= """
+                    An ordered list of prefered output type, the first
+                    encounterd will usually be used when converting discarding
+                    the others.
+                    """
+            )
+
+    def __init__(self, config=None, **kw):
+        super(GlobalConfigurable, self).__init__( config=config, **kw)
+
+class ConfigurableFilter(GlobalConfigurable):
+    """Configurable Jinja Filter"""
+
+    def __init__(self, config=None, **kw):
+        super(ConfigurableFilter, self).__init__(config=config, **kw)
+
+    def __call__(self, *args, **kwargs):
+        raise NotImplementedError('should be implemented by subclass')
+
+
+class FilterDataType(ConfigurableFilter):
+    """ return the preferd displayed format
+    """
+
+    def __call__(self, output):
+        """ return the first availlable format in priority """
+        for fmt in self.display_data_priority:
+            if fmt in output:
+                return [fmt]
+        raise Exception("did not found any format I can extract in output, shoudl at lest have one")
+
+
+
 def rm_fake(strng):
     return strng.replace('/files/', '')
 
 def python_comment(string):
     return '# '+'\n# '.join(string.split('\n'))
 
 LATEX_SUBS = (
     (re.compile(r'\\'), r'\\textbackslash'),
     (re.compile(r'([{}_#%&$])'), r'\\\1'),
     (re.compile(r'~'), r'\~{}'),
     (re.compile(r'\^'), r'\^{}'),
     (re.compile(r'"'), r"''"),
     (re.compile(r'\.\.\.+'), r'\\ldots'),
 )
 
 def escape_tex(value):
     newval = value
     for pattern, replacement in LATEX_SUBS:
         newval = pattern.sub(replacement, newval)
     return newval
 

converters/template.py

@@ -1,242 +1,230 b''
 """Base classes for the notebook conversion pipeline.
 
 This module defines Converter, from which all objects designed to implement
 a conversion of IPython notebooks to some other format should inherit.
 """
 #-----------------------------------------------------------------------------
 # Copyright (c) 2012, the IPython Development Team.
 #
 # Distributed under the terms of the Modified BSD License.
 #
 # The full license is in the file COPYING.txt, distributed with this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 from __future__ import print_function, absolute_import
 import converters.transformers as trans
 from converters.jinja_filters import (python_comment, indent,
         rm_fake, remove_ansi, markdown, highlight,
-        ansi2html, markdown2latex, escape_tex)
+        ansi2html, markdown2latex, escape_tex, FilterDataType)
 
 from converters.utils import  markdown2rst
 
 
 
 # Stdlib imports
 import io
 import os
 from IPython.utils import path
 from IPython.utils.traitlets import MetaHasTraits
 
 from jinja2 import Environment, FileSystemLoader
 env = Environment(
         loader=FileSystemLoader([
             './templates/',
             './templates/skeleton/',
             ]),
         extensions=['jinja2.ext.loopcontrols']
         )
 
 texenv = Environment(
         loader=FileSystemLoader([
             './templates/tex/',
             './templates/skeleton/tex/',
             ]),
         extensions=['jinja2.ext.loopcontrols']
         )
 
 # IPython imports
 from IPython.nbformat import current as nbformat
 from IPython.config.configurable import Configurable
 from IPython.utils.traitlets import ( Unicode, Any, List, Bool)
 
 #-----------------------------------------------------------------------------
 # Class declarations
 #-----------------------------------------------------------------------------
 class ConversionException(Exception):
     pass
 
 #todo move this out
 def header_body():
     """Return the body of the header as a list of strings."""
 
     from pygments.formatters import HtmlFormatter
 
     header = []
     static = os.path.join(path.get_ipython_package_dir(),
     'frontend', 'html', 'notebook', 'static',
     )
     here = os.path.split(os.path.realpath(__file__))[0]
     css = os.path.join(static, 'css')
     for sheet in [
         # do we need jquery and prettify?
         # os.path.join(static, 'jquery', 'css', 'themes', 'base',
         # 'jquery-ui.min.css'),
         # os.path.join(static, 'prettify', 'prettify.css'),
         os.path.join(css, 'boilerplate.css'),
         os.path.join(css, 'fbm.css'),
         os.path.join(css, 'notebook.css'),
         os.path.join(css, 'renderedhtml.css'),
+        os.path.join(css, 'style.min.css'),
         # our overrides:
         os.path.join(here, '..', 'css', 'static_html.css'),
     ]:
-
+        try:
         with io.open(sheet, encoding='utf-8') as f:
             s = f.read()
             header.append(s)
+        except IOError:
+            # new version of ipython with style.min.css, pass
+            pass
 
     pygments_css = HtmlFormatter().get_style_defs('.highlight')
     header.append(pygments_css)
     return header
 
 
 
 
 
 inlining = {}
 inlining['css'] = header_body()
 
 
 
 texenv.block_start_string = '((*'
 texenv.block_end_string = '*))'
+
 texenv.variable_start_string = '((('
 texenv.variable_end_string = ')))'
+
 texenv.comment_start_string = '((='
 texenv.comment_end_string = '=))'
+
 texenv.filters['escape_tex'] = escape_tex
 
 
 class ConverterTemplate(Configurable):
     """ A Jinja2 base converter templates
 
     Preprocess the ipynb files, feed it throug jinja templates,
     and spit an converted files and a data object with other data
 
     shoudl be mostly configurable
     """
 
-    display_data_priority = List(['html', 'pdf', 'svg', 'latex', 'png', 'jpg', 'jpeg' , 'text'],
-            config=True,
-              help= """
-                    An ordered list of prefered output type, the first
-                    encounterd will usually be used when converting discarding
-                    the others.
-                    """
-            )
-
     pre_transformer_order = List(['haspyout_transformer'],
             config=True,
               help= """
                     An ordered list of pre transformer to apply to the ipynb
                     file befor running through templates
                     """
             )
 
-    extract_figures = Bool(False,
-            config=True,
-              help= """
-                    wether to remove figure data from ipynb and store them in auxiliary
-                    dictionnary
-                    """
-            )
-
     tex_environement = Bool(False,
             config=True,
             help=""" is this a tex environment or not """)
 
     template_file = Unicode('',
             config=True,
             help=""" Name of the template file to use """ )
     #-------------------------------------------------------------------------
     # Instance-level attributes that are set in the constructor for this
     # class.
     #-------------------------------------------------------------------------
-    infile = Any()
-
-
-    infile_dir = Unicode()
 
-    #todo: move to filter
-    def filter_data_type(self, output):
-        """ return the first availlable format in priority """
-        for fmt in self.display_data_priority:
-            if fmt in output:
-                return [fmt]
-        raise Exception("did not found any format I can extract in output, shoudl at lest have one")
 
     preprocessors = []
 
     def __init__(self, preprocessors={}, jinja_filters={}, config=None, **kw):
-        """
-        config: the Configurable confg object to pass around
+        """ Init a new converter.
+
+
+        config: the Configurable confgg object to pass around
 
-        preprocessors: dict of **availlable** key/value function to run on ipynb json data before conversion
-        to extract/inline file,
+        preprocessors: dict of **availlable** key/value function to run on
+        ipynb json data before conversion to extract/inline file,
+
+        jinja_filter : dict of supplementary jinja filter that should be made
+        availlable in template. If those are of Configurable Class type, they
+        will be instanciated with the config object as argument.
 
         """
         super(ConverterTemplate, self).__init__(config=config, **kw)
         self.env = texenv  if self.tex_environement else env
         self.ext = '.tplx' if self.tex_environement else '.tpl'
 
         for name in self.pre_transformer_order:
             transformer = getattr(preprocessors, name, getattr(trans, name, None))
             if isinstance(transformer, MetaHasTraits):
                 transformer = transformer(config=config)
             self.preprocessors.append(transformer)
 
         ## for compat, remove later
-        if self.extract_figures:
             self.preprocessors.append(trans.ExtractFigureTransformer(config=config))
         self.preprocessors.append(trans.RevealHelpTransformer(config=config))
 
         ##
-        self.env.filters['filter_data_type'] = self.filter_data_type
+        self.env.filters['filter_data_type'] = FilterDataType(config=config)
         self.env.filters['pycomment'] = python_comment
         self.env.filters['indent'] = indent
         self.env.filters['rm_fake'] = rm_fake
         self.env.filters['rm_ansi'] = remove_ansi
         self.env.filters['markdown'] = markdown
         self.env.filters['highlight'] = highlight
         self.env.filters['ansi2html'] = ansi2html
         self.env.filters['markdown2latex'] = markdown2latex
         self.env.filters['markdown2rst'] = markdown2rst
-        for k, v in jinja_filters.iteritems():
-            self.env.filters[k] = v
+        for key, filtr in jinja_filters.iteritems():
+            if isinstance(filtr, MetaHasTraits):
+                self.env.filters[key] = filtr(config=config)
+            else :
+                self.env.filters[key] = filtr
 
         self.template = self.env.get_template(self.template_file+self.ext)
 
 
     def process(self, nb):
         """
         preprocess the notebook json for easier use with the templates.
         will call all the `preprocessor`s in order before returning it.
         """
 
         # dict of 'resources' that could be made by the preprocessors
         # like key/value data to extract files from ipynb like in latex conversion
         resources = {}
 
         for preprocessor in self.preprocessors:
             nb, resources = preprocessor(nb, resources)
 
         return nb, resources
 
     def convert(self, nb):
         """ convert the ipynb file
 
         return both the converted ipynb file and a dict containing potential
         other resources
         """
         nb, resources = self.process(nb)
         return self.template.render(nb=nb, inlining=inlining), resources
 
 
     def from_filename(self, filename):
         "read and parse notebook into NotebookNode called self.nb"
         with io.open(filename) as f:
             return self.convert(nbformat.read(f, 'json'))
 
+

converters/transformers.py

@@ -1,233 +1,225 b''
 """
 
 """
 
-from __future__ import print_function
+from __future__ import print_function, absolute_import
 
 from IPython.config.configurable import Configurable
 from IPython.utils.traitlets import Unicode, Bool, Dict, List
+from .jinja_filters import GlobalConfigurable
 
-class ConfigurableTransformers(Configurable):
+class ConfigurableTransformers(GlobalConfigurable):
     """ A configurable transformer """
 
     def __init__(self, config=None, **kw):
         super(ConfigurableTransformers, self).__init__(config=config, **kw)
 
     def __call__(self, nb, other):
         try :
             for worksheet in nb.worksheets :
                 for index, cell in enumerate(worksheet.cells):
                     worksheet.cells[index], other = self.cell_transform(cell, other, index)
             return nb, other
         except NotImplementedError as error :
             raise NotImplementedError('should be implemented by subclass')
 
     def cell_transform(self, cell, other, index):
         """
         Overwrite if you want to apply a transformation on each cell
         """
         raise NotImplementedError('should be implemented by subclass')
 
 
 class ActivatableTransformer(ConfigurableTransformers):
 
     enabled = Bool(False, config=True)
 
     def __call__(self, nb, other):
         if not self.enabled :
             return nb,other
         else :
             return super(ActivatableTransformer,self).__call__(nb, other)
 
 
 def cell_preprocessor(function):
     """ wrap a function to be executed on all cells of a notebook
 
     wrapped function  parameters :
     cell  : the cell
     other : external resources
     index : index of the cell
     """
     def wrappedfunc(nb, other):
         for worksheet in nb.worksheets :
             for index, cell in enumerate(worksheet.cells):
                 worksheet.cells[index], other = function(cell, other, index)
         return nb, other
     return wrappedfunc
 
 
 @cell_preprocessor
 def haspyout_transformer(cell, other, count):
     """
     Add a haspyout flag to cell that have it
 
     Easier for templating, where you can't know in advance
     wether to write the out prompt
 
     """
     cell.type = cell.cell_type
     cell.haspyout = False
     for out in cell.get('outputs', []):
         if out.output_type == 'pyout':
             cell.haspyout = True
             break
     return cell, other
 
 
 # todo, make the key part configurable.
 
 class ExtractFigureTransformer(ActivatableTransformer):
 
     extra_ext_map =  Dict({},
             config=True,
             help="""extra map to override extension based on type.
             Usefull for latex where svg will be converted to pdf before inclusion
             """
             )
-    display_data_priority = List(['html', 'pdf', 'svg', 'latex', 'png', 'jpg', 'jpeg' , 'text'],
-            config=True,
-              help= """
-                    An ordered list of prefered output type, the first
-                    encounterd will usually be used when converting discarding
-                    the others.
-                    """
-            )
-
 
     #to do change this to .format {} syntax
     key_tpl = Unicode('_fig_%02i.%s', config=True)
 
     def _get_ext(self, ext):
         if ext in self.extra_ext_map :
             return self.extra_ext_map[ext]
         return ext
 
     def _new_figure(self, data, fmt, count):
         """Create a new figure file in the given format.
 
         Returns a path relative to the input file.
         """
         figname = self.key_tpl % (count, self._get_ext(fmt))
         key     = self.key_tpl % (count, fmt)
 
         # Binary files are base64-encoded, SVG is already XML
         if fmt in ('png', 'jpg', 'pdf'):
             data = data.decode('base64')
 
         return figname, key, data
 
 
     def cell_transform(self, cell, other, count):
         if other.get('figures',None) is None :
             other['figures']={}
         for i, out in enumerate(cell.get('outputs', [])):
             for type in self.display_data_priority:
                 if out.hasattr(type):
                     figname, key, data = self._new_figure(out[type], type, count)
                     out['key_'+type] = figname
                     other['figures'][key] = data
                     count = count+1
         return cell, other
 
 class RevealHelpTransformer(ConfigurableTransformers):
 
     section_open = False
     subsection_open = False
     fragment_open = False
 
     def open_subsection(self):
         self.subsection_open = True
         return True
 
     def open_section(self):
         self.section_open = True
         return True
 
     def open_fragment(self):
         self.fragment_open = True
         return True
 
     # could probaly write those maybe_close/open
     # with a function functor
     def maybe_close_section(self):
         """return True is already open, false otherwise
         and change state to close
         """
         if self.section_open :
             self.section_open = False
             return True
         else :
             return False
 
     def maybe_open_section(self):
         """return True is already open, false otherwise
         and change state to close
         """
         if not self.section_open :
             self.section_open = True
             return True
         else :
             return False
 
     def maybe_open_subsection(self):
         """return True is already open, false otherwise
         and change state to close
         """
         if not self.subsection_open :
             self.subsection_open = True
             return True
         else :
             return False
 
     def maybe_close_subsection(self):
         """return True is already open, false otherwise
         and change state to close
         """
         if self.subsection_open :
             self.subsection_open = False
             return True
         else :
             return False
 
     def maybe_close_fragment(self):
         """return True is already open, false otherwise
         and change state to close
         """
         if self.fragment_open :
             self.fragment_open = False
             return True
         else :
             return False
 
     def cell_transform(self, cell, other,count):
         ctype = cell.metadata.get('slideshow',{}).get('slide_type',None)
         if ctype in [None, '-'] :
             cell.metadata.slideshow = {}
             cell.metadata.slideshow['slide_type'] = None
         elif ctype == 'fragment':
             cell.metadata.slideshow.close_fragment = self.maybe_close_fragment()
             cell.metadata.slideshow.close_subsection = False
             cell.metadata.slideshow.close_section = False
 
             cell.metadata.slideshow.open_section = self.maybe_open_section()
             cell.metadata.slideshow.open_subsection = self.maybe_open_subsection()
             cell.metadata.slideshow.open_fragment = self.open_fragment()
 
         elif ctype == 'subslide':
             cell.metadata.slideshow.close_fragment = self.maybe_close_fragment()
             cell.metadata.slideshow.close_subsection = self.maybe_close_subsection()
             cell.metadata.slideshow.close_section = False
 
             cell.metadata.slideshow.open_section = self.maybe_open_section()
             cell.metadata.slideshow.open_subsection = self.open_subsection()
             cell.metadata.slideshow.open_fragment = False
         elif ctype == 'slide':
             cell.metadata.slideshow.close_fragment = self.maybe_close_fragment()
             cell.metadata.slideshow.close_subsection = self.maybe_close_subsection()
             cell.metadata.slideshow.close_section = self.maybe_close_section()
 
             cell.metadata.slideshow.open_section = self.open_section()
             cell.metadata.slideshow.open_subsection = self.open_subsection()
             cell.metadata.slideshow.open_fragment = False
         return cell,other
 

runme.py

@@ -1,135 +1,138 b''
 #!/usr/bin/env python
 """
 ================================================================================
 
 |,---.     |    |                 ,   .|                                   |
 ||---',   .|--- |---.,---.,---.   |\  ||---.,---.,---.,---..    ,,---.,---.|---
 ||    |   ||    |   ||   ||   |   | \ ||   ||    |   ||   | \  / |---'|    |
 ``    `---|`---'`   '`---'`   '   `  `'`---'`---'`---'`   '  `'  `---'`    `---'
       `---'
 ================================================================================
 
 Highly experimental for now
 
 """
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 from __future__ import print_function
 import sys
 import io
 import os
 
 from converters.template import *
 from converters.template import ConverterTemplate
 from converters.html import ConverterHTML
 # From IPython
 
 # All the stuff needed for the configurable things
 from IPython.config.application import Application
 from IPython.config.loader import ConfigFileNotFound
 from IPython.utils.traitlets import List, Unicode, Type, Bool, Dict, CaselessStrEnum
 
 from converters.transformers import (ConfigurableTransformers,ExtractFigureTransformer)
 
+from converters.jinja_filters import GlobalConfigurable
+
 
 class NbconvertApp(Application):
 
     stdout = Bool(True, config=True)
     write = Bool(False, config=True)
 
     fileext = Unicode('txt', config=True)
 
     aliases = {
             'stdout':'NbconvertApp.stdout',
             'write':'NbconvertApp.write',
             }
 
     flags= {}
     flags['no-stdout'] = (
         {'NbconvertApp' : {'stdout' : False}},
     """the doc for this flag
 
     """
     )
 
     def __init__(self, **kwargs):
         super(NbconvertApp, self).__init__(**kwargs)
         self.classes.insert(0,ConverterTemplate)
         # register class here to have help with help all
         self.classes.insert(0,ExtractFigureTransformer)
+        self.classes.insert(0,GlobalConfigurable)
         # ensure those are registerd
 
     def load_config_file(self, profile_name):
         try:
             Application.load_config_file(
                 self,
                 profile_name+'.nbcv',
                 path=[os.path.join(os.getcwdu(),'profile')]
             )
         except ConfigFileNotFound:
             self.log.warn("Config file for profile '%s' not found, giving up ",profile_name)
             exit(1)
 
 
     def initialize(self, argv=None):
         self.parse_command_line(argv)
         cl_config = self.config
         profile_file = sys.argv[1]
         self.load_config_file(profile_file)
         self.update_config(cl_config)
 
 
 
     def run(self):
         """Convert a notebook to html in one step"""
         template_file = (self.extra_args or [None])[0]
         ipynb_file = (self.extra_args or [None])[1]
 
         template_file = sys.argv[1]
 
         C = ConverterTemplate(config=self.config)
 
         output,resources = C.from_filename(ipynb_file)
         if self.stdout :
             print(output.encode('utf-8'))
 
         out_root = ipynb_file[:-6].replace('.','_').replace(' ','_')
 
         keys = resources.get('figures',{}).keys()
         if self.write :
             with io.open(os.path.join(out_root+'.'+self.fileext),'w') as f:
                         f.write(output)
         if keys :
             if self.write and not os.path.exists(out_root+'_files'):
                 os.mkdir(out_root+'_files')
             for key in keys:
                 if self.write:
                     with io.open(os.path.join(out_root+'_files',key),'wb') as f:
                         print(' writing to ',os.path.join(out_root,key))
                         f.write(resources['figures'][key])
             if self.stdout:
                 print('''
 ====================== Keys in Resources ==================================
 ''')
                 print(resources['figures'].keys())
                 print("""
 ===========================================================================
 you are responsible from writing those data do a file in the right place if
 they need to be.
 ===========================================================================
                   """)
 
 def main():
     """Convert a notebook to html in one step"""
     app = NbconvertApp.instance()
     app.description = __doc__
     app.initialize()
     app.start()
     app.run()
 #-----------------------------------------------------------------------------
 # Script main
 #-----------------------------------------------------------------------------
 
 if __name__ == '__main__':
     main()