stateless converter

converters/template.py

@@ -1,227 +1,237 b''
 """Base classes for the notebook conversion pipeline.
 
 This module defines Converter, from which all objects designed to implement
 a conversion of IPython notebooks to some other format should inherit.
 """
 #-----------------------------------------------------------------------------
 # Copyright (c) 2012, the IPython Development Team.
 #
 # Distributed under the terms of the Modified BSD License.
 #
 # The full license is in the file COPYING.txt, distributed with this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 
 from __future__ import print_function, absolute_import
 import converters.transformers as trans
 from converters.jinja_filters import (python_comment, indent,
         rm_fake, remove_ansi, markdown, highlight,
         ansi2html, markdown2latex, escape_tex)
 
 
 
 # Stdlib imports
 import io
 import os
 from IPython.utils import path
 from IPython.utils.traitlets import MetaHasTraits
 
 from jinja2 import Environment, FileSystemLoader
 env = Environment(
         loader=FileSystemLoader([
             './templates/',
             './templates/skeleton/',
             ]),
         extensions=['jinja2.ext.loopcontrols']
         )
 
 texenv = Environment(
         loader=FileSystemLoader([
             './templates/tex/',
             './templates/skeleton/tex/',
             ]),
         extensions=['jinja2.ext.loopcontrols']
         )
 
 # IPython imports
 from IPython.nbformat import current as nbformat
 from IPython.config.configurable import Configurable
 from IPython.utils.traitlets import ( Unicode, Any, List, Bool)
 
 #-----------------------------------------------------------------------------
 # Class declarations
 #-----------------------------------------------------------------------------
 class ConversionException(Exception):
     pass
 
-
+#todo move this out
 def header_body():
     """Return the body of the header as a list of strings."""
 
     from pygments.formatters import HtmlFormatter
 
     header = []
     static = os.path.join(path.get_ipython_package_dir(),
     'frontend', 'html', 'notebook', 'static',
     )
     here = os.path.split(os.path.realpath(__file__))[0]
     css = os.path.join(static, 'css')
     for sheet in [
         # do we need jquery and prettify?
         # os.path.join(static, 'jquery', 'css', 'themes', 'base',
         # 'jquery-ui.min.css'),
         # os.path.join(static, 'prettify', 'prettify.css'),
         os.path.join(css, 'boilerplate.css'),
         os.path.join(css, 'fbm.css'),
         os.path.join(css, 'notebook.css'),
         os.path.join(css, 'renderedhtml.css'),
         # our overrides:
         os.path.join(here, '..', 'css', 'static_html.css'),
     ]:
 
         with io.open(sheet, encoding='utf-8') as f:
             s = f.read()
             header.append(s)
 
     pygments_css = HtmlFormatter().get_style_defs('.highlight')
     header.append(pygments_css)
     return header
 
 
 
 
 
 inlining = {}
 inlining['css'] = header_body()
 
 
 
 texenv.block_start_string = '((*'
 texenv.block_end_string = '*))'
 texenv.variable_start_string = '((('
 texenv.variable_end_string = ')))'
 texenv.comment_start_string = '((='
 texenv.comment_end_string = '=))'
 texenv.filters['escape_tex'] = escape_tex
 
 
 class ConverterTemplate(Configurable):
-    """ A Jinja2 base converter templates"""
+    """ A Jinja2 base converter templates
+
+    Preprocess the ipynb files, feed it throug jinja templates,
+    and spit an converted files and a data object with other data
+
+    shoudl be mostly configurable
+    """
 
     display_data_priority = List(['html', 'pdf', 'svg', 'latex', 'png', 'jpg', 'jpeg' , 'text'],
             config=True,
               help= """
                     An ordered list of prefered output type, the first
                     encounterd will usually be used when converting discarding
                     the others.
                     """
             )
 
-    pre_transformer_order = List(['haspyout_transformer', 'Foobar'],
+    pre_transformer_order = List(['haspyout_transformer'],
             config=True,
               help= """
                     An ordered list of pre transformer to apply to the ipynb
                     file befor running through templates
                     """
             )
 
     extract_figures = Bool(False,
             config=True,
               help= """
                     wether to remove figure data from ipynb and store them in auxiliary
                     dictionnary
                     """
             )
 
     tex_environement = Bool(False,
             config=True,
             help=""" is this a tex environment or not """)
 
     template_file = Unicode('',
             config=True,
-            help=""" whetever """ )
+            help=""" Name of the template file to use """ )
     #-------------------------------------------------------------------------
     # Instance-level attributes that are set in the constructor for this
     # class.
     #-------------------------------------------------------------------------
     infile = Any()
 
 
     infile_dir = Unicode()
 
+    #todo: move to filter
     def filter_data_type(self, output):
+        """ return the first availlable format in priority """
         for fmt in self.display_data_priority:
             if fmt in output:
                 return [fmt]
 
-    def __init__(self, preprocessors=[], config=None, **kw):
+    preprocessors = []
+
+    def __init__(self, preprocessors={}, jinja_filters={}, config=None, **kw):
         """
         config: the Configurable confg object to pass around
 
-        preprocessors: list of function to run on ipynb json data before conversion
+        preprocessors: dict of **availlable** key/value function to run on ipynb json data before conversion
         to extract/inline file,
 
         """
         super(ConverterTemplate, self).__init__(config=config, **kw)
         self.env = texenv  if self.tex_environement else env
         self.ext = '.tplx' if self.tex_environement else '.tpl'
-        self.nb = None
-        self.preprocessors = preprocessors
 
         for name in self.pre_transformer_order:
-            tr = getattr(trans, name)
-            if isinstance(tr, MetaHasTraits):
-                tr = tr(config=config)
-            self.preprocessors.append(tr)
+            transformer = getattr(preprocessors, name, getattr(trans, name, None))
+            if isinstance(transformer, MetaHasTraits):
+                transformer = transformer(config=config)
+            self.preprocessors.append(transformer)
+
+        ## for compat, remove later
         if self.extract_figures:
             self.preprocessors.append(trans.ExtractFigureTransformer(config=config))
 
+        ##
         self.env.filters['filter_data_type'] = self.filter_data_type
         self.env.filters['pycomment'] = python_comment
         self.env.filters['indent'] = indent
         self.env.filters['rm_fake'] = rm_fake
         self.env.filters['rm_ansi'] = remove_ansi
         self.env.filters['markdown'] = markdown
         self.env.filters['highlight'] = highlight
         self.env.filters['ansi2html'] = ansi2html
         self.env.filters['markdown2latex'] = markdown2latex
+        for k, v in jinja_filters.iteritems():
+            self.env.filters[k] = v
 
         self.template = self.env.get_template(self.template_file+self.ext)
 
 
-    def process(self):
+    def process(self, nb):
         """
         preprocess the notebook json for easier use with the templates.
         will call all the `preprocessor`s in order before returning it.
         """
-        nb = self.nb
 
         # dict of 'resources' that could be made by the preprocessors
         # like key/value data to extract files from ipynb like in latex conversion
         resources = {}
 
         for preprocessor in self.preprocessors:
             nb, resources = preprocessor(nb, resources)
 
         return nb, resources
 
-    def convert(self):
+    def convert(self, nb):
         """ convert the ipynb file
 
         return both the converted ipynb file and a dict containing potential
         other resources
         """
-        nb, resources = self.process()
+        nb, resources = self.process(nb)
         return self.template.render(nb=nb, inlining=inlining), resources
 
 
-    def read(self, filename):
+    def from_filename(self, filename):
         "read and parse notebook into NotebookNode called self.nb"
         with io.open(filename) as f:
-            self.nb = nbformat.read(f, 'json')
-
-
+            return self.convert(nbformat.read(f, 'json'))
 

runme.py

@@ -1,137 +1,136 b''
 #!/usr/bin/env python
 """
 ================================================================================
 
 |,---.     |    |                 ,   .|                                   |
 ||---',   .|--- |---.,---.,---.   |\  ||---.,---.,---.,---..    ,,---.,---.|---
 ||    |   ||    |   ||   ||   |   | \ ||   ||    |   ||   | \  / |---'|    |
 ``    `---|`---'`   '`---'`   '   `  `'`---'`---'`---'`   '  `'  `---'`    `---'
       `---'
 ================================================================================
 
 Highly experimental for now
 
 """
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 from __future__ import print_function
 import sys
 import io
 import os
 
 from converters.template import *
 from converters.template import ConverterTemplate
 from converters.html import ConverterHTML
 # From IPython
 
 # All the stuff needed for the configurable things
 from IPython.config.application import Application
 from IPython.config.loader import ConfigFileNotFound
 from IPython.utils.traitlets import List, Unicode, Type, Bool, Dict, CaselessStrEnum
 
 from converters.transformers import (ConfigurableTransformers,Foobar,ExtractFigureTransformer)
 
 
 class NbconvertApp(Application):
 
     stdout = Bool(True, config=True)
     write = Bool(False, config=True)
 
     fileext = Unicode('txt', config=True)
 
     aliases = {
             'stdout':'NbconvertApp.stdout',
             'write':'NbconvertApp.write',
             }
 
     flags= {}
     flags['no-stdout'] = (
         {'NbconvertApp' : {'stdout' : False}},
     """the doc for this flag
 
     """
     )
 
     def __init__(self, **kwargs):
         super(NbconvertApp, self).__init__(**kwargs)
         self.classes.insert(0,ConverterTemplate)
         # register class here to have help with help all
         self.classes.insert(0,ExtractFigureTransformer)
         self.classes.insert(0,Foobar)
         # ensure those are registerd
 
     def load_config_file(self, profile_name):
         try:
             Application.load_config_file(
                 self,
                 profile_name+'.nbcv',
                 path=[os.path.join(os.getcwdu(),'profile')]
             )
         except ConfigFileNotFound:
             self.log.warn("Config file for profile '%s' not found, giving up ",profile_name)
             exit(1)
 
 
     def initialize(self, argv=None):
         self.parse_command_line(argv)
         cl_config = self.config
         profile_file = sys.argv[1]
         self.load_config_file(profile_file)
         self.update_config(cl_config)
 
 
 
     def run(self):
         """Convert a notebook to html in one step"""
         template_file = (self.extra_args or [None])[0]
         ipynb_file = (self.extra_args or [None])[1]
 
         template_file = sys.argv[1]
 
         C = ConverterTemplate(config=self.config)
-        C.read(ipynb_file)
 
-        output,resources = C.convert()
+        output,resources = C.from_filename(ipynb_file)
         if self.stdout :
             print(output.encode('utf-8'))
 
         out_root = ipynb_file[:-6].replace('.','_').replace(' ','_')
 
         keys = resources.get('figures',{}).keys()
         if self.write :
             with io.open(os.path.join(out_root+'.'+self.fileext),'w') as f:
                         f.write(output)
         if keys :
             if self.write and not os.path.exists(out_root+'_files'):
                 os.mkdir(out_root+'_files')
             for key in keys:
                 if self.write:
                     with io.open(os.path.join(out_root+'_files',key),'wb') as f:
                         print(' writing to ',os.path.join(out_root,key))
                         f.write(resources['figures'][key])
             if self.stdout:
                 print('''
 ====================== Keys in Resources ==================================
 ''')
                 print(resources['figures'].keys())
                 print("""
 ===========================================================================
 you are responsible from writing those data do a file in the right place if
 they need to be.
 ===========================================================================
                   """)
 
 def main():
     """Convert a notebook to html in one step"""
     app = NbconvertApp.instance()
     app.description = __doc__
     app.initialize()
     app.start()
     app.run()
 #-----------------------------------------------------------------------------
 # Script main
 #-----------------------------------------------------------------------------
 
 if __name__ == '__main__':
     main()

tests/test_templates.py

@@ -1,30 +1,29 b''
 import io
 import nose.tools as nt
 import os
 from nose.tools import nottest
 from converters.template import ConverterTemplate
 from  IPython.config.loader import PyFileConfigLoader 
 
 
 
 
 def test_evens():
     reflist = [
             'tests/ipynbref/IntroNumPy.orig'
             ]
 
     test_profiles = [prof for prof in os.listdir('profile/test/') if prof.endswith('.py')]
 
     ### null template should return empty
     for prof in test_profiles :
         yield check_null_profile,prof
     ### end null test
 
 def check_null_profile(profile):
     loader = PyFileConfigLoader(profile, path=[os.path.join(os.getcwdu(),'profile/test')])
     config = loader.load_config()
     C = ConverterTemplate(config=config)
-    C.read('tests/ipynbref/IntroNumPy.orig.ipynb')
-    result,_ =  C.convert()
+    result,_ = C.from_filename('tests/ipynbref/IntroNumPy.orig.ipynb')
     nt.assert_equal(result,'')