base.py
266 lines
| 9.9 KiB
| text/x-python
|
PythonLexer
|
r10430 | """Exporter for the notebook conversion pipeline. | ||
|
r9578 | |||
|
r10430 | This module defines Exporter, a highly configurable converter | ||
that uses Jinja2 to export notebook files into different format. | ||||
|
r9665 | |||
You can register both pre-transformers that will act on the notebook format | ||||
befor conversion and jinja filter that would then be availlable in the templates | ||||
|
r9578 | """ | ||
|
r9819 | |||
|
r9578 | #----------------------------------------------------------------------------- | ||
|
r9665 | # Copyright (c) 2013, the IPython Development Team. | ||
|
r9578 | # | ||
# Distributed under the terms of the Modified BSD License. | ||||
# | ||||
# The full license is in the file COPYING.txt, distributed with this software. | ||||
#----------------------------------------------------------------------------- | ||||
#----------------------------------------------------------------------------- | ||||
# Imports | ||||
#----------------------------------------------------------------------------- | ||||
|
r10430 | from __future__ import print_function, absolute_import | ||
|
r9665 | |||
# Stdlib imports | ||||
import io | ||||
|
r9819 | import os | ||
|
r9665 | |||
# IPython imports | ||||
from IPython.config.configurable import Configurable | ||||
from IPython.nbformat import current as nbformat | ||||
|
r10430 | from IPython.utils.traitlets import MetaHasTraits, Unicode, List, Bool | ||
|
r10432 | from IPython.utils.text import indent | ||
|
r9701 | |||
|
r9665 | # other libs/dependencies | ||
from jinja2 import Environment, FileSystemLoader | ||||
|
r10432 | from markdown import markdown | ||
|
r9665 | |||
|
r10485 | # local import | ||
import filters.strings | ||||
import filters.markdown | ||||
import filters.latex | ||||
import filters.datatypefilter | ||||
import filters.pygments | ||||
import filters.ansi | ||||
import transformers.extractfigure | ||||
import transformers.csshtmlheader | ||||
import transformers.revealhelp | ||||
import transformers.coalescestreams | ||||
|
r10430 | |||
|
r10431 | |||
#----------------------------------------------------------------------------- | ||||
# Globals and constants | ||||
#----------------------------------------------------------------------------- | ||||
|
r9665 | |||
|
r10431 | #Standard Jinja2 environment constants | ||
TEMPLATE_PATH = "/../templates/" | ||||
TEMPLATE_SKELETON_PATH = "/../templates/skeleton/" | ||||
TEMPLATE_EXTENSION = ".tpl" | ||||
|
r9621 | |||
|
r10431 | #Jinja2 extensions to load. | ||
JINJA_EXTENSIONS = ['jinja2.ext.loopcontrols'] | ||||
#----------------------------------------------------------------------------- | ||||
# Classes and functions | ||||
|
r9665 | #----------------------------------------------------------------------------- | ||
|
r10430 | class Exporter(Configurable): | ||
|
r9640 | """ A Jinja2 base converter templates | ||
|
r10485 | Pre-process the IPYNB files, feed it through Jinja2 templates, | ||
|
r9640 | and spit an converted files and a data object with other data | ||
|
r9771 | should be mostly configurable | ||
|
r9640 | """ | ||
|
r9578 | |||
|
r9640 | pre_transformer_order = List(['haspyout_transformer'], | ||
|
r10431 | config=True, | ||
help= """ | ||||
|
r10485 | An ordered list of pre-transformer to apply to the IPYNB | ||
|
r10431 | file before running through templates | ||
""" | ||||
) | ||||
|
r9618 | |||
|
r10431 | template_file = Unicode( | ||
'', config=True, | ||||
help="Name of the template file to use") | ||||
|
r9768 | |||
|
r10435 | fileext = Unicode( | ||
'txt', config=True, | ||||
help="Extension of the file that should be written to disk" | ||||
) | ||||
stdout = Bool( | ||||
True, config=True, | ||||
|
r10485 | help="""Whether to print the converted IPYNB file to stdout | ||
|
r10435 | "use full do diff files without actually writing a new file""" | ||
) | ||||
write = Bool( | ||||
False, config=True, | ||||
help="""Should the converted notebook file be written to disk | ||||
along with potential extracted resources.""" | ||||
) | ||||
|
r10431 | #Processors that process the input data prior to the export, set in the | ||
#constructor for this class. | ||||
preprocessors = [] | ||||
|
r9640 | |||
|
r10435 | def __init__(self, preprocessors={}, jinja_filters={}, config=None, export_format, **kw): | ||
|
r9769 | """ Init a new converter. | ||
config: the Configurable config object to pass around. | ||||
|
r9603 | |||
|
r9650 | preprocessors: dict of **availlable** key/value function to run on | ||
|
r9665 | ipynb json data before conversion to extract/inline file. | ||
See `transformer.py` and `ConfigurableTransformers` | ||||
|
r9650 | |||
|
r9665 | set the order in which the transformers should apply | ||
with the `pre_transformer_order` trait of this class | ||||
|
r9603 | |||
|
r9665 | transformers registerd by this key will take precedence on | ||
default one. | ||||
jinja_filters: dict of supplementary jinja filter that should be made | ||||
availlable in template. If those are of Configurable Class type, | ||||
they will be instanciated with the config object as argument. | ||||
user defined filter will overwrite the one availlable by default. | ||||
|
r9603 | """ | ||
|
r10435 | |||
|
r10485 | #Set the default options for the exporter. | ||
default_config = self.config | ||||
#Set properties that must be set in the config class in order to | ||||
#propagate to other classes. | ||||
default_config.GlobalConfigurable.display_data_priority =['svg', 'png', 'latex', 'jpg', 'jpeg','text'] | ||||
default_config.ExtractFigureTransformer.display_data_priority=['svg', 'png', 'latex', 'jpg', 'jpeg','text'] | ||||
#Set default properties of the exporter. | ||||
#For most (or all cases), the template file name matches the format name. | ||||
self.display_data_priority= ['svg', 'png', 'latex', 'jpg', 'jpeg','text'] | ||||
self.template_file = export_format | ||||
|
r10435 | if not config == None: | ||
default_config._merge(config) | ||||
|
r10485 | config = default_config | ||
|
r10435 | #Call the base class constructor | ||
super(Exporter, self).__init__(config=config, **kw) | ||||
|
r9665 | |||
|
r10480 | #Standard environment | ||
self.ext = TEMPLATE_EXTENSION | ||||
|
r10485 | self._init_environment() | ||
|
r9624 | |||
|
r10485 | #TODO: Implement reflection style methods to get user transformers. | ||
#for name in self.pre_transformer_order: | ||||
# # get the user-defined transformer first | ||||
# transformer = preprocessors.get(name, getattr(trans, name, None)) | ||||
# if isinstance(transformer, MetaHasTraits): | ||||
# transformer = transformer(config=config) | ||||
# self.preprocessors.append(transformer) | ||||
|
r9640 | |||
|
r10431 | #For compatibility, TODO: remove later. | ||
|
r10485 | self.preprocessors.append(transformers.coalescestreams.coalesce_streams) | ||
self.preprocessors.append(transformers.extractfigure.ExtractFigureTransformer(config=config)) | ||||
self.preprocessors.append(transformers.revealhelp.RevealHelpTransformer(config=config)) | ||||
self.preprocessors.append(transformers.csshtmlheader.CSSHtmlHeaderTransformer(config=config)) | ||||
|
r9615 | |||
|
r10431 | #Add filters to the Jinja2 environment | ||
|
r10485 | self._register_filters(config) | ||
|
r10431 | |||
#Load user filters. Overwrite existing filters if need be. | ||||
for key, user_filter in jinja_filters.iteritems(): | ||||
if isinstance(user_filter, MetaHasTraits): | ||||
self.env.filters[key] = user_filter(config=config) | ||||
else: | ||||
self.env.filters[key] = user_filter | ||||
#Load the template file. | ||||
|
r9618 | self.template = self.env.get_template(self.template_file+self.ext) | ||
|
r9611 | |||
|
r9578 | |||
|
r10431 | def export(self, nb): | ||
"""Export notebook object | ||||
nb: Notebook object to export. | ||||
Returns both the converted ipynb file and a dict containing the | ||||
resources created along the way via the transformers and Jinja2 | ||||
processing. | ||||
|
r9601 | """ | ||
|
r10431 | |||
nb, resources = self._preprocess(nb) | ||||
|
r9654 | return self.template.render(nb=nb, resources=resources), resources | ||
|
r9578 | |||
|
r9640 | def from_filename(self, filename): | ||
|
r10431 | """Read and export a notebook from a filename | ||
|
r9624 | |||
|
r10431 | filename: Filename of the notebook file to export. | ||
|
r9650 | |||
|
r10431 | Returns both the converted ipynb file and a dict containing the | ||
resources created along the way via the transformers and Jinja2 | ||||
processing. | ||||
|
r9666 | """ | ||
|
r10431 | with io.open(filename) as f: | ||
return self.export(nbformat.read(f, 'json')) | ||||
|
r10432 | |||
|
r10431 | def from_file(self, file_stream): | ||
"""Read and export a notebook from a filename | ||||
|
r9666 | |||
|
r10431 | file_stream: File handle of file that contains notebook data. | ||
|
r9666 | |||
|
r10431 | Returns both the converted ipynb file and a dict containing the | ||
resources created along the way via the transformers and Jinja2 | ||||
processing. | ||||
|
r9666 | """ | ||
|
r10431 | |||
|
r10432 | return self.export(nbformat.read(file_stream, 'json')) | ||
|
r10485 | def _init_environment(self): | ||
self.env = Environment( | ||||
loader=FileSystemLoader([ | ||||
os.path.dirname(os.path.realpath(__file__)) + TEMPLATE_PATH, | ||||
os.path.dirname(os.path.realpath(__file__)) + TEMPLATE_SKELETON_PATH, | ||||
]), | ||||
extensions=JINJA_EXTENSIONS | ||||
) | ||||
def _register_filters(self, config): | ||||
self.env.filters['indent'] = indent | ||||
self.env.filters['markdown'] = markdown | ||||
self.env.filters['ansi2html'] = filters.ansi.ansi2html | ||||
self.env.filters['filter_data_type'] = filters.datatypefilter.DataTypeFilter(config=config) | ||||
self.env.filters['get_lines'] = filters.strings.get_lines | ||||
self.env.filters['highlight'] = filters.pygments.highlight | ||||
self.env.filters['highlight2html'] = filters.pygments.highlight | ||||
self.env.filters['highlight2latex'] = filters.pygments.highlight2latex | ||||
self.env.filters['markdown2latex'] = filters.markdown.markdown2latex | ||||
self.env.filters['markdown2rst'] = filters.markdown.markdown2rst | ||||
self.env.filters['pycomment'] = filters.strings.python_comment | ||||
self.env.filters['rm_ansi'] = filters.ansi.remove_ansi | ||||
self.env.filters['rm_dollars'] = filters.strings.strip_dollars | ||||
self.env.filters['rm_fake'] = filters.strings.rm_fake | ||||
self.env.filters['rm_math_space'] = filters.latex.rm_math_space | ||||
self.env.filters['wrap'] = filters.strings.wrap | ||||
|
r10432 | def _preprocess(self, nb): | ||
""" Preprocess the notebook using the transformers specific | ||||
for the current export format. | ||||
nb: Notebook to preprocess | ||||
""" | ||||
#Dict of 'resources' that can be filled by the preprocessors. | ||||
resources = {} | ||||
#Run each transformer on the notebook. Carry the output along | ||||
#to each transformer | ||||
for transformer in self.preprocessors: | ||||
nb, resources = transformer(nb, resources) | ||||
return nb, resources | ||||