exporter.py
519 lines
| 18.8 KiB
| text/x-python
|
PythonLexer
|
r10690 | """This module defines Exporter, a highly configurable converter | ||
that uses Jinja2 to export notebook files into different formats. | ||||
|
r9578 | """ | ||
|
r9819 | |||
|
r9578 | #----------------------------------------------------------------------------- | ||
|
r9665 | # Copyright (c) 2013, the IPython Development Team. | ||
|
r9578 | # | ||
# Distributed under the terms of the Modified BSD License. | ||||
# | ||||
# The full license is in the file COPYING.txt, distributed with this software. | ||||
#----------------------------------------------------------------------------- | ||||
#----------------------------------------------------------------------------- | ||||
# Imports | ||||
#----------------------------------------------------------------------------- | ||||
|
r10677 | |||
|
r10430 | from __future__ import print_function, absolute_import | ||
|
r9665 | |||
# Stdlib imports | ||||
import io | ||||
|
r9819 | import os | ||
|
r10631 | import inspect | ||
|
r11383 | import copy | ||
import collections | ||||
import datetime | ||||
|
r9665 | |||
|
r11089 | # other libs/dependencies | ||
|
r11751 | from jinja2 import Environment, FileSystemLoader, ChoiceLoader, TemplateNotFound | ||
|
r11089 | |||
|
r9665 | # IPython imports | ||
|
r11844 | from IPython.config.configurable import LoggingConfigurable | ||
|
r10862 | from IPython.config import Config | ||
|
r9665 | from IPython.nbformat import current as nbformat | ||
|
r11844 | from IPython.utils.traitlets import MetaHasTraits, DottedObjectName, Unicode, List, Dict, Any | ||
|
r11367 | from IPython.utils.importstring import import_item | ||
|
r12413 | from IPython.utils import text | ||
|
r11547 | from IPython.utils import py3compat | ||
|
r9701 | |||
|
r11383 | from IPython.nbconvert import transformers as nbtransformers | ||
|
r11089 | from IPython.nbconvert import filters | ||
|
r10430 | |||
|
r10431 | #----------------------------------------------------------------------------- | ||
# Globals and constants | ||||
#----------------------------------------------------------------------------- | ||||
|
r9665 | |||
|
r10431 | #Jinja2 extensions to load. | ||
JINJA_EXTENSIONS = ['jinja2.ext.loopcontrols'] | ||||
|
r10838 | default_filters = { | ||
|
r12413 | 'indent': text.indent, | ||
|
r11685 | 'markdown2html': filters.markdown2html, | ||
|
r11089 | 'ansi2html': filters.ansi2html, | ||
'filter_data_type': filters.DataTypeFilter, | ||||
'get_lines': filters.get_lines, | ||||
|
r11685 | 'highlight2html': filters.highlight2html, | ||
|
r11089 | 'highlight2latex': filters.highlight2latex, | ||
|
r11711 | 'ipython2python': filters.ipython2python, | ||
|
r11972 | 'posix_path': filters.posix_path, | ||
|
r11089 | 'markdown2latex': filters.markdown2latex, | ||
'markdown2rst': filters.markdown2rst, | ||||
|
r11685 | 'comment_lines': filters.comment_lines, | ||
'strip_ansi': filters.strip_ansi, | ||||
'strip_dollars': filters.strip_dollars, | ||||
'strip_files_prefix': filters.strip_files_prefix, | ||||
'html2text' : filters.html2text, | ||||
|
r11302 | 'add_anchor': filters.add_anchor, | ||
|
r11089 | 'ansi2latex': filters.ansi2latex, | ||
|
r11685 | 'strip_math_space': filters.strip_math_space, | ||
'wrap_text': filters.wrap_text, | ||||
|
r11972 | 'escape_latex': filters.escape_latex, | ||
|
r12468 | 'path2url': filters.path2url, | ||
|
r10838 | } | ||
|
r10431 | #----------------------------------------------------------------------------- | ||
|
r10677 | # Class | ||
|
r9665 | #----------------------------------------------------------------------------- | ||
|
r10677 | |||
|
r11398 | class ResourcesDict(collections.defaultdict): | ||
def __missing__(self, key): | ||||
return '' | ||||
|
r11844 | class Exporter(LoggingConfigurable): | ||
|
r10690 | """ | ||
Exports notebooks into other file formats. Uses Jinja 2 templating engine | ||||
to output new formats. Inherit from this class if you are creating a new | ||||
template type along with new filters/transformers. If the filters/ | ||||
transformers provided by default suffice, there is no need to inherit from | ||||
this class. Instead, override the template_file and file_extension | ||||
traits via a config file. | ||||
|
r11750 | |||
|
r10838 | {filters} | ||
""" | ||||
|
r10874 | |||
# finish the docstring | ||||
|
r10838 | __doc__ = __doc__.format(filters = '- '+'\n - '.join(default_filters.keys())) | ||
|
r11745 | template_file = Unicode(u'default', | ||
|
r11733 | config=True, | ||
|
r10431 | help="Name of the template file to use") | ||
|
r11745 | def _template_file_changed(self, name, old, new): | ||
if new=='default': | ||||
self.template_file = self.default_template | ||||
else: | ||||
self.template_file = new | ||||
|
r11860 | self.template = None | ||
|
r11844 | self._load_template() | ||
|
r11745 | default_template = Unicode(u'') | ||
|
r11844 | template = Any() | ||
environment = Any() | ||||
|
r9768 | |||
|
r10587 | file_extension = Unicode( | ||
|
r10435 | 'txt', config=True, | ||
help="Extension of the file that should be written to disk" | ||||
) | ||||
|
r11655 | template_path = List(['.'], config=True) | ||
|
r11844 | def _template_path_changed(self, name, old, new): | ||
self._load_template() | ||||
|
r11648 | |||
default_template_path = Unicode( | ||||
os.path.join("..", "templates"), | ||||
|
r10690 | help="Path where the template files are located.") | ||
template_skeleton_path = Unicode( | ||||
|
r11648 | os.path.join("..", "templates", "skeleton"), | ||
|
r10690 | help="Path where the template skeleton files are located.") | ||
#Jinja block definitions | ||||
|
r10693 | jinja_comment_block_start = Unicode("", config=True) | ||
jinja_comment_block_end = Unicode("", config=True) | ||||
jinja_variable_block_start = Unicode("", config=True) | ||||
jinja_variable_block_end = Unicode("", config=True) | ||||
jinja_logic_block_start = Unicode("", config=True) | ||||
jinja_logic_block_end = Unicode("", config=True) | ||||
|
r10690 | |||
|
r10624 | #Extension that the template files use. | ||
|
r10690 | template_extension = Unicode(".tpl", config=True) | ||
|
r10624 | |||
|
r11367 | #Configurability, allows the user to easily add filters and transformers. | ||
transformers = List(config=True, | ||||
help="""List of transformers, by name or namespace, to enable.""") | ||||
|
r9640 | |||
|
r11367 | filters = Dict(config=True, | ||
help="""Dictionary of filters, by name and namespace, to add to the Jinja | ||||
environment.""") | ||||
|
r11383 | |||
default_transformers = List([nbtransformers.coalesce_streams, | ||||
|
r11429 | nbtransformers.SVG2PDFTransformer, | ||
|
r11634 | nbtransformers.ExtractOutputTransformer, | ||
|
r11429 | nbtransformers.CSSHTMLHeaderTransformer, | ||
nbtransformers.RevealHelpTransformer, | ||||
nbtransformers.LatexTransformer, | ||||
nbtransformers.SphinxTransformer], | ||||
|
r11383 | config=True, | ||
help="""List of transformers available by default, by name, namespace, | ||||
instance, or type.""") | ||||
|
r11429 | |||
|
r11367 | def __init__(self, config=None, extra_loaders=None, **kw): | ||
|
r10690 | """ | ||
Public constructor | ||||
|
r10485 | |||
|
r10690 | Parameters | ||
---------- | ||||
config : config | ||||
User configuration instance. | ||||
|
r11279 | extra_loaders : list[of Jinja Loaders] | ||
|
r11845 | ordered list of Jinja loader to find templates. Will be tried in order | ||
before the default FileSystem ones. | ||||
|
r11745 | template : str (optional, kw arg) | ||
Template to use when exporting. | ||||
|
r10690 | """ | ||
|
r11852 | if not config: | ||
config = self.default_config | ||||
|
r10690 | |||
|
r11845 | super(Exporter, self).__init__(config=config, **kw) | ||
|
r9665 | |||
|
r11383 | #Init | ||
|
r11852 | self._init_template() | ||
|
r11279 | self._init_environment(extra_loaders=extra_loaders) | ||
|
r11383 | self._init_transformers() | ||
self._init_filters() | ||||
|
r9624 | |||
|
r10862 | |||
@property | ||||
def default_config(self): | ||||
|
r10963 | return Config() | ||
|
r11845 | |||
def _config_changed(self, name, old, new): | ||||
|
r11848 | """When setting config, make sure to start with our default_config""" | ||
|
r11845 | c = self.default_config | ||
if new: | ||||
c.merge(new) | ||||
|
r11852 | if c != old: | ||
|
r11845 | self.config = c | ||
|
r11852 | super(Exporter, self)._config_changed(name, old, c) | ||
|
r11845 | |||
|
r10963 | |||
|
r11844 | def _load_template(self): | ||
|
r11848 | """Load the Jinja template object from the template file | ||
This is a no-op if the template attribute is already defined, | ||||
or the Jinja environment is not setup yet. | ||||
This is triggered by various trait changes that would change the template. | ||||
""" | ||||
|
r11844 | if self.template is not None: | ||
return | ||||
# called too early, do nothing | ||||
if self.environment is None: | ||||
return | ||||
# Try different template names during conversion. First try to load the | ||||
# template by name with extension added, then try loading the template | ||||
# as if the name is explicitly specified, then try the name as a | ||||
# 'flavor', and lastly just try to load the template by module name. | ||||
module_name = self.__module__.rsplit('.', 1)[-1] | ||||
|
r11952 | try_names = [] | ||
if self.template_file: | ||||
try_names.extend([ | ||||
self.template_file + self.template_extension, | ||||
self.template_file, | ||||
module_name + '_' + self.template_file + self.template_extension, | ||||
]) | ||||
try_names.append(module_name + self.template_extension) | ||||
|
r11844 | for try_name in try_names: | ||
self.log.debug("Attempting to load template %s", try_name) | ||||
try: | ||||
self.template = self.environment.get_template(try_name) | ||||
|
r11951 | except (TemplateNotFound, IOError): | ||
|
r11844 | pass | ||
|
r11951 | except Exception as e: | ||
self.log.warn("Unexpected exception loading template: %s", try_name, exc_info=True) | ||||
|
r11844 | else: | ||
self.log.info("Loaded template %s", try_name) | ||||
break | ||||
|
r10588 | |||
|
r11376 | def from_notebook_node(self, nb, resources=None, **kw): | ||
|
r10690 | """ | ||
Convert a notebook from a notebook node instance. | ||||
Parameters | ||||
---------- | ||||
nb : Notebook node | ||||
|
r11367 | resources : dict (**kw) | ||
of additional resources that can be accessed read/write by | ||||
transformers and filters. | ||||
|
r10690 | """ | ||
|
r11383 | nb_copy = copy.deepcopy(nb) | ||
resources = self._init_resources(resources) | ||||
|
r11379 | |||
|
r11745 | # Preprocess | ||
|
r11379 | nb_copy, resources = self._transform(nb_copy, resources) | ||
|
r9578 | |||
|
r11844 | self._load_template() | ||
|
r11745 | |||
|
r11844 | if self.template is not None: | ||
|
r11745 | output = self.template.render(nb=nb_copy, resources=resources) | ||
else: | ||||
raise IOError('template file "%s" could not be found' % self.template_file) | ||||
|
r11367 | return output, resources | ||
|
r9578 | |||
|
r11367 | |||
|
r11379 | def from_filename(self, filename, resources=None, **kw): | ||
|
r10690 | """ | ||
Convert a notebook from a notebook file. | ||||
Parameters | ||||
---------- | ||||
filename : str | ||||
Full filename of the notebook file to open and convert. | ||||
""" | ||||
|
r11383 | |||
#Pull the metadata from the filesystem. | ||||
|
r11425 | if resources is None: | ||
resources = ResourcesDict() | ||||
if not 'metadata' in resources or resources['metadata'] == '': | ||||
|
r11383 | resources['metadata'] = ResourcesDict() | ||
basename = os.path.basename(filename) | ||||
notebook_name = basename[:basename.rfind('.')] | ||||
resources['metadata']['name'] = notebook_name | ||||
modified_date = datetime.datetime.fromtimestamp(os.path.getmtime(filename)) | ||||
|
r12413 | resources['metadata']['modified_date'] = modified_date.strftime(text.date_format) | ||
|
r10690 | |||
|
r10431 | with io.open(filename) as f: | ||
|
r11367 | return self.from_notebook_node(nbformat.read(f, 'json'), resources=resources,**kw) | ||
|
r10431 | |||
|
r10432 | |||
|
r11376 | def from_file(self, file_stream, resources=None, **kw): | ||
|
r10690 | """ | ||
Convert a notebook from a notebook file. | ||||
Parameters | ||||
---------- | ||||
file_stream : file-like object | ||||
Notebook file-like object to convert. | ||||
""" | ||||
|
r11367 | return self.from_notebook_node(nbformat.read(file_stream, 'json'), resources=resources, **kw) | ||
|
r10578 | |||
|
r11428 | def register_transformer(self, transformer, enabled=False): | ||
|
r10690 | """ | ||
Register a transformer. | ||||
Transformers are classes that act upon the notebook before it is | ||||
passed into the Jinja templating engine. Transformers are also | ||||
capable of passing additional information to the Jinja | ||||
templating engine. | ||||
Parameters | ||||
---------- | ||||
transformer : transformer | ||||
""" | ||||
|
r11428 | if transformer is None: | ||
raise TypeError('transformer') | ||||
|
r11547 | isclass = isinstance(transformer, type) | ||
|
r11428 | constructed = not isclass | ||
|
r11367 | |||
#Handle transformer's registration based on it's type | ||||
|
r11547 | if constructed and isinstance(transformer, py3compat.string_types): | ||
|
r11428 | #Transformer is a string, import the namespace and recursively call | ||
#this register_transformer method | ||||
transformer_cls = import_item(transformer) | ||||
return self.register_transformer(transformer_cls, enabled) | ||||
|
r10690 | |||
|
r11428 | if constructed and hasattr(transformer, '__call__'): | ||
|
r11367 | #Transformer is a function, no need to construct it. | ||
|
r11428 | #Register and return the transformer. | ||
if enabled: | ||||
transformer.enabled = True | ||||
|
r11367 | self._transformers.append(transformer) | ||
|
r10631 | return transformer | ||
|
r11367 | |||
|
r11428 | elif isclass and isinstance(transformer, MetaHasTraits): | ||
|
r11367 | #Transformer is configurable. Make sure to pass in new default for | ||
#the enabled flag if one was specified. | ||||
|
r11428 | self.register_transformer(transformer(parent=self), enabled) | ||
|
r11367 | |||
|
r11428 | elif isclass: | ||
|
r11367 | #Transformer is not configurable, construct it | ||
|
r11428 | self.register_transformer(transformer(), enabled) | ||
|
r11367 | |||
|
r10587 | else: | ||
|
r11428 | #Transformer is an instance of something without a __call__ | ||
#attribute. | ||||
raise TypeError('transformer') | ||||
|
r10587 | |||
|
r11547 | def register_filter(self, name, jinja_filter): | ||
|
r10690 | """ | ||
Register a filter. | ||||
A filter is a function that accepts and acts on one string. | ||||
The filters are accesible within the Jinja templating engine. | ||||
Parameters | ||||
---------- | ||||
name : str | ||||
name to give the filter in the Jinja engine | ||||
filter : filter | ||||
""" | ||||
|
r11547 | if jinja_filter is None: | ||
|
r11428 | raise TypeError('filter') | ||
|
r11547 | isclass = isinstance(jinja_filter, type) | ||
|
r11428 | constructed = not isclass | ||
#Handle filter's registration based on it's type | ||||
|
r11547 | if constructed and isinstance(jinja_filter, py3compat.string_types): | ||
|
r11428 | #filter is a string, import the namespace and recursively call | ||
#this register_filter method | ||||
|
r11547 | filter_cls = import_item(jinja_filter) | ||
|
r11428 | return self.register_filter(name, filter_cls) | ||
|
r11547 | if constructed and hasattr(jinja_filter, '__call__'): | ||
|
r11428 | #filter is a function, no need to construct it. | ||
|
r11547 | self.environment.filters[name] = jinja_filter | ||
return jinja_filter | ||||
|
r11428 | |||
|
r11547 | elif isclass and isinstance(jinja_filter, MetaHasTraits): | ||
|
r11428 | #filter is configurable. Make sure to pass in new default for | ||
#the enabled flag if one was specified. | ||||
|
r11547 | filter_instance = jinja_filter(parent=self) | ||
self.register_filter(name, filter_instance ) | ||||
|
r11428 | |||
elif isclass: | ||||
#filter is not configurable, construct it | ||||
|
r11547 | filter_instance = jinja_filter() | ||
self.register_filter(name, filter_instance) | ||||
|
r11428 | |||
|
r10578 | else: | ||
|
r11428 | #filter is an instance of something without a __call__ | ||
#attribute. | ||||
raise TypeError('filter') | ||||
|
r10578 | |||
|
r10588 | |||
|
r11852 | def _init_template(self): | ||
|
r11733 | """ | ||
Make sure a template name is specified. If one isn't specified, try to | ||||
build one from the information we know. | ||||
""" | ||||
|
r11745 | self._template_file_changed('template_file', self.template_file, self.template_file) | ||
|
r11733 | |||
|
r11279 | def _init_environment(self, extra_loaders=None): | ||
|
r10690 | """ | ||
Create the Jinja templating environment. | ||||
""" | ||||
|
r11197 | here = os.path.dirname(os.path.realpath(__file__)) | ||
|
r11279 | loaders = [] | ||
if extra_loaders: | ||||
loaders.extend(extra_loaders) | ||||
|
r11655 | paths = self.template_path | ||
|
r11648 | paths.extend([os.path.join(here, self.default_template_path), | ||
os.path.join(here, self.template_skeleton_path)]) | ||||
loaders.append(FileSystemLoader(paths)) | ||||
|
r11279 | |||
self.environment = Environment( | ||||
loader= ChoiceLoader(loaders), | ||||
|
r10485 | extensions=JINJA_EXTENSIONS | ||
) | ||||
|
r10690 | |||
#Set special Jinja2 syntax that will not conflict with latex. | ||||
|
r10693 | if self.jinja_logic_block_start: | ||
|
r10690 | self.environment.block_start_string = self.jinja_logic_block_start | ||
|
r10693 | if self.jinja_logic_block_end: | ||
|
r10690 | self.environment.block_end_string = self.jinja_logic_block_end | ||
|
r10693 | if self.jinja_variable_block_start: | ||
|
r10690 | self.environment.variable_start_string = self.jinja_variable_block_start | ||
|
r10693 | if self.jinja_variable_block_end: | ||
|
r10690 | self.environment.variable_end_string = self.jinja_variable_block_end | ||
|
r10693 | if self.jinja_comment_block_start: | ||
|
r10690 | self.environment.comment_start_string = self.jinja_comment_block_start | ||
|
r10693 | if self.jinja_comment_block_end: | ||
|
r10690 | self.environment.comment_end_string = self.jinja_comment_block_end | ||
|
r10485 | |||
|
r11383 | |||
def _init_transformers(self): | ||||
""" | ||||
Register all of the transformers needed for this exporter, disabled | ||||
unless specified explicitly. | ||||
""" | ||||
self._transformers = [] | ||||
#Load default transformers (not necessarly enabled by default). | ||||
if self.default_transformers: | ||||
for transformer in self.default_transformers: | ||||
self.register_transformer(transformer) | ||||
#Load user transformers. Enable by default. | ||||
if self.transformers: | ||||
for transformer in self.transformers: | ||||
self.register_transformer(transformer, enabled=True) | ||||
def _init_filters(self): | ||||
""" | ||||
Register all of the filters required for the exporter. | ||||
""" | ||||
#Add default filters to the Jinja2 environment | ||||
|
r11547 | for key, value in default_filters.items(): | ||
|
r11383 | self.register_filter(key, value) | ||
#Load user filters. Overwrite existing filters if need be. | ||||
if self.filters: | ||||
|
r11547 | for key, user_filter in self.filters.items(): | ||
|
r11383 | self.register_filter(key, user_filter) | ||
def _init_resources(self, resources): | ||||
|
r10485 | |||
|
r11383 | #Make sure the resources dict is of ResourcesDict type. | ||
if resources is None: | ||||
resources = ResourcesDict() | ||||
if not isinstance(resources, ResourcesDict): | ||||
new_resources = ResourcesDict() | ||||
new_resources.update(resources) | ||||
resources = new_resources | ||||
#Make sure the metadata extension exists in resources | ||||
if 'metadata' in resources: | ||||
if not isinstance(resources['metadata'], ResourcesDict): | ||||
resources['metadata'] = ResourcesDict(resources['metadata']) | ||||
else: | ||||
resources['metadata'] = ResourcesDict() | ||||
|
r11402 | if not resources['metadata']['name']: | ||
|
r11403 | resources['metadata']['name'] = 'Notebook' | ||
|
r11383 | |||
#Set the output extension | ||||
resources['output_extension'] = self.file_extension | ||||
return resources | ||||
|
r10485 | |||
|
r11379 | def _transform(self, nb, resources): | ||
|
r10690 | """ | ||
Preprocess the notebook before passing it into the Jinja engine. | ||||
To preprocess the notebook is to apply all of the | ||||
Parameters | ||||
---------- | ||||
nb : notebook node | ||||
notebook that is being exported. | ||||
|
r10837 | resources : a dict of additional resources that | ||
can be accessed read/write by transformers | ||||
and filters. | ||||
|
r10690 | """ | ||
|
r11399 | # Do a copy.deepcopy first, | ||
|
r10867 | # we are never safe enough with what the transformers could do. | ||
|
r11399 | nbc = copy.deepcopy(nb) | ||
resc = copy.deepcopy(resources) | ||||
|
r11367 | |||
|
r10432 | #Run each transformer on the notebook. Carry the output along | ||
#to each transformer | ||||
|
r11367 | for transformer in self._transformers: | ||
nbc, resc = transformer(nbc, resc) | ||||
return nbc, resc | ||||