From b2eb244ed689cf2feb5fd0fed3cba707f387ef5f 2013-07-15 23:15:32 From: Jonathan Frederic Date: 2013-07-15 23:15:32 Subject: [PATCH] Added writers and supporting code. --- diff --git a/IPython/nbconvert/__init__.py b/IPython/nbconvert/__init__.py index 4b639b4..121eed9 100755 --- a/IPython/nbconvert/__init__.py +++ b/IPython/nbconvert/__init__.py @@ -3,3 +3,4 @@ from .exporters import * import filters import transformers +import writers diff --git a/IPython/nbconvert/exporters/export.py b/IPython/nbconvert/exporters/export.py index ced9d8c..4e51c88 100755 --- a/IPython/nbconvert/exporters/export.py +++ b/IPython/nbconvert/exporters/export.py @@ -16,6 +16,7 @@ Module containing single call export functions. from functools import wraps from IPython.nbformat.v3.nbbase import NotebookNode +from IPython.config import Config from .exporter import Exporter from .basichtml import BasicHTMLExporter @@ -37,17 +38,10 @@ def DocDecorator(f): #Set docstring of function f.__doc__ = f.__doc__ + """ nb : Notebook node - config : config + config : config (optional, keyword arg) User configuration instance. - transformers : list[of transformer] - Custom transformers to apply to the notebook prior to engaging - the Jinja template engine. Any transformers specified here - will override existing transformers if a naming conflict - occurs. - filters : list[of filter] - Custom filters to make accessible to the Jinja templates. Any - filters specified here will override existing filters if a - naming conflict occurs. + resources : dict (optional, keyword arg) + Resources used in the conversion process. Returns ---------- @@ -89,7 +83,7 @@ __all__ = [ ] @DocDecorator -def export(exporter_type, nb, config=None, transformers=None, filters=None): +def export(exporter_type, nb, **kw): """ Export a notebook object using specific exporter class. @@ -106,111 +100,110 @@ def export(exporter_type, nb, config=None, transformers=None, filters=None): raise TypeError("Exporter is None") elif not issubclass(exporter_type, Exporter): raise TypeError("Exporter type does not inherit from Exporter (base)") - if nb is None: raise TypeError("nb is None") #Create the exporter - exporter_instance = exporter_type(preprocessors=transformers, - jinja_filters=filters, config=config) + exporter_instance = exporter_type(config=kw.get('config', Config())) #Try to convert the notebook using the appropriate conversion function. + resources = kw.get('resources', {}) if isinstance(nb, NotebookNode): - output, resources = exporter_instance.from_notebook_node(nb) + output, resources = exporter_instance.from_notebook_node(nb, resources) elif isinstance(nb, basestring): - output, resources = exporter_instance.from_filename(nb) + output, resources = exporter_instance.from_filename(nb, resources) else: - output, resources = exporter_instance.from_file(nb) - return output, resources, exporter_instance + output, resources = exporter_instance.from_file(nb, resources) + return output, resources @DocDecorator -def export_sphinx_manual(nb, config=None, transformers=None, filters=None): +def export_sphinx_manual(nb, **kw): """ Export a notebook object to Sphinx Manual LaTeX """ - return export(SphinxManualExporter, nb, config, transformers, filters) + return export(SphinxManualExporter, nb, **kw) @DocDecorator -def export_sphinx_howto(nb, config=None, transformers=None, filters=None): +def export_sphinx_howto(nb, **kw): """ Export a notebook object to Sphinx HowTo LaTeX """ - return export(SphinxHowtoExporter, nb, config, transformers, filters) + return export(SphinxHowtoExporter, nb, **kw) @DocDecorator -def export_basic_html(nb, config=None, transformers=None, filters=None): +def export_basic_html(nb, **kw): """ Export a notebook object to Basic HTML """ - return export(BasicHTMLExporter, nb, config, transformers, filters) + return export(BasicHTMLExporter, nb, **kw) @DocDecorator -def export_full_html(nb, config=None, transformers=None, filters=None): +def export_full_html(nb, **kw): """ Export a notebook object to Full HTML """ - return export(FullHTMLExporter, nb, config, transformers, filters) + return export(FullHTMLExporter, nb, **kw) @DocDecorator -def export_latex(nb, config=None, transformers=None, filters=None): +def export_latex(nb, **kw): """ Export a notebook object to LaTeX """ - return export(LatexExporter, nb, config, transformers, filters) + return export(LatexExporter, nb, **kw) @DocDecorator -def export_markdown(nb, config=None, transformers=None, filters=None): +def export_markdown(nb, **kw): """ Export a notebook object to Markdown """ - return export(MarkdownExporter, nb, config, transformers, filters) + return export(MarkdownExporter, nb, **kw) @DocDecorator -def export_python(nb, config=None, transformers=None, filters=None): +def export_python(nb, **kw): """ Export a notebook object to Python """ - return export(PythonExporter, nb, config, transformers, filters) + return export(PythonExporter, nb, **kw) @DocDecorator -def export_reveal(nb, config=None, transformers=None, filters=None): +def export_reveal(nb, **kw): """ Export a notebook object to Reveal """ - return export(RevealExporter, nb, config, transformers, filters) + return export(RevealExporter, nb, **kw) @DocDecorator -def export_rst(nb, config=None, transformers=None, filters=None): +def export_rst(nb, **kw): """ Export a notebook object to RST """ - return export(RstExporter, nb, config, transformers, filters) + return export(RstExporter, nb, **kw) @DocDecorator -def export_by_name(template_name, nb, config=None, transformers=None, filters=None): +def export_by_name(format_name, nb, **kw): """ Export a notebook object to a template type by its name. Reflection (Inspect) is used to find the template's corresponding explicit export method defined in this module. That method is then called directly. - template_name : str + format_name : str Name of the template style to export to. """ - function_name = "export_" + template_name.lower() + function_name = "export_" + format_name.lower() if function_name in globals(): - return globals()[function_name](nb, config, transformers, filters) + return globals()[function_name](nb, **kw) else: raise NameError("template for `%s` not found" % function_name) @@ -218,6 +211,7 @@ def get_export_names(): "Return a list of the currently supported export targets" # grab everything after 'export_' l = [x[len('export_'):] for x in __all__ if x.startswith('export_')] - # filter out the one method that is not a template + + # filter out the one method that is not a template l = [x for x in l if 'by_name' not in x] return sorted(l) diff --git a/IPython/nbconvert/exporters/exporter.py b/IPython/nbconvert/exporters/exporter.py index cc19ee5..7a04d1d 100755 --- a/IPython/nbconvert/exporters/exporter.py +++ b/IPython/nbconvert/exporters/exporter.py @@ -20,6 +20,7 @@ from __future__ import print_function, absolute_import import io import os import inspect +import types from copy import deepcopy # other libs/dependencies @@ -29,7 +30,8 @@ from jinja2 import Environment, FileSystemLoader, ChoiceLoader from IPython.config.configurable import Configurable from IPython.config import Config from IPython.nbformat import current as nbformat -from IPython.utils.traitlets import MetaHasTraits, Unicode +from IPython.utils.traitlets import MetaHasTraits, DottedObjectName, Unicode, List, Dict +from IPython.utils.importstring import import_item from IPython.utils.text import indent from IPython.nbconvert import filters @@ -110,27 +112,19 @@ class Exporter(Configurable): #Extension that the template files use. template_extension = Unicode(".tpl", config=True) - #Processors that process the input data prior to the export, set in the - #constructor for this class. - transformers = None - + #Configurability, allows the user to easily add filters and transformers. + transformers = List(config=True, + help="""List of transformers, by name or namespace, to enable.""") + filters = Dict(config=True, + help="""Dictionary of filters, by name and namespace, to add to the Jinja + environment.""") - def __init__(self, transformers=None, filters=None, config=None, extra_loaders=None, **kw): + def __init__(self, config=None, extra_loaders=None, **kw): """ Public constructor Parameters ---------- - transformers : list[of transformer] - Custom transformers to apply to the notebook prior to engaging - the Jinja template engine. Any transformers specified here - will override existing transformers if a naming conflict - occurs. - filters : dict[of filter] - filters specified here will override existing filters if a naming - conflict occurs. Filters are availlable in jinja template through - the name of the corresponding key. Cf class docstring for - availlable default filters. config : config User configuration instance. extra_loaders : list[of Jinja Loaders] @@ -149,49 +143,53 @@ class Exporter(Configurable): self._init_environment(extra_loaders=extra_loaders) #Add transformers + self._transformers = [] self._register_transformers() #Add filters to the Jinja2 environment self._register_filters() - #Load user transformers. Overwrite existing transformers if need be. - if transformers : - for transformer in transformers: - self.register_transformer(transformer) + #Load user transformers. Enabled by default. + if self.transformers: + for transformer in self.transformers: + self.register_transformer(transformer, True) #Load user filters. Overwrite existing filters if need be. - if not filters is None: - for key, user_filter in filters.iteritems(): + if self.filters: + for key, user_filter in self.filters.iteritems(): self.register_filter(key, user_filter) + @property def default_config(self): return Config() - - def from_notebook_node(self, nb, resources=None): + def from_notebook_node(self, nb, resources={}, **kw): """ Convert a notebook from a notebook node instance. Parameters ---------- nb : Notebook node - resources : a dict of additional resources that - can be accessed read/write by transformers - and filters. + resources : dict (**kw) + of additional resources that can be accessed read/write by + transformers and filters. """ - if resources is None: - resources = {} + + #Preprocess nb, resources = self._preprocess(nb, resources) - - #Load the template file. - self.template = self.environment.get_template(self.template_file+self.template_extension) - - return self.template.render(nb=nb, resources=resources), resources + #Convert + self.template = self.environment.get_template(self.template_file + self.template_extension) + output = self.template.render(nb=nb, resources=resources) + + #Set output extension in resources dict + resources['output_extension'] = self.file_extension + return output, resources - def from_filename(self, filename): + + def from_filename(self, filename, resources={}, **kw): """ Convert a notebook from a notebook file. @@ -202,10 +200,10 @@ class Exporter(Configurable): """ with io.open(filename) as f: - return self.from_notebook_node(nbformat.read(f, 'json')) + return self.from_notebook_node(nbformat.read(f, 'json'), resources=resources,**kw) - def from_file(self, file_stream): + def from_file(self, file_stream, resources={}, **kw): """ Convert a notebook from a notebook file. @@ -214,10 +212,10 @@ class Exporter(Configurable): file_stream : file-like object Notebook file-like object to convert. """ - return self.from_notebook_node(nbformat.read(file_stream, 'json')) + return self.from_notebook_node(nbformat.read(file_stream, 'json'), resources=resources, **kw) - def register_transformer(self, transformer): + def register_transformer(self, transformer, enabled=None): """ Register a transformer. Transformers are classes that act upon the notebook before it is @@ -229,20 +227,35 @@ class Exporter(Configurable): ---------- transformer : transformer """ - if self.transformers is None: - self.transformers = [] - + + #Handle transformer's registration based on it's type if inspect.isfunction(transformer): - self.transformers.append(transformer) + #Transformer is a function, no need to construct it. + self._transformers.append(transformer) return transformer + + elif isinstance(transformer, types.StringTypes): + #Transformer is a string, import the namespace and recursively call + #this register_transformer method + transformer_cls = import_item(DottedObjectName(transformer)) + return self.register_transformer(transformer_cls, enabled=None) + elif isinstance(transformer, MetaHasTraits): - transformer_instance = transformer(config=self.config) - self.transformers.append(transformer_instance) - return transformer_instance + #Transformer is configurable. Make sure to pass in new default for + #the enabled flag if one was specified. + c = Config() + if not enabled is None: + c = Config({transformer.__name__: {'enabled': enabled}}) + c.merge(self.config) + transformer_instance = transformer(config=c) + else: + #Transformer is not configurable, construct it transformer_instance = transformer() - self.transformers.append(transformer_instance) - return transformer_instance + + #Register and return the transformer. + self._transformers.append(transformer_instance) + return transformer_instance def register_filter(self, name, filter): @@ -259,6 +272,9 @@ class Exporter(Configurable): """ if inspect.isfunction(filter): self.environment.filters[name] = filter + elif isinstance(filter, types.StringTypes): + filter_cls = import_item(DottedObjectName(filter)) + self.register_filter(name, filter_cls) elif isinstance(filter, MetaHasTraits): self.environment.filters[name] = filter(config=self.config) else: @@ -268,22 +284,20 @@ class Exporter(Configurable): def _register_transformers(self): """ - Register all of the transformers needed for this exporter. + Register all of the transformers needed for this exporter, disabled + unless specified explicitly. """ - + self.register_transformer(transformers.coalesce_streams) - - #Remember the figure extraction transformer so it can be enabled and - #disabled easily later. - self.extract_figure_transformer = self.register_transformer(transformers.ExtractFigureTransformer) + self.register_transformer(transformers.ExtractFigureTransformer) def _register_filters(self): """ Register all of the filters required for the exporter. """ - for k, v in default_filters.iteritems(): - self.register_filter(k, v) + for key, value in default_filters.iteritems(): + self.register_filter(key, value) def _init_environment(self, extra_loaders=None): @@ -338,9 +352,9 @@ class Exporter(Configurable): # we are never safe enough with what the transformers could do. nbc = deepcopy(nb) resc = deepcopy(resources) + #Run each transformer on the notebook. Carry the output along #to each transformer - for transformer in self.transformers: - nb, resources = transformer(nbc, resc) - return nb, resources - + for transformer in self._transformers: + nbc, resc = transformer(nbc, resc) + return nbc, resc diff --git a/IPython/nbconvert/exporters/latex.py b/IPython/nbconvert/exporters/latex.py index d66f3fe..5e80dd2 100755 --- a/IPython/nbconvert/exporters/latex.py +++ b/IPython/nbconvert/exporters/latex.py @@ -85,23 +85,26 @@ class LatexExporter(Exporter): """ Register all of the transformers needed for this exporter. """ + + #Register ConvertSvgTransformer before any other transformers! + #Important because it allows the conversion of svg->png BEFORE the + #extract figure transformer acts on the data. + self.register_transformer(transformers.ConvertSvgTransformer, True) - #Register the transformers of the base class. + #Register transformers super(LatexExporter, self)._register_transformers() - - #Register latex transformer - self.register_transformer(transformers.LatexTransformer) + self.register_transformer(transformers.LatexTransformer, True) @property def default_config(self): c = Config({ 'GlobalConfigurable': { - 'display_data_priority' : ['latex', 'svg', 'png', 'jpg', 'jpeg' , 'text'] + 'display_data_priority' : ['latex', 'png', 'jpg', 'jpeg'] }, 'ExtractFigureTransformer': { - 'enabled':True, - 'extra_ext_map':{'svg':'pdf'}, + 'enabled':True } + }) c.merge(super(LatexExporter,self).default_config) return c diff --git a/IPython/nbconvert/exporters/reveal.py b/IPython/nbconvert/exporters/reveal.py index d42f599..c22f56a 100644 --- a/IPython/nbconvert/exporters/reveal.py +++ b/IPython/nbconvert/exporters/reveal.py @@ -45,7 +45,7 @@ class RevealExporter(BasicHTMLExporter): super(RevealExporter, self)._register_transformers() #Register reveal help transformer - self.register_transformer(transformers.RevealHelpTransformer) + self.register_transformer(transformers.RevealHelpTransformer, True) @property def default_config(self): diff --git a/IPython/nbconvert/exporters/sphinx_howto.py b/IPython/nbconvert/exporters/sphinx_howto.py index e92d6c7..29a1999 100644 --- a/IPython/nbconvert/exporters/sphinx_howto.py +++ b/IPython/nbconvert/exporters/sphinx_howto.py @@ -42,13 +42,4 @@ class SphinxHowtoExporter(LatexExporter): super(SphinxHowtoExporter, self)._register_transformers() #Register sphinx latex transformer - self.register_transformer(transformers.SphinxTransformer) - - @property - def default_config(self): - c = Config({ - 'SphinxTransformer': {'enabled':True} - }) - c.merge(super(SphinxHowtoExporter,self).default_config) - return c - + self.register_transformer(transformers.SphinxTransformer, True) diff --git a/IPython/nbconvert/nbconvertapp.py b/IPython/nbconvert/nbconvertapp.py index 031341a..defa671 100755 --- a/IPython/nbconvert/nbconvertapp.py +++ b/IPython/nbconvert/nbconvertapp.py @@ -19,207 +19,145 @@ readme.rst for usage information #Stdlib imports from __future__ import print_function import sys -import io import os +import glob #From IPython -from IPython.config.application import Application -from IPython.utils.traitlets import Bool, Unicode +from IPython.core.application import BaseIPythonApplication +from IPython.config.application import catch_config_error +from IPython.utils.traitlets import Unicode, List, Instance, DottedObjectName, Type +from IPython.utils.importstring import import_item from .exporters.export import export_by_name, get_export_names from .exporters.exporter import Exporter -from .transformers import extractfigure +from .writers.base import WriterBase from .utils.config import GlobalConfigurable #----------------------------------------------------------------------------- -#Globals and constants +#Classes and functions #----------------------------------------------------------------------------- -#'Keys in resources' user prompt. -KEYS_PROMPT_HEAD = "====================== Keys in Resources ==================================" -KEYS_PROMPT_BODY = """ -=========================================================================== -You are responsible for writing these files into the appropriate -directory(ies) if need be. If you do not want to see this message, enable -the 'write' (boolean) flag of the converter. -=========================================================================== -""" +class NbConvertApp(BaseIPythonApplication): + """Application used to convert to and from notebook file type (*.ipynb)""" -_examples = """ -ipython nbconvert rst Untitled0.ipynb # convert ipynb to ReStructured Text -ipython nbconvert latex Untitled0.ipynb # convert ipynb to LaTeX -ipython nbconvert reveal Untitled0.ipynb # convert to Reveal (HTML/JS) slideshow -""" + description = Unicode( + u"""This application is used to convert notebook files (*.ipynb). + An ipython config file can be used to batch convert notebooks in the + current directory.""") -#----------------------------------------------------------------------------- -#Classes and functions -#----------------------------------------------------------------------------- + examples = Unicode(u""" + Running `ipython nbconvert` will read the directory config file and then + apply it to one or more notebooks. -class NbConvertApp(Application): - __doc__ = """IPython notebook conversion utility + Multiple notebooks can be given at the command line in a couple of + different ways: + + > ipython nbconvert notebook*.ipynb + > ipython nbconvert notebook1.ipynb notebook2.ipynb + > ipython nbconvert # this will use the config file to fill in the notebooks + """) -Convert to and from notebook file type (*.ipynb) + config_file_name = Unicode(u'ipython_nbconvert_config.py') - ipython nbconvert TARGET FILENAME + #Writer specific variables + writer = Instance('IPython.nbconvert.writers.base.WriterBase', + help="""Instance of the writer class used to write the + results of the conversion.""") + writer_class = DottedObjectName('FilesWriter', config=True, + help="""Writer class used to write the + results of the conversion""") + writer_aliases = {'FilesWriter': 'IPython.nbconvert.writers.files.FilesWriter', + 'DebugWriter': 'IPython.nbconvert.writers.debug.DebugWriter', + 'StdoutWriter': 'IPython.nbconvert.writers.stdout.StdoutWriter'} + writer_factory = Type() -Supported export TARGETs are: %s -""" % (" ".join(get_export_names())) - description = Unicode(__doc__) + def _writer_class_changed(self, name, old, new): + if new in self.writer_aliases: + new = self.writer_aliases[new] + self.writer_factory = import_item(new) - examples = _examples - stdout = Bool( - False, config=True, - help="""Whether to print the converted IPYNB file to stdout - use full do diff files without actually writing a new file""" - ) + #Other configurable variables + export_format = Unicode( + "", config=True, + help="""If specified, nbconvert will convert the document(s) specified + using this format.""") - write = Bool( - True, config=True, - help="""Should the converted notebook file be written to disk - along with potential extracted resources.""" - ) + notebooks = List([], config=True, help="""List of notebooks to convert. + Search patterns are supported.""") - aliases = { - 'stdout':'NbConvertApp.stdout', - 'write':'NbConvertApp.write', - } + aliases = {'format':'NbConvertApp.export_format', + 'notebooks':'NbConvertApp.notebooks', + 'writer':'NbConvertApp.writer_class'} - flags = {} - flags['stdout'] = ( - {'NbConvertApp' : {'stdout' : True}}, - """Print converted file to stdout, equivalent to --stdout=True - """ - ) + @catch_config_error + def initialize(self, argv=None): + super(NbConvertApp, self).initialize(argv) - flags['no-write'] = ( - {'NbConvertApp' : {'write' : True}}, - """Do not write to disk, equivalent to --write=False - """ - ) + #Register class here to have help with help all + self.classes.insert(0, Exporter) + self.classes.insert(0, WriterBase) + self.classes.insert(0, GlobalConfigurable) + #Init + self.init_config(self.extra_args) + self.init_writer() - def __init__(self, **kwargs): - """Public constructor""" - #Call base class - super(NbConvertApp, self).__init__(**kwargs) + def init_config(self, extra_args): + """ + Add notebooks to the config if needed. Glob each notebook to replace + notebook patterns with filenames. + """ - #Register class here to have help with help all - self.classes.insert(0, Exporter) - self.classes.insert(0, GlobalConfigurable) + #Get any additional notebook patterns from the commandline + if len(extra_args) > 0: + for pattern in extra_args: + self.notebooks.append(pattern) + + #Use glob to replace all the notebook patterns with filenames. + filenames = [] + for pattern in self.notebooks: + for filename in glob.glob(pattern): + if not filename in filenames: + filenames.append(filename) + self.notebooks = filenames + + + def init_writer(self): + """ + Initialize the writer (which is stateless) + """ + self._writer_class_changed(None, self.writer_class, self.writer_class) + self.writer = self.writer_factory(parent=self) def start(self, argv=None): - """Entrypoint of NbConvert application. - - Parameters - ---------- - argv : list - Commandline arguments """ - - #Parse the commandline options. - self.parse_command_line(argv) + Entrypoint of NbConvert application. + """ #Call base super(NbConvertApp, self).start() - #The last arguments in list will be used by nbconvert - if len(self.extra_args) is not 3: - print( "Wrong number of arguments, use --help flag for usage", file=sys.stderr) - sys.exit(-1) - export_type = (self.extra_args)[1] - ipynb_file = (self.extra_args)[2] - - #Export - try: - return_value = export_by_name(export_type, ipynb_file) - except NameError as e: - print("Error: '%s' exporter not found." % export_type, - file=sys.stderr) - print("Known exporters are:", - "\n\t" + "\n\t".join(get_export_names()), - file=sys.stderr) - sys.exit(-1) - else: - (output, resources, exporter) = return_value - - #TODO: Allow user to set output directory and file. - destination_filename = None - destination_directory = None - if self.write: - - #Get the file name without the '.ipynb' (6 chars) extension and then - #remove any addition periods and spaces. The resulting name will - #be used to create the directory that the files will be exported - #into. - out_root = ipynb_file[:-6].replace('.', '_').replace(' ', '_') - destination_filename = os.path.join(out_root+'.'+exporter.file_extension) - - destination_directory = out_root+'_files' - if not os.path.exists(destination_directory): - os.mkdir(destination_directory) - - #Write the results - if self.stdout or not (destination_filename is None and destination_directory is None): - self._write_results(output, resources, destination_filename, destination_directory) - - - def _write_results(self, output, resources, destination_filename=None, destination_directory=None): - """Output the conversion results to the console and/or filesystem - - Parameters - ---------- - output : str - Output of conversion - resources : dictionary - Additional input/output used by the transformers. For - example, the ExtractFigure transformer outputs the - figures it extracts into this dictionary. This method - relies on the figures being in this dictionary when - attempting to write the figures to the file system. - destination_filename : str, Optional - Filename to write output into. If None, output is not - written to a file. - destination_directory : str, Optional - Directory to write notebook data (i.e. figures) to. If - None, figures are not written to the file system. - """ - - if self.stdout: - print(output.encode('utf-8')) - - #Write file output from conversion. - if not destination_filename is None: - with io.open(destination_filename, 'w') as f: - f.write(output) - - #Get the key names used by the extract figure transformer - figures_key = extractfigure.FIGURES_KEY - binary_key = extractfigure.BINARY_KEY - text_key = extractfigure.TEXT_KEY - - #Output any associate figures into the same "root" directory. - binkeys = resources.get(figures_key, {}).get(binary_key,{}).keys() - textkeys = resources.get(figures_key, {}).get(text_key,{}).keys() - if binkeys or textkeys : - if not destination_directory is None: - for key in binkeys: - with io.open(os.path.join(destination_directory, key), 'wb') as f: - f.write(resources[figures_key][binary_key][key]) - for key in textkeys: - with io.open(os.path.join(destination_directory, key), 'w') as f: - f.write(resources[figures_key][text_key][key]) - - #Figures that weren't exported which will need to be created by the - #user. Tell the user what figures these are. - if self.stdout: - print(KEYS_PROMPT_HEAD, file=sys.stderr) - print(resources[figures_key].keys(), file=sys.stderr) - print(KEYS_PROMPT_BODY , file=sys.stderr) + #Export each notebook + for notebook_filename in self.notebooks: + + #Get a unique key for the notebook and set it in the resources object. + basename = os.path.basename(notebook_filename) + notebook_name = basename[:basename.rfind('.')] + resources = {} + resources['unique_key'] = notebook_name + + #Export & write + output, resources = export_by_name(self.export_format, + notebook_filename, + resources=resources, + config=self.config) + self.writer.write(output, resources, notebook_name=notebook_name) + #----------------------------------------------------------------------------- # Main entry point diff --git a/IPython/nbconvert/transformers/__init__.py b/IPython/nbconvert/transformers/__init__.py index c231eb7..fcf906f 100755 --- a/IPython/nbconvert/transformers/__init__.py +++ b/IPython/nbconvert/transformers/__init__.py @@ -1,6 +1,8 @@ # Class base Transformers from .activatable import ActivatableTransformer from .base import ConfigurableTransformer +from .convertfigures import ConvertFiguresTransformer +from .convertsvg import ConvertSvgTransformer from .extractfigure import ExtractFigureTransformer from .revealhelp import RevealHelpTransformer from .latex import LatexTransformer diff --git a/IPython/nbconvert/transformers/base.py b/IPython/nbconvert/transformers/base.py index 233c1f4..2e5a176 100755 --- a/IPython/nbconvert/transformers/base.py +++ b/IPython/nbconvert/transformers/base.py @@ -31,7 +31,7 @@ class ConfigurableTransformer(GlobalConfigurable): Any configurable traitlets this class exposed will be configurable in profiles using c.SubClassName.atribute=value - you can overwrite cell_transform to apply a transformation independently on each cell + you can overwrite transform_cell to apply a transformation independently on each cell or __call__ if you prefer your own logic. See corresponding docstring for informations. """ @@ -59,7 +59,7 @@ class ConfigurableTransformer(GlobalConfigurable): You should return modified nb, resources. If you wish to apply your transform on each cell, you might want to - overwrite cell_transform method instead. + overwrite transform_cell method instead. Parameters ---------- @@ -72,13 +72,13 @@ class ConfigurableTransformer(GlobalConfigurable): try : for worksheet in nb.worksheets : for index, cell in enumerate(worksheet.cells): - worksheet.cells[index], resources = self.cell_transform(cell, resources, index) + worksheet.cells[index], resources = self.transform_cell(cell, resources, index) return nb, resources except NotImplementedError: raise NotImplementedError('should be implemented by subclass') - def cell_transform(self, cell, resources, index): + def transform_cell(self, cell, resources, index): """ Overwrite if you want to apply a transformation on each cell. You should return modified cell and resource dictionary. diff --git a/IPython/nbconvert/transformers/convertfigures.py b/IPython/nbconvert/transformers/convertfigures.py new file mode 100644 index 0000000..f8098e9 --- /dev/null +++ b/IPython/nbconvert/transformers/convertfigures.py @@ -0,0 +1,74 @@ +"""Module containing a transformer that converts outputs in the notebook from +one format to another. +""" +#----------------------------------------------------------------------------- +# Copyright (c) 2013, the IPython Development Team. +# +# Distributed under the terms of the Modified BSD License. +# +# The full license is in the file COPYING.txt, distributed with this software. +#----------------------------------------------------------------------------- + +#----------------------------------------------------------------------------- +# Imports +#----------------------------------------------------------------------------- + +from .activatable import ActivatableTransformer + +#----------------------------------------------------------------------------- +# Classes +#----------------------------------------------------------------------------- + +class ConvertFiguresTransformer(ActivatableTransformer): + """ + Converts all of the outputs in a notebook from one format to another. + """ + + + def __init__(self, from_formats, to_format, **kw): + """ + Public constructor + + Parameters + ---------- + from_formats : list [of string] + Formats that the converter can convert from + to_format : string + Format that the converter converts to + config : Config + Configuration file structure + **kw : misc + Additional arguments + """ + super(ConvertFiguresTransformer, self).__init__(**kw) + + self._from_formats = from_formats + self._to_format = to_format + + + def convert_figure(self, data_format, data): + raise NotImplementedError() + + + def transform_cell(self, cell, resources, cell_index): + """ + Apply a transformation on each cell, + + See base.py + """ + + #Loop through all of the datatypes of the outputs in the cell. + for index, cell_out in enumerate(cell.get('outputs', [])): + for data_type, data in cell_out.items(): + self._convert_figure(cell_out, data_type, data) + return cell, resources + + + def _convert_figure(self, cell_out, data_type, data): + """ + Convert a figure and output the results to the cell output + """ + + if not self._to_format in cell_out: + if data_type in self._from_formats: + cell_out[self._to_format] = self.convert_figure(data_type, data) diff --git a/IPython/nbconvert/transformers/convertsvg.py b/IPython/nbconvert/transformers/convertsvg.py new file mode 100644 index 0000000..02bebbc --- /dev/null +++ b/IPython/nbconvert/transformers/convertsvg.py @@ -0,0 +1,70 @@ +"""Module containing a transformer that converts outputs in the notebook from +one format to another. +""" +#----------------------------------------------------------------------------- +# Copyright (c) 2013, the IPython Development Team. +# +# Distributed under the terms of the Modified BSD License. +# +# The full license is in the file COPYING.txt, distributed with this software. +#----------------------------------------------------------------------------- + +#----------------------------------------------------------------------------- +# Imports +#----------------------------------------------------------------------------- + +import os +from IPython.utils.tempdir import TemporaryDirectory + +from .convertfigures import ConvertFiguresTransformer + + +#----------------------------------------------------------------------------- +# Constants +#----------------------------------------------------------------------------- + +INKSCAPE_COMMAND = "inkscape --without-gui --export-pdf=\"{to_filename}\" \"{from_filename}\"" + + +#----------------------------------------------------------------------------- +# Classes +#----------------------------------------------------------------------------- + +class ConvertSvgTransformer(ConvertFiguresTransformer): + """ + Converts all of the outputs in a notebook from one format to another. + """ + + + def __init__(self, **kw): + """ + Constructor + """ + super(ConvertSvgTransformer, self).__init__(['svg'], 'pdf', **kw) + + + def convert_figure(self, data_format, data): + """ + Convert a single Svg figure. Returns converted data. + """ + + #Work in a temporary directory + with TemporaryDirectory() as tmpdir: + + #Write fig to temp file + input_filename = os.path.join(tmpdir, 'figure.' + data_format) + with open(input_filename, 'w') as f: + f.write(data) + + #Call conversion application + output_filename = os.path.join(tmpdir, 'figure.pdf') + shell = INKSCAPE_COMMAND.format(from_filename=input_filename, + to_filename=output_filename) + subprocess.call(shell, shell=True) #Shell=True okay since input is trusted. + + #Read output from drive + if os.path.isfile(output_filename): + with open(output_filename) as f: + return f.read() + else: + return TypeError("Inkscape svg to png conversion failed") diff --git a/IPython/nbconvert/transformers/extractfigure.py b/IPython/nbconvert/transformers/extractfigure.py index 5e3021b..0df9814 100755 --- a/IPython/nbconvert/transformers/extractfigure.py +++ b/IPython/nbconvert/transformers/extractfigure.py @@ -12,7 +12,6 @@ notebook file. The extracted figures are returned in the 'resources' dictionary #----------------------------------------------------------------------------- # Imports #----------------------------------------------------------------------------- -import itertools from IPython.utils.traitlets import Dict, Unicode from .activatable import ActivatableTransformer @@ -21,9 +20,10 @@ from .activatable import ActivatableTransformer # Constants #----------------------------------------------------------------------------- -FIGURES_KEY = "figures" -BINARY_KEY = "binary" -TEXT_KEY = "text" +# win64 is win32 for backwards compatability, for now. See +# http://mail.python.org/pipermail/patches/2000-May/000648.html +# for the original patch that this decision was made. +WINDOWS_PLATFORMS = ['win32'] #----------------------------------------------------------------------------- # Classes @@ -35,36 +35,12 @@ class ExtractFigureTransformer(ActivatableTransformer): figures are returned in the 'resources' dictionary. """ - extra_extension_map = Dict({}, - config=True, - help="""Extra map to override extension based on type. - Useful for latex where SVG will be converted to PDF before inclusion - """) - - key_format_map = Dict({}, config=True,) - figure_name_format_map = Dict({}, config=True) - #TODO: Change this to .format {} syntax - default_key_template = Unicode('_fig_{index:02d}.{ext}', config=True) + figure_filename_template = Unicode( + "{unique_key}_{cell_index}_{index}.{extension}", config=True) - def __init__(self, config=None, **kw): - """ - Public constructor - - Parameters - ---------- - config : Config - Configuration file structure - **kw : misc - Additional arguments - """ - - super(ExtractFigureTransformer, self).__init__(config=config, **kw) - - # A unique index for association with extracted figures - self.index_generator = itertools.count(1) - def cell_transform(self, cell, resources, index): + def transform_cell(self, cell, resources, cell_index): """ Apply a transformation on each cell, @@ -75,69 +51,47 @@ class ExtractFigureTransformer(ActivatableTransformer): resources : dictionary Additional resources used in the conversion process. Allows transformers to pass variables into the Jinja engine. - index : int + cell_index : int Index of the cell being processed (see base.py) """ + + #Get the unique key from the resource dict if it exists. If it does not + #exist, use 'figure' as the default. + unique_key = resources.get('unique_key', 'figure') - if resources.get(FIGURES_KEY, None) is None : - resources[FIGURES_KEY] = {TEXT_KEY:{},BINARY_KEY:{}} + #Make sure figures key exists + if not 'figures' in resources: + resources['figures'] = {} - for out in cell.get('outputs', []): + #Loop through all of the outputs in the cell + for index, out in enumerate(cell.get('outputs', [])): + + #Get the output in data formats that the template is interested in. for out_type in self.display_data_priority: - if out.hasattr(out_type): - figname, key, data, binary = self._new_figure(out[out_type], out_type) - out['key_'+out_type] = figname - - if binary : - resources[FIGURES_KEY][BINARY_KEY][key] = data - else : - resources[FIGURES_KEY][TEXT_KEY][key] = data - - index += 1 - return cell, resources - - - def _get_override_extension(self, extension): - """Gets the overriden extension if it exists, else returns extension. + data = out[out_type] - Parameters - ---------- - extension : str - File extension. - """ - - if extension in self.extra_extension_map : - return self.extra_extension_map[extension] - - return extension - - - def _new_figure(self, data, format): - """Create a new figure file in the given format. + #Binary files are base64-encoded, SVG is already XML + if out_type in ('png', 'jpg', 'pdf'): + data = data.decode('base64') + elif sys.platform in WINDOWS_PLATFORMS: + data = data.replace('\n', '\r\n') + + #Build a figure name + figure_name = self.figure_filename_template.format( + unique_key=unique_key, + cell_index=cell_index, + index=index, + extension=out_type) + + #On the cell, make the figure available via + # cell.outputs[i].svg_filename ... etc (svg in example) + # Where + # cell.outputs[i].svg contains the data + out[out_type + '_filename'] = figure_name + + #In the resources, make the figure available via + # resources['figures']['filename'] = data + resources['figures'][figure_name] = data - Parameters - ---------- - data : str - Cell data (from Notebook node cell) - format : str - Figure format - index : int - Index of the figure being extracted - """ - - figure_name_template = self.figure_name_format_map.get(format, self.default_key_template) - key_template = self.key_format_map.get(format, self.default_key_template) - - #TODO: option to pass the hash as data? - index = next(self.index_generator) - figure_name = figure_name_template.format(index=index, ext=self._get_override_extension(format)) - key = key_template.format(index=index, ext=self._get_override_extension(format)) - - #Binary files are base64-encoded, SVG is already XML - binary = False - if format in ('png', 'jpg', 'pdf'): - data = data.decode('base64') - binary = True - - return figure_name, key, data, binary + return cell, resources diff --git a/IPython/nbconvert/transformers/latex.py b/IPython/nbconvert/transformers/latex.py index 5f856ec..5718b57 100755 --- a/IPython/nbconvert/transformers/latex.py +++ b/IPython/nbconvert/transformers/latex.py @@ -29,7 +29,7 @@ class LatexTransformer(ActivatableTransformer): Converter for latex destined documents. """ - def cell_transform(self, cell, resources, index): + def transform_cell(self, cell, resources, index): """ Apply a transformation on each cell, diff --git a/IPython/nbconvert/transformers/revealhelp.py b/IPython/nbconvert/transformers/revealhelp.py index 38cfa0e..8bdc65c 100755 --- a/IPython/nbconvert/transformers/revealhelp.py +++ b/IPython/nbconvert/transformers/revealhelp.py @@ -39,9 +39,8 @@ class RevealHelpTransformer(ConfigurableTransformer): transformers to pass variables into the Jinja engine. """ - for worksheet in nb.worksheets : - for i, cell in enumerate(worksheet.cells): + for index, cell in enumerate(worksheet.cells): #Make sure the cell has slideshow metadata. cell.metadata.align_type = cell.get('metadata', {}).get('slideshow', {}).get('align_type', 'Left') @@ -50,9 +49,9 @@ class RevealHelpTransformer(ConfigurableTransformer): #Get the slide type. If type is start of subslide or slide, #end the last subslide/slide. if cell.metadata.slide_type in ['slide']: - worksheet.cells[i - 1].metadata.slide_helper = 'slide_end' + worksheet.cells[index - 1].metadata.slide_helper = 'slide_end' if cell.metadata.slide_type in ['subslide']: - worksheet.cells[i - 1].metadata.slide_helper = 'subslide_end' + worksheet.cells[index - 1].metadata.slide_helper = 'subslide_end' if 'reveal' not in resources: