Merge pull request #3846 from minrk/nbconvert_log

Add basic logging to nbconvert

IPython/nbconvert/exporters/export.py

@@ -85,7 +85,6 __all__ = [
 class ExporterNameError(NameError):
     pass
 
-
 @DocDecorator
 def export(exporter, nb, **kw):
     """
@@ -112,7 +111,7 def export(exporter, nb, **kw):
         exporter_instance = exporter
     else:
         exporter_instance = exporter(**kw)
-
+    
     #Try to convert the notebook using the appropriate conversion function.
     if isinstance(nb, NotebookNode):
         output, resources = exporter_instance.from_notebook_node(nb, resources)
@@ -122,62 +121,27 def export(exporter, nb, **kw):
         output, resources = exporter_instance.from_file(nb, resources)
     return output, resources
 
+exporter_map = dict(
+    custom=Exporter,
+    html=HTMLExporter,
+    slides=SlidesExporter,
+    latex=LatexExporter,
+    markdown=MarkdownExporter,
+    python=PythonExporter,
+    rst=RSTExporter,
+)
+
+def _make_exporter(name, E):
+    """make an export_foo function from a short key and Exporter class E"""
+    def _export(nb, **kw):
+        return export(E, nb, **kw)
+    _export.__doc__ = """Export a notebook object to {0} format""".format(name)
+    return _export
+    
+g = globals()
 
-@DocDecorator
-def export_custom(nb, **kw):
-    """
-    Export a notebook object to a custom format
-    """
-    return export(Exporter, nb, **kw)
-
-
-@DocDecorator
-def export_html(nb, **kw):
-    """
-    Export a notebook object to HTML
-    """
-    return export(HTMLExporter, nb, **kw)
-
-
-@DocDecorator
-def export_slides(nb, **kw):
-    """
-    Export a notebook object to Slides
-    """
-    return export(SlidesExporter, nb, **kw)
-
-
-@DocDecorator
-def export_latex(nb, **kw):
-    """
-    Export a notebook object to LaTeX
-    """
-    return export(LatexExporter, nb, **kw)
-
-
-@DocDecorator
-def export_markdown(nb, **kw):
-    """
-    Export a notebook object to Markdown
-    """
-    return export(MarkdownExporter, nb, **kw)
-
-
-@DocDecorator
-def export_python(nb, **kw):
-    """
-    Export a notebook object to Python
-    """
-    return export(PythonExporter, nb, **kw)
-
-
-@DocDecorator
-def export_rst(nb, **kw):
-    """
-    Export a notebook object to reStructuredText
-    """
-    return export(RSTExporter, nb, **kw)
-
+for name, E in exporter_map.items():
+    g['export_%s' % name] = DocDecorator(_make_exporter(name, E))
 
 @DocDecorator
 def export_by_name(format_name, nb, **kw):
@@ -202,10 +166,4 def get_export_names():
     """Return a list of the currently supported export targets
 
     WARNING: API WILL CHANGE IN FUTURE RELEASES OF NBCONVERT"""
-    
-    # grab everything after 'export_'
-    l = [x[len('export_'):] for x in __all__ if x.startswith('export_')]
-    
-    # filter out the one method that is not a template
-    l = [x for x in l if 'by_name' not in x]
-    return sorted(l)
+    return sorted(exporter_map.keys())

IPython/nbconvert/exporters/exporter.py

@@ -28,10 +28,10 import datetime
 from jinja2 import Environment, FileSystemLoader, ChoiceLoader, TemplateNotFound
 
 # IPython imports
-from IPython.config.configurable import Configurable
+from IPython.config.configurable import LoggingConfigurable
 from IPython.config import Config
 from IPython.nbformat import current as nbformat
-from IPython.utils.traitlets import MetaHasTraits, DottedObjectName, Unicode, List, Dict
+from IPython.utils.traitlets import MetaHasTraits, DottedObjectName, Unicode, List, Dict, Any
 from IPython.utils.importstring import import_item
 from IPython.utils.text import indent
 from IPython.utils import py3compat
@@ -78,7 +78,7 class ResourcesDict(collections.defaultdict):
         return ''
 
 
-class Exporter(Configurable):
+class Exporter(LoggingConfigurable):
     """
     Exports notebooks into other file formats.  Uses Jinja 2 templating engine
     to output new formats.  Inherit from this class if you are creating a new
@@ -102,7 +102,12 class Exporter(Configurable):
             self.template_file = self.default_template
         else:
             self.template_file = new
+        self.template = None
+        self._load_template()
+    
     default_template = Unicode(u'')
+    template = Any()
+    environment = Any()
 
     file_extension = Unicode(
         'txt', config=True, 
@@ -110,6 +115,8 class Exporter(Configurable):
         )
 
     template_path = List(['.'], config=True)
+    def _template_path_changed(self, name, old, new):
+        self._load_template()
 
     default_template_path = Unicode(
         os.path.join("..", "templates"), 
@@ -159,21 +166,18 class Exporter(Configurable):
         config : config
             User configuration instance.
         extra_loaders : list[of Jinja Loaders]
-            ordered list of Jinja loder to find templates. Will be tried in order
-            before the default FileSysteme ones.
+            ordered list of Jinja loader to find templates. Will be tried in order
+            before the default FileSystem ones.
         template : str (optional, kw arg)
             Template to use when exporting.
         """
+        if not config:
+            config = self.default_config
         
-        #Call the base class constructor
-        c = self.default_config
-        if config:
-            c.merge(config)
-
-        super(Exporter, self).__init__(config=c, **kw)
+        super(Exporter, self).__init__(config=config, **kw)
 
         #Init
-        self._init_template(**kw)
+        self._init_template()
         self._init_environment(extra_loaders=extra_loaders)
         self._init_transformers()
         self._init_filters()
@@ -182,7 +186,48 class Exporter(Configurable):
     @property
     def default_config(self):
         return Config()
+    
+    def _config_changed(self, name, old, new):
+        """When setting config, make sure to start with our default_config"""
+        c = self.default_config
+        if new:
+            c.merge(new)
+        if c != old:
+            self.config = c
+        super(Exporter, self)._config_changed(name, old, c)
+        
 
+    def _load_template(self):
+        """Load the Jinja template object from the template file
+        
+        This is a no-op if the template attribute is already defined,
+        or the Jinja environment is not setup yet.
+        
+        This is triggered by various trait changes that would change the template.
+        """
+        if self.template is not None:
+            return
+        # called too early, do nothing
+        if self.environment is None:
+            return
+        # Try different template names during conversion.  First try to load the
+        # template by name with extension added, then try loading the template
+        # as if the name is explicitly specified, then try the name as a 
+        # 'flavor', and lastly just try to load the template by module name.
+        module_name = self.__module__.rsplit('.', 1)[-1]
+        try_names = [self.template_file + self.template_extension,
+                     self.template_file,
+                     module_name + '_' + self.template_file + self.template_extension,
+                     module_name + self.template_extension]
+        for try_name in try_names:
+            self.log.debug("Attempting to load template %s", try_name)
+            try:
+                self.template = self.environment.get_template(try_name)
+            except TemplateNotFound:
+                pass
+            else:
+                self.log.info("Loaded template %s", try_name)
+                break
     
     def from_notebook_node(self, nb, resources=None, **kw):
         """
@@ -201,23 +246,9 class Exporter(Configurable):
         # Preprocess
         nb_copy, resources = self._transform(nb_copy, resources)
 
-        # Try different template names during conversion.  First try to load the
-        # template by name with extension added, then try loading the template
-        # as if the name is explicitly specified, then try the name as a 
-        # 'flavor', and lastly just try to load the template by module name.
-        module_name = self.__module__.split('.')[-1]
-        try_names = [self.template_file + self.template_extension,
-                     self.template_file,
-                     module_name + '_' + self.template_file + self.template_extension,
-                     module_name + self.template_extension]
-        for try_name in try_names:
-            try:
-                self.template = self.environment.get_template(try_name)
-                break
-            except TemplateNotFound:
-                pass
+        self._load_template()
 
-        if hasattr(self, 'template'):
+        if self.template is not None:
             output = self.template.render(nb=nb_copy, resources=resources)
         else:
             raise IOError('template file "%s" could not be found' % self.template_file)
@@ -355,14 +386,12 class Exporter(Configurable):
             raise TypeError('filter')
 
         
-    def _init_template(self, **kw):
+    def _init_template(self):
         """
         Make sure a template name is specified.  If one isn't specified, try to
         build one from the information we know.
         """
         self._template_file_changed('template_file', self.template_file, self.template_file)
-        if 'template' in kw:
-            self.template_file = kw['template']
         
 
     def _init_environment(self, extra_loaders=None):

IPython/nbconvert/exporters/tests/test_exporter.py

@@ -95,7 +95,7 class TestExporter(ExportersTestsBase):
 
     def test_transformer_via_method(self):
         """
-        Can a transformer be added via the Exporter convinience method?
+        Can a transformer be added via the Exporter convenience method?
         """
         exporter = self._make_exporter()
         exporter.register_transformer(CheeseTransformer, enabled=True)
@@ -108,6 +108,5 class TestExporter(ExportersTestsBase):
     def _make_exporter(self, config=None):
         #Create the exporter instance, make sure to set a template name since
         #the base Exporter doesn't have a template associated with it.
-        exporter = Exporter(config=config)
-        exporter.template_file = 'python'
+        exporter = Exporter(config=config, template_file='python')
         return exporter        
  No newline at end of file

IPython/nbconvert/exporters/tests/test_html.py

@@ -46,7 +46,7 class TestHTMLExporter(ExportersTestsBase):
         """
         Can a HTMLExporter export using the 'basic' template?
         """
-        (output, resources) = HTMLExporter(template='basic').from_filename(self._get_notebook())
+        (output, resources) = HTMLExporter(template_file='basic').from_filename(self._get_notebook())
         assert len(output) > 0
 
 
@@ -55,5 +55,5 class TestHTMLExporter(ExportersTestsBase):
         """
         Can a HTMLExporter export using the 'full' template?
         """
-        (output, resources) = HTMLExporter(template='full').from_filename(self._get_notebook())
+        (output, resources) = HTMLExporter(template_file='full').from_filename(self._get_notebook())
         assert len(output) > 0

IPython/nbconvert/exporters/tests/test_latex.py

@@ -46,7 +46,7 class TestLatexExporter(ExportersTestsBase):
         """
         Can a LatexExporter export using 'book' template?
         """
-        (output, resources) = LatexExporter(template='book').from_filename(self._get_notebook())
+        (output, resources) = LatexExporter(template_file='book').from_filename(self._get_notebook())
         assert len(output) > 0
 
 
@@ -55,7 +55,7 class TestLatexExporter(ExportersTestsBase):
         """
         Can a LatexExporter export using 'basic' template?
         """
-        (output, resources) = LatexExporter(template='basic').from_filename(self._get_notebook())
+        (output, resources) = LatexExporter(template_file='basic').from_filename(self._get_notebook())
         assert len(output) > 0
 
 
@@ -64,5 +64,5 class TestLatexExporter(ExportersTestsBase):
         """
         Can a LatexExporter export using 'article' template?
         """
-        (output, resources) = LatexExporter(template='article').from_filename(self._get_notebook())
+        (output, resources) = LatexExporter(template_file='article').from_filename(self._get_notebook())
         assert len(output) > 0
  No newline at end of file

IPython/nbconvert/exporters/tests/test_slides.py

@@ -46,5 +46,5 class TestSlidesExporter(ExportersTestsBase):
         """
         Can a SlidesExporter export using the 'reveal' template?
         """
-        (output, resources) = SlidesExporter(template='reveal').from_filename(self._get_notebook())
+        (output, resources) = SlidesExporter(template_file='reveal').from_filename(self._get_notebook())
         assert len(output) > 0

IPython/nbconvert/nbconvertapp.py

@@ -17,6 +17,8 Command-line interface for the NbConvert conversion utility.
 
 # Stdlib imports
 from __future__ import print_function
+
+import logging
 import sys
 import os
 import glob
@@ -30,7 +32,7 from IPython.utils.traitlets import (
 from IPython.utils.importstring import import_item
 from IPython.utils.text import dedent
 
-from .exporters.export import export_by_name, get_export_names, ExporterNameError
+from .exporters.export import get_export_names, exporter_map
 from IPython.nbconvert import exporters, transformers, writers, post_processors
 from .utils.base import NbConvertBase
 from .utils.exceptions import ConversionException
@@ -80,6 +82,9 class NbConvertApp(BaseIPythonApplication):
     aliases = nbconvert_aliases
     flags = nbconvert_flags
     
+    def _log_level_default(self):
+        return logging.INFO
+    
     def _classes_default(self):
         classes = [NbConvertBase]
         for pkg in (exporters, transformers, writers):
@@ -229,6 +234,8 class NbConvertApp(BaseIPythonApplication):
             # notebooks without having to type the extension.
             globbed_files = glob.glob(pattern)
             globbed_files.extend(glob.glob(pattern + '.ipynb'))
+            if not globbed_files:
+                self.log.warn("pattern %r matched no files", pattern)
 
             for filename in globbed_files:
                 if not filename in filenames:
@@ -266,13 +273,16 class NbConvertApp(BaseIPythonApplication):
         conversion_success = 0
 
         if self.output_base != '' and len(self.notebooks) > 1:
-            print(dedent(
+            self.log.error(
             """UsageError: --output flag or `NbConvertApp.output_base` config option
             cannot be used when converting multiple notebooks.
-            """))
+            """)
             self.exit(1)
+        
+        exporter = exporter_map[self.export_format](config=self.config)
 
         for notebook_filename in self.notebooks:
+            self.log.info("Converting notebook %s to %s", notebook_filename, self.export_format)
 
             # Get a unique key for the notebook and set it in the resources object.
             basename = os.path.basename(notebook_filename)
@@ -282,24 +292,14 class NbConvertApp(BaseIPythonApplication):
             resources = {}
             resources['unique_key'] = notebook_name
             resources['output_files_dir'] = '%s_files' % notebook_name
+            self.log.info("Support files will be in %s", os.path.join(resources['output_files_dir'], ''))
 
             # Try to export
             try:
-                output, resources = export_by_name(self.export_format,
-                                              notebook_filename, 
-                                              resources=resources,
-                                              config=self.config)
-            except ExporterNameError as e:
-                print("Error while converting '%s': '%s' exporter not found."
-                      %(notebook_filename, self.export_format),
-                      file=sys.stderr)
-                print("Known exporters are:",
-                      "\n\t" + "\n\t".join(get_export_names()),
-                      file=sys.stderr)
-                self.exit(1)
+                output, resources = exporter.from_filename(notebook_filename, resources=resources)
             except ConversionException as e:
-                print("Error while converting '%s': %s" %(notebook_filename, e),
-                      file=sys.stderr)
+                self.log.error("Error while converting '%s'", notebook_filename,
+                      exc_info=True)
                 self.exit(1)
             else:
                 write_resultes = self.writer.write(output, resources, notebook_name=notebook_name)

IPython/nbconvert/post_processors/pdf.py

@@ -43,6 +43,7 class PDFPostProcessor(PostProcessorBase):
             See files.py for more...
             """        
             command = self.compiler.format(input)
+            self.log.info("Building PDF: `%s`", command)
             for index in range(self.iteration_count):
                 if self.verbose:
                     subprocess.Popen(command, shell=True)

IPython/nbconvert/transformers/base.py

@@ -79,8 +79,9 class Transformer(NbConvertBase):
             Additional resources used in the conversion process.  Allows
             transformers to pass variables into the Jinja engine.
         """
+        self.log.debug("Applying transform: %s", self.__class__.__name__)
         try :
-            for worksheet in nb.worksheets :
+            for worksheet in nb.worksheets:
                 for index, cell in enumerate(worksheet.cells):
                     worksheet.cells[index], resources = self.transform_cell(cell, resources, index)
             return nb, resources

IPython/nbconvert/utils/base.py

@@ -12,13 +12,13
 #-----------------------------------------------------------------------------
 
 from IPython.utils.traitlets import List
-from IPython.config.configurable import Configurable
+from IPython.config.configurable import LoggingConfigurable
 
 #-----------------------------------------------------------------------------
 # Classes and functions
 #-----------------------------------------------------------------------------
            
-class NbConvertBase(Configurable):
+class NbConvertBase(LoggingConfigurable):
     """Global configurable class for shared config
 
     Usefull for display data priority that might be use by many trasnformers

IPython/nbconvert/writers/files.py

@@ -45,7 +45,12 class FilesWriter(WriterBase):
         super(FilesWriter, self).__init__(**kw)
         self._build_directory_changed('build_directory', self.build_directory, 
                                       self.build_directory)
-
+    
+    def _makedir(self, path):
+        """Make a directory if it doesn't already exist"""
+        if not os.path.isdir(path):
+            self.log.info("Making directory %s", path)
+            os.makedirs(path)
 
     def write(self, output, resources, notebook_name=None, **kw):
             """
@@ -67,10 +72,10 class FilesWriter(WriterBase):
                 # Determine where to write the file to
                 dest = os.path.join(self.build_directory, filename)
                 path = os.path.dirname(dest)
-                if not os.path.isdir(path):
-                    os.makedirs(path)
+                self._makedir(path)
 
                 # Write file
+                self.log.debug("Writing %i bytes to support file %s", len(data), dest)
                 with io.open(dest, 'wb') as f:
                     f.write(data)
 
@@ -84,11 +89,11 class FilesWriter(WriterBase):
                         # Make sure folder exists.
                         dest = os.path.join(self.build_directory, filename)
                         path = os.path.dirname(dest)
-                        if not os.path.isdir(path):
-                            os.makedirs(path)
+                        self._makedir(path)
 
                         # Copy if destination is different.
                         if not os.path.normpath(dest) == os.path.normpath(matching_filename):
+                            self.log.info("Linking %s -> %s", matching_filename, dest)
                             link_or_copy(matching_filename, dest)
 
             # Determine where to write conversion results.
@@ -97,6 +102,7 class FilesWriter(WriterBase):
                 dest = os.path.join(self.build_directory, dest)
 
             # Write conversion results.
+            self.log.info("Writing %i bytes to %s", len(output), dest)
             with io.open(dest, 'w') as f:
                 f.write(output)
             return dest
  No newline at end of file