Merge pull request #3734 from jdfreder/file_subdir

Nbconvert: Export extracted files into `nbname_files` subdirectory

Default build directory changed to .
Files extracted from notebook now are placed into a ./notebook_files/ directory by default
Spaces added between # symbols and comments
It may be best to look at the diffs of the individual commits... The addition of spaces in the comments makes the overall diff hard to read.

IPython/nbconvert/exporters/exporter.py

@@ -131,7 +131,7 b' class Exporter(Configurable):'
 
     default_transformers = List([nbtransformers.coalesce_streams,
                                  nbtransformers.SVG2PDFTransformer,
-                                 nbtransformers.ExtractFigureTransformer,
+                                 nbtransformers.ExtractOutputTransformer,
                                  nbtransformers.CSSHTMLHeaderTransformer,
                                  nbtransformers.RevealHelpTransformer,
                                  nbtransformers.LatexTransformer,

IPython/nbconvert/exporters/latex.py

@@ -88,7 +88,7 b' class LatexExporter(Exporter):'
             'NbConvertBase': {
                 'display_data_priority' : ['latex', 'pdf', 'png', 'jpg', 'svg', 'jpeg', 'text']
                 },
-             'ExtractFigureTransformer': {
+             'ExtractOutputTransformer': {
                     'enabled':True
                  },
              'SVG2PDFTransformer': {

IPython/nbconvert/exporters/rst.py

@@ -37,6 +37,6 b' class RSTExporter(Exporter):'
 
     @property
     def default_config(self):
-        c = Config({'ExtractFigureTransformer':{'enabled':True}})
+        c = Config({'ExtractOutputTransformer':{'enabled':True}})
         c.merge(super(RSTExporter,self).default_config)
         return c

IPython/nbconvert/exporters/tests/test_exporter.py

@@ -45,16 +45,16 b' class TestExporter(ExportersTestsBase):'
         assert len(output) > 0
 
 
-    def test_extract_figures(self):
+    def test_extract_outputs(self):
         """
-        If the ExtractFigureTransformer is enabled, are figures extracted?
+        If the ExtractOutputTransformer is enabled, are outputs extracted?
         """
-        config = Config({'ExtractFigureTransformer': {'enabled': True}})
+        config = Config({'ExtractOutputTransformer': {'enabled': True}})
         exporter = self._make_exporter(config=config)
         (output, resources) = exporter.from_filename(self._get_notebook())
         assert resources is not None
-        assert 'figures' in resources
-        assert len(resources['figures']) > 0
+        assert 'outputs' in resources
+        assert len(resources['outputs']) > 0
 
 
     def test_transformer_class(self):

IPython/nbconvert/nbconvertapp.py

@@ -15,13 +15,13 b' Command-line interface for the NbConvert conversion utility.'
 #Imports
 #-----------------------------------------------------------------------------
 
-#Stdlib imports
+# Stdlib imports
 from __future__ import print_function
 import sys
 import os
 import glob
 
-#From IPython
+# From IPython
 from IPython.core.application import BaseIPythonApplication, base_aliases, base_flags
 from IPython.config import catch_config_error, Configurable
 from IPython.utils.traitlets import (
@@ -103,7 +103,7 b' class NbConvertApp(BaseIPythonApplication):'
         
         > ipython nbconvert --config mycfg.py
         """.format(get_export_names()))
-    #Writer specific variables
+    # Writer specific variables
     writer = Instance('IPython.nbconvert.writers.base.WriterBase',  
                       help="""Instance of the writer class used to write the 
                       results of the conversion.""")
@@ -121,7 +121,7 b' class NbConvertApp(BaseIPythonApplication):'
         self.writer_factory = import_item(new)
 
 
-    #Other configurable variables
+    # Other configurable variables
     export_format = CaselessStrEnum(get_export_names(),
         default_value="full_html",
         config=True,
@@ -153,7 +153,7 b' class NbConvertApp(BaseIPythonApplication):'
         else:
             patterns = self.notebooks
 
-        #Use glob to replace all the notebook patterns with filenames.
+        # Use glob to replace all the notebook patterns with filenames.
         filenames = []
         for pattern in patterns:
             for filename in glob.glob(pattern):
@@ -179,17 +179,18 b' class NbConvertApp(BaseIPythonApplication):'
         """
         Convert the notebooks in the self.notebook traitlet
         """
-        #Export each notebook
+        # Export each notebook
         conversion_success = 0
         for notebook_filename in self.notebooks:
 
-            #Get a unique key for the notebook and set it in the resources object.
+            # Get a unique key for the notebook and set it in the resources object.
             basename = os.path.basename(notebook_filename)
             notebook_name = basename[:basename.rfind('.')]
             resources = {}
             resources['unique_key'] = notebook_name
+            resources['output_files_dir'] = '%s_files' % notebook_name
 
-            #Try to export
+            # Try to export
             try:
                 output, resources = export_by_name(self.export_format,
                                               notebook_filename, 
@@ -202,22 +203,22 b' class NbConvertApp(BaseIPythonApplication):'
                       "\n\t" + "\n\t".join(get_export_names()),
                       file=sys.stderr)
                 sys.exit(-1)
-            #except Exception as e:
-                #print("Error: could not export '%s'" % notebook_filename, file=sys.stderr)
-                #print(e, file=sys.stderr)
+            # except Exception as e:
+                # print("Error: could not export '%s'" % notebook_filename, file=sys.stderr)
+                # print(e, file=sys.stderr)
             else:
                 self.writer.write(output, resources, notebook_name=notebook_name)
                 conversion_success += 1
 
-        #If nothing was converted successfully, help the user.
+        # If nothing was converted successfully, help the user.
         if conversion_success == 0:
 
-            #No notebooks were specified, show help.
+            # No notebooks were specified, show help.
             if len(self.notebooks) == 0:
                 self.print_help()
 
-            #Notebooks were specified, but not converted successfully.  Show how
-            #to access help.
+            # Notebooks were specified, but not converted successfully.  Show how
+            # to access help.
             else:
                 print('For help, use "ipython nbconvert --help"')
 

IPython/nbconvert/tests/test_nbconvertapp.py

@@ -23,7 +23,7 b' from IPython.utils import py3compat'
 # Constants
 #-----------------------------------------------------------------------------
 
-#Define ipython commandline name
+# Define ipython commandline name
 if py3compat.PY3:
     IPYTHON = 'ipython3'
 else:
@@ -53,8 +53,8 b' class TestNbConvertApp(TestsBase):'
         with self.create_temp_cwd(['notebook*.ipynb']):
             assert not 'error' in self.call([IPYTHON, 'nbconvert', 
                 '--format="python"', '--notebooks=["*.ipynb"]']).lower()
-            assert os.path.isfile(os.path.join('nbconvert_build', 'notebook1.py'))
-            assert os.path.isfile(os.path.join('nbconvert_build', 'notebook2.py'))
+            assert os.path.isfile('notebook1.py')
+            assert os.path.isfile('notebook2.py')
 
 
     def test_glob_subdir(self):
@@ -65,8 +65,8 b' class TestNbConvertApp(TestsBase):'
             self.copy_files_to(['notebook*.ipynb'], 'subdir/')
             assert not 'error' in self.call([IPYTHON, 'nbconvert', '--format="python"', 
                 '--notebooks=["%s"]' % os.path.join('subdir', '*.ipynb')]).lower()
-            assert os.path.isfile(os.path.join('nbconvert_build', 'notebook1.py'))
-            assert os.path.isfile(os.path.join('nbconvert_build', 'notebook2.py'))
+            assert os.path.isfile('notebook1.py')
+            assert os.path.isfile('notebook2.py')
 
 
     def test_explicit(self):
@@ -76,8 +76,8 b' class TestNbConvertApp(TestsBase):'
         with self.create_temp_cwd(['notebook*.ipynb']):
             assert not 'error' in self.call([IPYTHON, 'nbconvert', '--format="python"', 
                 '--notebooks=["notebook2.ipynb"]']).lower()
-            assert not os.path.isfile(os.path.join('nbconvert_build', 'notebook1.py'))
-            assert os.path.isfile(os.path.join('nbconvert_build', 'notebook2.py'))
+            assert not os.path.isfile('notebook1.py')
+            assert os.path.isfile('notebook2.py')
 
 
     def test_glob_explicit(self):
@@ -87,8 +87,8 b' class TestNbConvertApp(TestsBase):'
         with self.create_temp_cwd(['notebook*.ipynb']):
             assert not 'error' in self.call([IPYTHON, 'nbconvert', '--format="python"', 
                 '--notebooks=["*.ipynb", "notebook1.ipynb", "notebook2.ipynb"]']).lower()
-            assert os.path.isfile(os.path.join('nbconvert_build', 'notebook1.py'))
-            assert os.path.isfile(os.path.join('nbconvert_build', 'notebook2.py'))
+            assert os.path.isfile('notebook1.py')
+            assert os.path.isfile('notebook2.py')
 
 
     def test_explicit_glob(self):
@@ -98,8 +98,8 b' class TestNbConvertApp(TestsBase):'
         with self.create_temp_cwd(['notebook*.ipynb']):
             assert not 'error' in self.call([IPYTHON, 'nbconvert', '--format="python"', 
                 '--notebooks=["notebook1.ipynb", "notebook2.ipynb", "*.ipynb"]']).lower()
-            assert os.path.isfile(os.path.join('nbconvert_build', 'notebook1.py'))
-            assert os.path.isfile(os.path.join('nbconvert_build', 'notebook2.py'))
+            assert os.path.isfile('notebook1.py')
+            assert os.path.isfile('notebook2.py')
 
 
     def test_default_config(self):
@@ -108,8 +108,8 b' class TestNbConvertApp(TestsBase):'
         """
         with self.create_temp_cwd(['notebook*.ipynb', 'ipython_nbconvert_config.py']):
             assert not 'error' in self.call([IPYTHON, 'nbconvert']).lower()
-            assert os.path.isfile(os.path.join('nbconvert_build', 'notebook1.py'))
-            assert not os.path.isfile(os.path.join('nbconvert_build', 'notebook2.py'))
+            assert os.path.isfile('notebook1.py')
+            assert not os.path.isfile('notebook2.py')
 
 
     def test_override_config(self):
@@ -119,5 +119,5 b' class TestNbConvertApp(TestsBase):'
         with self.create_temp_cwd(['notebook*.ipynb', 'ipython_nbconvert_config.py', 
                                    'override.py']):
             assert not 'error' in self.call([IPYTHON, 'nbconvert', '--config="override.py"']).lower()
-            assert not os.path.isfile(os.path.join('nbconvert_build', 'notebook1.py'))
-            assert os.path.isfile(os.path.join('nbconvert_build', 'notebook2.py'))
+            assert not os.path.isfile('notebook1.py')
+            assert os.path.isfile('notebook2.py')

IPython/nbconvert/transformers/__init__.py

@@ -2,7 +2,7 b''
 from .base import Transformer
 from .convertfigures import ConvertFiguresTransformer
 from .svg2pdf import SVG2PDFTransformer
-from .extractfigure import ExtractFigureTransformer
+from .extractoutput import ExtractOutputTransformer
 from .revealhelp import RevealHelpTransformer
 from .latex import LatexTransformer
 from .sphinx import SphinxTransformer

IPython/nbconvert/transformers/convertfigures.py

@@ -46,10 +46,10 b' class ConvertFiguresTransformer(Transformer):'
         See base.py
         """
 
-        #Loop through all of the datatypes of the outputs in the cell.
+        # Loop through all of the datatypes of the outputs in the cell.
         for index, cell_out in enumerate(cell.get('outputs', [])):
             for data_type, data in cell_out.items():
-                # this must run *before* extract figures,
+                # this must run *before* extract outputs,
                 # so figure_name and filename do not exist
                 self._convert_figure(cell_out, resources, data_type, data)
         return cell, resources

IPython/nbconvert/transformers/extractoutput.py

@@ -1,5 +1,5 b''
-"""Module containing a transformer that extracts all of the figures from the
-notebook file.  The extracted figures are returned in the 'resources' dictionary.
+"""Module containing a transformer that extracts all of the outputs from the
+notebook file.  The extracted outputs are returned in the 'resources' dictionary.
 """
 #-----------------------------------------------------------------------------
 # Copyright (c) 2013, the IPython Development Team.
@@ -15,6 +15,7 b" notebook file. The extracted figures are returned in the 'resources' dictionary"
 
 import base64
 import sys
+import os
 
 from IPython.utils.traitlets import Unicode
 from .base import Transformer
@@ -24,10 +25,10 b' from IPython.utils import py3compat'
 # Classes
 #-----------------------------------------------------------------------------
 
-class ExtractFigureTransformer(Transformer):
+class ExtractOutputTransformer(Transformer):
     """
-    Extracts all of the figures from the notebook file.  The extracted 
-    figures are returned in the 'resources' dictionary.
+    Extracts all of the outputs from the notebook file.  The extracted 
+    outputs are returned in the 'resources' dictionary.
     """
 
     figure_filename_template = Unicode(
@@ -50,12 +51,14 b' class ExtractFigureTransformer(Transformer):'
         """
 
         #Get the unique key from the resource dict if it exists.  If it does not 
-        #exist, use 'figure' as the default.
+        #exist, use 'figure' as the default.  Also, get files directory if it
+        #has been specified
         unique_key = resources.get('unique_key', 'figure')
+        output_files_dir = resources.get('output_files_dir', None)
         
-        #Make sure figures key exists
-        if not 'figures' in resources:
-            resources['figures'] = {}
+        #Make sure outputs key exists
+        if not 'outputs' in resources:
+            resources['outputs'] = {}
             
         #Loop through all of the outputs in the cell
         for index, out in enumerate(cell.get('outputs', [])):
@@ -77,7 +80,7 b' class ExtractFigureTransformer(Transformer):'
                         data = data.encode("UTF-8")
                     
                     #Build a figure name
-                    figure_name = self.figure_filename_template.format( 
+                    filename = self.figure_filename_template.format( 
                                     unique_key=unique_key,
                                     cell_index=cell_index,
                                     index=index,
@@ -87,10 +90,12 b' class ExtractFigureTransformer(Transformer):'
                     #   cell.outputs[i].svg_filename  ... etc (svg in example)
                     # Where
                     #   cell.outputs[i].svg  contains the data
-                    out[out_type + '_filename'] = figure_name
+                    if output_files_dir is not None:
+                        filename = os.path.join(output_files_dir, filename)
+                    out[out_type + '_filename'] = filename
 
                     #In the resources, make the figure available via
-                    #   resources['figures']['filename'] = data
-                    resources['figures'][figure_name] = data
+                    #   resources['outputs']['filename'] = data
+                    resources['outputs'][filename] = data
 
         return cell, resources

IPython/nbconvert/writers/debug.py

@@ -34,10 +34,10 b' class DebugWriter(WriterBase):'
         See base for more...
         """
 
-        if 'figures' in resources:
-            print("Figures extracted from %s" % notebook_name)
+        if 'outputs' in resources:
+            print("outputs extracted from %s" % notebook_name)
             print('-' * 80)
-            pprint.pprint(resources['figures'], indent=2, width=70)
+            pprint.pprint(resources['outputs'], indent=2, width=70)
         else:
-            print("No figures extracted from %s" % notebook_name)
+            print("No outputs extracted from %s" % notebook_name)
         print('=' * 80)

IPython/nbconvert/writers/files.py

@@ -31,12 +31,12 b' class FilesWriter(WriterBase):'
     """Consumes nbconvert output and produces files."""
 
 
-    build_directory = Unicode("nbconvert_build", config=True, 
+    build_directory = Unicode(".", config=True, 
                               help="""Directory to write output to.  Leave blank
                               to output to the current directory""")
 
 
-    #Make sure that the output directory exists.
+    # Make sure that the output directory exists.
     def _build_directory_changed(self, name, old, new):
         if new and not os.path.isdir(new):
             os.makedirs(new)
@@ -57,42 +57,46 b' class FilesWriter(WriterBase):'
             See base for more...
             """
 
-            #Pull the extension from the resources dict.
+            # Pull the extension and subdir from the resources dict.
             output_extension = resources['output_extension']
 
-            #Write all of the extracted resources to the destination directory.
-            #NOTE: WE WRITE EVERYTHING AS-IF IT'S BINARY.  THE EXTRACT FIG
-            #TRANSFORMER SHOULD HANDLE UNIX/WINDOWS LINE ENDINGS...
-            for filename, data in resources.get('figures', {}).items():
+            # Write all of the extracted resources to the destination directory.
+            # NOTE: WE WRITE EVERYTHING AS-IF IT'S BINARY.  THE EXTRACT FIG
+            # TRANSFORMER SHOULD HANDLE UNIX/WINDOWS LINE ENDINGS...
+            for filename, data in resources.get('outputs', {}).items():
 
-                #Determine where to write the file to
+                # Determine where to write the file to
                 dest = os.path.join(self.build_directory, filename)
+                path = os.path.dirname(dest)
+                if not os.path.isdir(path):
+                    os.makedirs(path)
 
-                #Write file
+                # Write file
                 with io.open(dest, 'wb') as f:
                     f.write(data)
 
-            #Copy referenced files to output directory
+            # Copy referenced files to output directory
             if self.build_directory:
                 for filename in self.files:
 
-                    #Copy files that match search pattern
+                    # Copy files that match search pattern
                     for matching_filename in glob.glob(filename):
 
-                        #Make sure folder exists.
+                        # Make sure folder exists.
                         dest = os.path.join(self.build_directory, filename)
                         path = os.path.dirname(dest)
                         if not os.path.isdir(path):
                             os.makedirs(path)
 
-                        #Copy
-                        shutil.copyfile(matching_filename, dest)
+                        # Copy if destination is different.
+                        if not os.path.normpath(dest) == os.path.normpath(matching_filename):
+                            shutil.copyfile(matching_filename, dest)
 
-            #Determine where to write conversion results.
+            # Determine where to write conversion results.
             dest = notebook_name + '.' + output_extension
             if self.build_directory:
                 dest = os.path.join(self.build_directory, dest)
 
-            #Write conversion results.
+            # Write conversion results.
             with io.open(dest, 'w') as f:
                 f.write(output)

IPython/nbconvert/writers/stdout.py

@@ -22,9 +22,7 b' from .base import WriterBase'
 
 class StdoutWriter(WriterBase):
     """Consumes output from nbconvert export...() methods and writes to the 
-    stdout stream.  Allows for quick debuging of nbconvert output.  Using the
-    debug flag makes the writer pretty-print the figures contained within the
-    notebook."""
+    stdout stream."""
 
 
     def write(self, output, resources, **kw):