Merge pull request #3734 from jdfreder/file_subdir

Nbconvert: Export extracted files into `nbname_files` subdirectory

Default build directory changed to .
Files extracted from notebook now are placed into a ./notebook_files/ directory by default
Spaces added between # symbols and comments
It may be best to look at the diffs of the individual commits... The addition of spaces in the comments makes the overall diff hard to read.

IPython/nbconvert/exporters/exporter.py

@@ -131,7 +131,7 b' class Exporter(Configurable):'
 
     default_transformers = List([nbtransformers.coalesce_streams,
                                  nbtransformers.SVG2PDFTransformer,
-                                 nbtransformers.ExtractFigureTransformer,
+                                 nbtransformers.ExtractOutputTransformer,
                                  nbtransformers.CSSHTMLHeaderTransformer,
                                  nbtransformers.RevealHelpTransformer,
                                  nbtransformers.LatexTransformer,

IPython/nbconvert/exporters/latex.py

@@ -88,7 +88,7 b' class LatexExporter(Exporter):'
             'NbConvertBase': {
                 'display_data_priority' : ['latex', 'pdf', 'png', 'jpg', 'svg', 'jpeg', 'text']
                 },
-             'ExtractFigureTransformer': {
+             'ExtractOutputTransformer': {
                     'enabled':True
                  },
              'SVG2PDFTransformer': {

IPython/nbconvert/exporters/rst.py

@@ -37,6 +37,6 b' class RSTExporter(Exporter):'
 
     @property
     def default_config(self):
-        c = Config({'ExtractFigureTransformer':{'enabled':True}})
+        c = Config({'ExtractOutputTransformer':{'enabled':True}})
         c.merge(super(RSTExporter,self).default_config)
         return c

IPython/nbconvert/exporters/tests/test_exporter.py

@@ -45,16 +45,16 b' class TestExporter(ExportersTestsBase):'
         assert len(output) > 0
 
 
-    def test_extract_figures(self):
+    def test_extract_outputs(self):
         """
-        If the ExtractFigureTransformer is enabled, are figures extracted?
+        If the ExtractOutputTransformer is enabled, are outputs extracted?
         """
-        config = Config({'ExtractFigureTransformer': {'enabled': True}})
+        config = Config({'ExtractOutputTransformer': {'enabled': True}})
         exporter = self._make_exporter(config=config)
         (output, resources) = exporter.from_filename(self._get_notebook())
         assert resources is not None
-        assert 'figures' in resources
-        assert len(resources['figures']) > 0
+        assert 'outputs' in resources
+        assert len(resources['outputs']) > 0
 
 
     def test_transformer_class(self):

IPython/nbconvert/nbconvertapp.py

@@ -188,6 +188,7 b' class NbConvertApp(BaseIPythonApplication):'
             notebook_name = basename[:basename.rfind('.')]
             resources = {}
             resources['unique_key'] = notebook_name
+            resources['output_files_dir'] = '%s_files' % notebook_name
 
             #Try to export
             try:

IPython/nbconvert/tests/test_nbconvertapp.py

@@ -53,8 +53,8 b' class TestNbConvertApp(TestsBase):'
         with self.create_temp_cwd(['notebook*.ipynb']):
             assert not 'error' in self.call([IPYTHON, 'nbconvert', 
                 '--format="python"', '--notebooks=["*.ipynb"]']).lower()
-            assert os.path.isfile(os.path.join('nbconvert_build', 'notebook1.py'))
-            assert os.path.isfile(os.path.join('nbconvert_build', 'notebook2.py'))
+            assert os.path.isfile('notebook1.py')
+            assert os.path.isfile('notebook2.py')
 
 
     def test_glob_subdir(self):
@@ -65,8 +65,8 b' class TestNbConvertApp(TestsBase):'
             self.copy_files_to(['notebook*.ipynb'], 'subdir/')
             assert not 'error' in self.call([IPYTHON, 'nbconvert', '--format="python"', 
                 '--notebooks=["%s"]' % os.path.join('subdir', '*.ipynb')]).lower()
-            assert os.path.isfile(os.path.join('nbconvert_build', 'notebook1.py'))
-            assert os.path.isfile(os.path.join('nbconvert_build', 'notebook2.py'))
+            assert os.path.isfile('notebook1.py')
+            assert os.path.isfile('notebook2.py')
 
 
     def test_explicit(self):
@@ -76,8 +76,8 b' class TestNbConvertApp(TestsBase):'
         with self.create_temp_cwd(['notebook*.ipynb']):
             assert not 'error' in self.call([IPYTHON, 'nbconvert', '--format="python"', 
                 '--notebooks=["notebook2.ipynb"]']).lower()
-            assert not os.path.isfile(os.path.join('nbconvert_build', 'notebook1.py'))
-            assert os.path.isfile(os.path.join('nbconvert_build', 'notebook2.py'))
+            assert not os.path.isfile('notebook1.py')
+            assert os.path.isfile('notebook2.py')
 
 
     def test_glob_explicit(self):
@@ -87,8 +87,8 b' class TestNbConvertApp(TestsBase):'
         with self.create_temp_cwd(['notebook*.ipynb']):
             assert not 'error' in self.call([IPYTHON, 'nbconvert', '--format="python"', 
                 '--notebooks=["*.ipynb", "notebook1.ipynb", "notebook2.ipynb"]']).lower()
-            assert os.path.isfile(os.path.join('nbconvert_build', 'notebook1.py'))
-            assert os.path.isfile(os.path.join('nbconvert_build', 'notebook2.py'))
+            assert os.path.isfile('notebook1.py')
+            assert os.path.isfile('notebook2.py')
 
 
     def test_explicit_glob(self):
@@ -98,8 +98,8 b' class TestNbConvertApp(TestsBase):'
         with self.create_temp_cwd(['notebook*.ipynb']):
             assert not 'error' in self.call([IPYTHON, 'nbconvert', '--format="python"', 
                 '--notebooks=["notebook1.ipynb", "notebook2.ipynb", "*.ipynb"]']).lower()
-            assert os.path.isfile(os.path.join('nbconvert_build', 'notebook1.py'))
-            assert os.path.isfile(os.path.join('nbconvert_build', 'notebook2.py'))
+            assert os.path.isfile('notebook1.py')
+            assert os.path.isfile('notebook2.py')
 
 
     def test_default_config(self):
@@ -108,8 +108,8 b' class TestNbConvertApp(TestsBase):'
         """
         with self.create_temp_cwd(['notebook*.ipynb', 'ipython_nbconvert_config.py']):
             assert not 'error' in self.call([IPYTHON, 'nbconvert']).lower()
-            assert os.path.isfile(os.path.join('nbconvert_build', 'notebook1.py'))
-            assert not os.path.isfile(os.path.join('nbconvert_build', 'notebook2.py'))
+            assert os.path.isfile('notebook1.py')
+            assert not os.path.isfile('notebook2.py')
 
 
     def test_override_config(self):
@@ -119,5 +119,5 b' class TestNbConvertApp(TestsBase):'
         with self.create_temp_cwd(['notebook*.ipynb', 'ipython_nbconvert_config.py', 
                                    'override.py']):
             assert not 'error' in self.call([IPYTHON, 'nbconvert', '--config="override.py"']).lower()
-            assert not os.path.isfile(os.path.join('nbconvert_build', 'notebook1.py'))
-            assert os.path.isfile(os.path.join('nbconvert_build', 'notebook2.py'))
+            assert not os.path.isfile('notebook1.py')
+            assert os.path.isfile('notebook2.py')

IPython/nbconvert/transformers/__init__.py

@@ -2,7 +2,7 b''
 from .base import Transformer
 from .convertfigures import ConvertFiguresTransformer
 from .svg2pdf import SVG2PDFTransformer
-from .extractfigure import ExtractFigureTransformer
+from .extractoutput import ExtractOutputTransformer
 from .revealhelp import RevealHelpTransformer
 from .latex import LatexTransformer
 from .sphinx import SphinxTransformer

IPython/nbconvert/transformers/convertfigures.py

@@ -49,7 +49,7 b' class ConvertFiguresTransformer(Transformer):'
         #Loop through all of the datatypes of the outputs in the cell.
         for index, cell_out in enumerate(cell.get('outputs', [])):
             for data_type, data in cell_out.items():
-                # this must run *before* extract figures,
+                # this must run *before* extract outputs,
                 # so figure_name and filename do not exist
                 self._convert_figure(cell_out, resources, data_type, data)
         return cell, resources

IPython/nbconvert/transformers/extractoutput.py

@@ -1,5 +1,5 b''
-"""Module containing a transformer that extracts all of the figures from the
-notebook file.  The extracted figures are returned in the 'resources' dictionary.
+"""Module containing a transformer that extracts all of the outputs from the
+notebook file.  The extracted outputs are returned in the 'resources' dictionary.
 """
 #-----------------------------------------------------------------------------
 # Copyright (c) 2013, the IPython Development Team.
@@ -15,6 +15,7 b" notebook file. The extracted figures are returned in the 'resources' dictionary"
 
 import base64
 import sys
+import os
 
 from IPython.utils.traitlets import Unicode
 from .base import Transformer
@@ -24,10 +25,10 b' from IPython.utils import py3compat'
 # Classes
 #-----------------------------------------------------------------------------
 
-class ExtractFigureTransformer(Transformer):
+class ExtractOutputTransformer(Transformer):
     """
-    Extracts all of the figures from the notebook file.  The extracted 
-    figures are returned in the 'resources' dictionary.
+    Extracts all of the outputs from the notebook file.  The extracted 
+    outputs are returned in the 'resources' dictionary.
     """
 
     figure_filename_template = Unicode(
@@ -50,12 +51,14 b' class ExtractFigureTransformer(Transformer):'
         """
 
         #Get the unique key from the resource dict if it exists.  If it does not 
-        #exist, use 'figure' as the default.
+        #exist, use 'figure' as the default.  Also, get files directory if it
+        #has been specified
         unique_key = resources.get('unique_key', 'figure')
+        output_files_dir = resources.get('output_files_dir', None)
         
-        #Make sure figures key exists
-        if not 'figures' in resources:
-            resources['figures'] = {}
+        #Make sure outputs key exists
+        if not 'outputs' in resources:
+            resources['outputs'] = {}
             
         #Loop through all of the outputs in the cell
         for index, out in enumerate(cell.get('outputs', [])):
@@ -77,7 +80,7 b' class ExtractFigureTransformer(Transformer):'
                         data = data.encode("UTF-8")
                     
                     #Build a figure name
-                    figure_name = self.figure_filename_template.format( 
+                    filename = self.figure_filename_template.format( 
                                     unique_key=unique_key,
                                     cell_index=cell_index,
                                     index=index,
@@ -87,10 +90,12 b' class ExtractFigureTransformer(Transformer):'
                     #   cell.outputs[i].svg_filename  ... etc (svg in example)
                     # Where
                     #   cell.outputs[i].svg  contains the data
-                    out[out_type + '_filename'] = figure_name
+                    if output_files_dir is not None:
+                        filename = os.path.join(output_files_dir, filename)
+                    out[out_type + '_filename'] = filename
 
                     #In the resources, make the figure available via
-                    #   resources['figures']['filename'] = data
-                    resources['figures'][figure_name] = data
+                    #   resources['outputs']['filename'] = data
+                    resources['outputs'][filename] = data
 
         return cell, resources

IPython/nbconvert/writers/debug.py

@@ -34,10 +34,10 b' class DebugWriter(WriterBase):'
         See base for more...
         """
 
-        if 'figures' in resources:
-            print("Figures extracted from %s" % notebook_name)
+        if 'outputs' in resources:
+            print("outputs extracted from %s" % notebook_name)
             print('-' * 80)
-            pprint.pprint(resources['figures'], indent=2, width=70)
+            pprint.pprint(resources['outputs'], indent=2, width=70)
         else:
-            print("No figures extracted from %s" % notebook_name)
+            print("No outputs extracted from %s" % notebook_name)
         print('=' * 80)

IPython/nbconvert/writers/files.py

@@ -31,7 +31,7 b' class FilesWriter(WriterBase):'
     """Consumes nbconvert output and produces files."""
 
 
-    build_directory = Unicode("nbconvert_build", config=True, 
+    build_directory = Unicode(".", config=True, 
                               help="""Directory to write output to.  Leave blank
                               to output to the current directory""")
 
@@ -57,16 +57,19 b' class FilesWriter(WriterBase):'
             See base for more...
             """
 
-            #Pull the extension from the resources dict.
+            # Pull the extension and subdir from the resources dict.
             output_extension = resources['output_extension']
 
             #Write all of the extracted resources to the destination directory.
             #NOTE: WE WRITE EVERYTHING AS-IF IT'S BINARY.  THE EXTRACT FIG
             #TRANSFORMER SHOULD HANDLE UNIX/WINDOWS LINE ENDINGS...
-            for filename, data in resources.get('figures', {}).items():
+            for filename, data in resources.get('outputs', {}).items():
 
                 #Determine where to write the file to
                 dest = os.path.join(self.build_directory, filename)
+                path = os.path.dirname(dest)
+                if not os.path.isdir(path):
+                    os.makedirs(path)
 
                 #Write file
                 with io.open(dest, 'wb') as f:
@@ -85,7 +88,8 b' class FilesWriter(WriterBase):'
                         if not os.path.isdir(path):
                             os.makedirs(path)
 
-                        #Copy
+                        # Copy if destination is different.
+                        if not os.path.normpath(dest) == os.path.normpath(matching_filename):
                         shutil.copyfile(matching_filename, dest)
 
             #Determine where to write conversion results.

IPython/nbconvert/writers/stdout.py

@@ -22,9 +22,7 b' from .base import WriterBase'
 
 class StdoutWriter(WriterBase):
     """Consumes output from nbconvert export...() methods and writes to the 
-    stdout stream.  Allows for quick debuging of nbconvert output.  Using the
-    debug flag makes the writer pretty-print the figures contained within the
-    notebook."""
+    stdout stream."""
 
 
     def write(self, output, resources, **kw):