Backport PR #5535: fix n^2 performance issue in coalesce_streams preprocessor

for n consecutive stream outputs, `\r` fix would be compiled n times, and applied to each ith output (n-i) times.

- move pattern to module level
- apply replacement after coalescing outputs

An example notebook from nbviewer with ~1k outputs that was taking 90 seconds to render now takes 3 seconds.

IPython/html/nbconvert/tests/test_nbconvert_handlers.py

@@ -58,7 +58,7 b' class APITest(NotebookTestBase):'
         nb.worksheets = [ws]
         ws.cells.append(new_heading_cell(u'Created by test ³'))
         cc1 = new_code_cell(input=u'print(2*6)')
-        cc1.outputs.append(new_output(output_text=u'12'))
+        cc1.outputs.append(new_output(output_text=u'12', output_type='stream'))
         cc1.outputs.append(new_output(output_png=png_green_pixel, output_type='pyout'))
         ws.cells.append(cc1)
         

IPython/nbconvert/preprocessors/base.py

@@ -1,27 +1,11 b''
-"""
-Module that re-groups preprocessor that would be applied to ipynb files
-before going through the templating machinery.
-
-It exposes a convenient class to inherit from to access configurability.
-"""
-#-----------------------------------------------------------------------------
-# Copyright (c) 2013, the IPython Development Team.
-#
-# Distributed under the terms of the Modified BSD License.
-#
-# The full license is in the file COPYING.txt, distributed with this software.
-#-----------------------------------------------------------------------------
+"""Base class for preprocessors"""
 
-#-----------------------------------------------------------------------------
-# Imports
-#-----------------------------------------------------------------------------
+# Copyright (c) IPython Development Team.
+# Distributed under the terms of the Modified BSD License.
 
 from ..utils.base import NbConvertBase
 from IPython.utils.traitlets import Bool
 
-#-----------------------------------------------------------------------------
-# Classes and Functions
-#-----------------------------------------------------------------------------
 
 class Preprocessor(NbConvertBase):
     """ A configurable preprocessor
@@ -30,7 +14,7 b' class Preprocessor(NbConvertBase):'
     preprocessor.
 
     Any configurable traitlets this class exposed will be configurable in
-    profiles using c.SubClassName.atribute=value
+    profiles using c.SubClassName.attribute = value
 
     you can overwrite :meth:`preprocess_cell` to apply a transformation
     independently on each cell or :meth:`preprocess` if you prefer your own
@@ -59,6 +43,7 b' class Preprocessor(NbConvertBase):'
        
     def __call__(self, nb, resources):
         if self.enabled:
+            self.log.debug("Applying preprocessor: %s", self.__class__.__name__)
             return self.preprocess(nb,resources)
         else:
             return nb, resources
@@ -68,9 +53,10 b' class Preprocessor(NbConvertBase):'
         """
         Preprocessing to apply on each notebook.
         
-        You should return modified nb, resources.
+        Must return modified nb, resources.
+
         If you wish to apply your preprocessing to each cell, you might want
-        to overwrite preprocess_cell method instead.
+        to override preprocess_cell method instead.
         
         Parameters
         ----------
@@ -80,20 +66,16 b' class Preprocessor(NbConvertBase):'
             Additional resources used in the conversion process.  Allows
             preprocessors to pass variables into the Jinja engine.
         """
-        self.log.debug("Applying preprocess: %s", self.__class__.__name__)
-        try :
-            for worksheet in nb.worksheets:
-                for index, cell in enumerate(worksheet.cells):
-                    worksheet.cells[index], resources = self.preprocess_cell(cell, resources, index)
-            return nb, resources
-        except NotImplementedError:
-            raise NotImplementedError('should be implemented by subclass')
+        for worksheet in nb.worksheets:
+            for index, cell in enumerate(worksheet.cells):
+                worksheet.cells[index], resources = self.preprocess_cell(cell, resources, index)
+        return nb, resources
 
 
     def preprocess_cell(self, cell, resources, index):
         """
-        Overwrite if you want to apply some preprocessing to each cell.  You 
-        should return modified cell and resource dictionary.
+        Override if you want to apply some preprocessing to each cell.
+        Must return modified cell and resource dictionary.
         
         Parameters
         ----------

IPython/nbconvert/preprocessors/coalescestreams.py

@@ -1,23 +1,10 b''
-"""Module that allows latex output notebooks to be conditioned before
-they are converted.  Exposes a decorator (@cell_preprocessor) in
-addition to the coalesce_streams pre-proccessor.
-"""
-#-----------------------------------------------------------------------------
-# Copyright (c) 2013, the IPython Development Team.
-#
+"""Preprocessor for merging consecutive stream outputs for easier handling."""
+
+# Copyright (c) IPython Development Team.
 # Distributed under the terms of the Modified BSD License.
-#
-# The full license is in the file COPYING.txt, distributed with this software.
-#-----------------------------------------------------------------------------
 
-#-----------------------------------------------------------------------------
-# Imports
-#-----------------------------------------------------------------------------
 import re
 
-#-----------------------------------------------------------------------------
-# Functions
-#-----------------------------------------------------------------------------
 def cell_preprocessor(function):
     """
     Wrap a function to be executed on all cells of a notebook
@@ -34,12 +21,18 b' def cell_preprocessor(function):'
     """
     
     def wrappedfunc(nb, resources):
-        for worksheet in nb.worksheets :
+        from IPython.config import Application
+        if Application.initialized():
+            Application.instance().log.debug(
+                "Applying preprocessor: %s", function.__name__
+            )
+        for worksheet in nb.worksheets:
             for index, cell in enumerate(worksheet.cells):
                 worksheet.cells[index], resources = function(cell, resources, index)
         return nb, resources
     return wrappedfunc
 
+cr_pat = re.compile(r'.*\r(?=[^\n])')
 
 @cell_preprocessor
 def coalesce_streams(cell, resources, index):
@@ -64,7 +57,6 b' def coalesce_streams(cell, resources, index):'
     
     last = outputs[0]
     new_outputs = [last]
-    
     for output in outputs[1:]:
         if (output.output_type == 'stream' and
             last.output_type == 'stream' and
@@ -72,12 +64,14 b' def coalesce_streams(cell, resources, index):'
         ):
             last.text += output.text
 
-            # Respect \r characters.
-            cr_pat = re.compile(r'.*\r(?=[^\n])')
-            last.text = cr_pat.sub('', last.text)
         else:
             new_outputs.append(output)
             last = output
 
+    # process \r characters
+    for output in new_outputs:
+        if output.output_type == 'stream':
+            output.text = cr_pat.sub('', output.text)
+
     cell.outputs = new_outputs
     return cell, resources

IPython/nbformat/v3/nbbase.py

@@ -4,22 +4,10 b' The Python representation of a notebook is a nested structure of'
 dictionary subclasses that support attribute access
 (IPython.utils.ipstruct.Struct). The functions in this module are merely
 helpers to build the structs in the right form.
-
-Authors:
-
-* Brian Granger
 """
 
-#-----------------------------------------------------------------------------
-#  Copyright (C) 2008-2011  The IPython Development Team
-#
-#  Distributed under the terms of the BSD License.  The full license is in
-#  the file COPYING, distributed as part of this software.
-#-----------------------------------------------------------------------------
-
-#-----------------------------------------------------------------------------
-# Imports
-#-----------------------------------------------------------------------------
+# Copyright (c) IPython Development Team.
+# Distributed under the terms of the Modified BSD License.
 
 import pprint
 import uuid
@@ -51,15 +39,14 b' def from_dict(d):'
         return d
 
 
-def new_output(output_type=None, output_text=None, output_png=None,
+def new_output(output_type, output_text=None, output_png=None,
     output_html=None, output_svg=None, output_latex=None, output_json=None,
     output_javascript=None, output_jpeg=None, prompt_number=None,
     ename=None, evalue=None, traceback=None, stream=None, metadata=None):
     """Create a new output, to go in the ``cell.outputs`` list of a code cell.
     """
     output = NotebookNode()
-    if output_type is not None:
-        output.output_type = unicode_type(output_type)
+    output.output_type = unicode_type(output_type)
 
     if metadata is None:
         metadata = {}

IPython/nbformat/v3/tests/test_nbbase.py

@@ -143,15 +143,15 b' class TestMetadata(TestCase):'
 
 class TestOutputs(TestCase):
     def test_binary_png(self):
-        out = new_output(output_png=b'\x89PNG\r\n\x1a\n')
+        out = new_output(output_png=b'\x89PNG\r\n\x1a\n', output_type='display_data')
 
     def test_b64b6tes_png(self):
-        out = new_output(output_png=b'iVBORw0KG')
+        out = new_output(output_png=b'iVBORw0KG', output_type='display_data')
     
     def test_binary_jpeg(self):
-        out = new_output(output_jpeg=b'\xff\xd8')
+        out = new_output(output_jpeg=b'\xff\xd8', output_type='display_data')
 
     def test_b64b6tes_jpeg(self):
-        out = new_output(output_jpeg=b'/9')
+        out = new_output(output_jpeg=b'/9', output_type='display_data')