From 48f15a5272451f203f836e0b9272b37608e69780 2014-09-10 01:17:49
From: MinRK <benjaminrk@gmail.com>
Date: 2014-09-10 01:17:49
Subject: [PATCH] always pass single lines to transform.push

in InputSplitter.flush_transform

Previous behavior allowed early transforms to confuse later ones
if they handle multiple lines at a time.
---

diff --git a/IPython/core/inputsplitter.py b/IPython/core/inputsplitter.py
index 1630ab4..a3b1f01 100644
--- a/IPython/core/inputsplitter.py
+++ b/IPython/core/inputsplitter.py
@@ -12,31 +12,15 @@ The code to actually do these transformations is in :mod:`IPython.core.inputtran
 and stores the results.
 
 For more details, see the class docstrings below.
-
-Authors
--------
-
-* Fernando Perez
-* Brian Granger
-* Thomas Kluyver
 """
-#-----------------------------------------------------------------------------
-#  Copyright (C) 2010  The IPython Development Team
-#
-#  Distributed under the terms of the BSD License.  The full license is in
-#  the file COPYING, distributed as part of this software.
-#-----------------------------------------------------------------------------
 
-#-----------------------------------------------------------------------------
-# Imports
-#-----------------------------------------------------------------------------
-# stdlib
+# Copyright (c) IPython Development Team.
+# Distributed under the terms of the Modified BSD License.
 import ast
 import codeop
 import re
 import sys
 
-# IPython modules
 from IPython.utils.py3compat import cast_unicode
 from IPython.core.inputtransformer import (leading_indent,
                                            classic_prompt,
@@ -512,18 +496,42 @@ class IPythonInputSplitter(InputSplitter):
     
     def flush_transformers(self):
         def _flush(transform, out):
-            if out is not None:
-                tmp = transform.push(out)
-                return tmp or transform.reset() or None
-            else:
-                return transform.reset() or None
+            """yield transformed lines
+            
+            always strings, never None
+            
+            transform: the current transform
+            out: an iterable of previously transformed inputs.
+                 Each may be multiline, which will be passed
+                 one line at a time to transform.
+            """
+            anything = False
+            for out in out:
+                anything = True
+                tmp = None
+                for line in out.splitlines():
+                    # push one line at a time
+                    tmp = transform.push(line)
+                    if tmp is not None:
+                        yield tmp
+                if tmp is None:
+                    # transformer is still consuming, reset
+                    tmp = transform.reset()
+                    if tmp is not None:
+                        yield tmp
+            if not anything:
+                # nothing was pushed, reset
+                tmp = transform.reset()
+                if tmp is not None:
+                    yield tmp
         
-        out = None
+        out = []
         for t in self.transforms_in_use:
             out = _flush(t, out)
         
-        if out is not None:
-            self._store(out)
+        out = list(out)
+        if out:
+            self._store('\n'.join(out))
 
     def raw_reset(self):
         """Return raw input only and perform a full reset.
diff --git a/IPython/core/tests/test_inputsplitter.py b/IPython/core/tests/test_inputsplitter.py
index c5c841e..6447f0c 100644
--- a/IPython/core/tests/test_inputsplitter.py
+++ b/IPython/core/tests/test_inputsplitter.py
@@ -1,31 +1,18 @@
 # -*- coding: utf-8 -*-
-"""Tests for the inputsplitter module.
+"""Tests for the inputsplitter module."""
 
-Authors
--------
-* Fernando Perez
-* Robert Kern
-"""
 from __future__ import print_function
-#-----------------------------------------------------------------------------
-#  Copyright (C) 2010-2011  The IPython Development Team
-#
-#  Distributed under the terms of the BSD License.  The full license is in
-#  the file COPYING, distributed as part of this software.
-#-----------------------------------------------------------------------------
 
-#-----------------------------------------------------------------------------
-# Imports
-#-----------------------------------------------------------------------------
-# stdlib
+# Copyright (c) IPython Development Team.
+# Distributed under the terms of the Modified BSD License.
+
 import unittest
 import sys
 
-# Third party
 import nose.tools as nt
 
-# Our own
 from IPython.core import inputsplitter as isp
+from IPython.core.inputtransformer import InputTransformer
 from IPython.core.tests.test_inputtransformer import syntax, syntax_ml
 from IPython.testing import tools as tt
 from IPython.utils import py3compat
@@ -466,8 +453,33 @@ class IPythonInputTestCase(InputSplitterTestCase):
             )
             out = isp.transform_cell(raw)
             self.assertEqual(out.rstrip(), expected.rstrip())
+
+    def test_multiline_passthrough(self):
+        isp = self.isp
+        class CommentTransformer(InputTransformer):
+            def __init__(self):
+                self._lines = []
+            
+            def push(self, line):
+                self._lines.append(line + '#')
+            
+            def reset(self):
+                text = '\n'.join(self._lines)
+                self._lines = []
+                return text
         
+        isp.physical_line_transforms.insert(0, CommentTransformer())
         
+        for raw, expected in [
+            ("a=5", "a=5#"),
+            ("%ls foo", "get_ipython().magic(%r)" % u'ls foo#'),
+            ("!ls foo\n%ls bar", "get_ipython().system(%r)\nget_ipython().magic(%r)" % (
+                u'ls foo#', u'ls bar#'
+            )),
+            ("1\n2\n3\n%ls foo\n4\n5", "1#\n2#\n3#\nget_ipython().magic(%r)\n4#\n5#" % u'ls foo#'),
+        ]:
+            out = isp.transform_cell(raw)
+            self.assertEqual(out.rstrip(), expected.rstrip())
 
 #-----------------------------------------------------------------------------
 # Main - use as a script, mostly for developer experiments