Merge pull request #3203 from takluyver/cell-magics-wholecell

Separate InputSplitter for transforming whole cells

This allows cell magics to include blank lines once again (closes  #3178).

InteractiveShell now has two input splitters:

- `input_splitter` is used to determine when a block of input is complete, so you can use a cell magic in the terminal by leaving a blank line at the end.
- `input_transformer_manager` is used to transform whole cells, so cell magics affect the whole of their cell.

I removed the similar concept of 'input modes' for InputSplitter, because it was bound to get conflated, and it made the code rather hard to follow.

IPython/core/inputsplitter.py

@@ -247,8 +247,6 b' class InputSplitter(object):'
     # synced to the source, so it can be queried at any time to obtain the code
     # object; it will be None if the source doesn't compile to valid Python.
     code = None
-    # Input mode
-    input_mode = 'line'
 
     # Private attributes
 
@@ -261,32 +259,12 b' class InputSplitter(object):'
     # Boolean indicating whether the current block is complete
     _is_complete = None
 
-    def __init__(self, input_mode=None):
+    def __init__(self):
         """Create a new InputSplitter instance.
-
-        Parameters
-        ----------
-        input_mode : str
-
-          One of ['line', 'cell']; default is 'line'.
-
-       The input_mode parameter controls how new inputs are used when fed via
-       the :meth:`push` method:
-
-       - 'line': meant for line-oriented clients, inputs are appended one at a
-         time to the internal buffer and the whole buffer is compiled.
-
-       - 'cell': meant for clients that can edit multi-line 'cells' of text at
-          a time.  A cell can contain one or more blocks that can be compile in
-          'single' mode by Python.  In this mode, each new input new input
-          completely replaces all prior inputs.  Cell mode is thus equivalent
-          to prepending a full reset() to every push() call.
         """
         self._buffer = []
         self._compile = codeop.CommandCompiler()
         self.encoding = get_input_encoding()
-        self.input_mode = InputSplitter.input_mode if input_mode is None \
-                          else input_mode
 
     def reset(self):
         """Reset the input buffer and associated state."""
@@ -326,9 +304,6 b' class InputSplitter(object):'
           this value is also stored as a private attribute (``_is_complete``), so it
           can be queried at any time.
         """
-        if self.input_mode == 'cell':
-            self.reset()
-
         self._store(lines)
         source = self.source
 
@@ -365,20 +340,14 b' class InputSplitter(object):'
         This method is meant to be used by line-oriented frontends, who need to
         guess whether a block is complete or not based solely on prior and
         current input lines.  The InputSplitter considers it has a complete
-        interactive block and will not accept more input only when either a
-        SyntaxError is raised, or *all* of the following are true:
-
-        1. The input compiles to a complete statement.
-
-        2. The indentation level is flush-left (because if we are indented,
-           like inside a function definition or for loop, we need to keep
-           reading new input).
+        interactive block and will not accept more input when either:
+        
+        * A SyntaxError is raised
 
-        3. There is one extra line consisting only of whitespace.
+        * The code is complete and consists of a single line or a single
+          non-compound statement
 
-        Because of condition #3, this method should be used only by
-        *line-oriented* frontends, since it means that intermediate blank lines
-        are not allowed in function definitions (or any other indented block).
+        * The code is complete and has a blank line at the end
 
         If the current input produces a syntax error, this method immediately
         returns False but does *not* raise the syntax error exception, as
@@ -388,39 +357,37 b' class InputSplitter(object):'
         """
 
         # With incomplete input, unconditionally accept more
+        # A syntax error also sets _is_complete to True - see push()
         if not self._is_complete:
+            #print("Not complete")  # debug
             return True
-
-        # If we already have complete input and we're flush left, the answer
-        # depends.  In line mode, if there hasn't been any indentation,
-        # that's it. If we've come back from some indentation, we need
-        # the blank final line to finish.
-        # In cell mode, we need to check how many blocks the input so far
-        # compiles into, because if there's already more than one full
-        # independent block of input, then the client has entered full
-        # 'cell' mode and is feeding lines that each is complete.  In this
-        # case we should then keep accepting. The Qt terminal-like console
-        # does precisely this, to provide the convenience of terminal-like
-        # input of single expressions, but allowing the user (with a
-        # separate keystroke) to switch to 'cell' mode and type multiple
-        # expressions in one shot.
+        
+        # The user can make any (complete) input execute by leaving a blank line
+        last_line = self.source.splitlines()[-1]
+        if (not last_line) or last_line.isspace():
+            #print("Blank line")  # debug
+            return False
+        
+        # If there's just a single line or AST node, and we're flush left, as is
+        # the case after a simple statement such as 'a=1', we want to execute it
+        # straight away.
         if self.indent_spaces==0:
-            if self.input_mode=='line':
-                if not self._full_dedent:
-                    return False
+            if len(self.source.splitlines()) <= 1:
+                return False
+            
+            try:
+                code_ast = ast.parse(u''.join(self._buffer))
+            except Exception:
+                #print("Can't parse AST")  # debug
+                return False
             else:
-                try:
-                    code_ast = ast.parse(u''.join(self._buffer))
-                except Exception:
+                if len(code_ast.body) == 1 and \
+                                    not hasattr(code_ast.body[0], 'body'):
+                    #print("Simple statement")  # debug
                     return False
-                else:
-                    if len(code_ast.body) == 1:
-                        return False
 
-        # When input is complete, then termination is marked by an extra blank
-        # line at the end.
-        last_line = self.source.splitlines()[-1]
-        return bool(last_line and not last_line.isspace())
+        # General fallback - accept more code
+        return True
 
     #------------------------------------------------------------------------
     # Private interface
@@ -510,9 +477,9 b' class IPythonInputSplitter(InputSplitter):'
     # List with lines of raw input accumulated so far.
     _buffer_raw = None
 
-    def __init__(self, input_mode=None, physical_line_transforms=None,
+    def __init__(self, line_input_checker=False, physical_line_transforms=None,
                     logical_line_transforms=None, python_line_transforms=None):
-        super(IPythonInputSplitter, self).__init__(input_mode)
+        super(IPythonInputSplitter, self).__init__()
         self._buffer_raw = []
         self._validate = True
         
@@ -528,7 +495,7 b' class IPythonInputSplitter(InputSplitter):'
         if logical_line_transforms is not None:
             self.logical_line_transforms = logical_line_transforms
         else:
-            self.logical_line_transforms = [cellmagic(),
+            self.logical_line_transforms = [cellmagic(end_on_blank_line=line_input_checker),
                                             help_end(),
                                             escaped_commands(),
                                             assign_from_magic(),
@@ -641,44 +608,14 b' class IPythonInputSplitter(InputSplitter):'
         if not lines_list:
             lines_list = ['']
 
-        # Transform logic
-        #
-        # We only apply the line transformers to the input if we have either no
-        # input yet, or complete input, or if the last line of the buffer ends
-        # with ':' (opening an indented block).  This prevents the accidental
-        # transformation of escapes inside multiline expressions like
-        # triple-quoted strings or parenthesized expressions.
-        #
-        # The last heuristic, while ugly, ensures that the first line of an
-        # indented block is correctly transformed.
-        #
-        # FIXME: try to find a cleaner approach for this last bit.
-
-        # If we were in 'block' mode, since we're going to pump the parent
-        # class by hand line by line, we need to temporarily switch out to
-        # 'line' mode, do a single manual reset and then feed the lines one
-        # by one.  Note that this only matters if the input has more than one
-        # line.
-        changed_input_mode = False
-
-        if self.input_mode == 'cell':
-            self.reset()
-            changed_input_mode = True
-            saved_input_mode = 'cell'
-            self.input_mode = 'line'
-
         # Store raw source before applying any transformations to it.  Note
         # that this must be done *after* the reset() call that would otherwise
         # flush the buffer.
         self._store(lines, self._buffer_raw, 'source_raw')
 
-        try:
-            for line in lines_list:
-                out = self.push_line(line)
-        finally:
-            if changed_input_mode:
-                self.input_mode = saved_input_mode
-        
+        for line in lines_list:
+            out = self.push_line(line)
+
         return out
     
     def push_line(self, line):

IPython/core/inputtransformer.py

@@ -60,8 +60,8 b' class InputTransformer(object):'
         will allow instantiation with the decorated object.
         """
         @functools.wraps(func)
-        def transformer_factory():
-            return cls(func)
+        def transformer_factory(**kwargs):
+            return cls(func, **kwargs)
         
         return transformer_factory
 
@@ -84,9 +84,9 b' class StatelessInputTransformer(InputTransformer):'
 
 class CoroutineInputTransformer(InputTransformer):
     """Wrapper for an input transformer implemented as a coroutine."""
-    def __init__(self, coro):
+    def __init__(self, coro, **kwargs):
         # Prime it
-        self.coro = coro()
+        self.coro = coro(**kwargs)
         next(self.coro)
     
     def __repr__(self):
@@ -316,7 +316,7 b' def help_end(line):'
 
 
 @CoroutineInputTransformer.wrap
-def cellmagic():
+def cellmagic(end_on_blank_line=False):
     """Captures & transforms cell magics.
     
     After a cell magic is started, this stores up any lines it gets until it is
@@ -337,7 +337,8 b' def cellmagic():'
         first = line
         body = []
         line = (yield None)
-        while (line is not None) and (line.strip() != ''):
+        while (line is not None) and \
+                                ((line.strip() != '') or not end_on_blank_line):
             body.append(line)
             line = (yield None)
         

IPython/core/interactiveshell.py

@@ -283,10 +283,16 b' class InteractiveShell(SingletonConfigurable):'
     filename = Unicode("<ipython console>")
     ipython_dir= Unicode('', config=True) # Set to get_ipython_dir() in __init__
 
-    # Input splitter, to split entire cells of input into either individual
-    # interactive statements or whole blocks.
+    # Input splitter, to transform input line by line and detect when a block
+    # is ready to be executed.
     input_splitter = Instance('IPython.core.inputsplitter.IPythonInputSplitter',
-                              (), {})
+                              (), {'line_input_checker': True})
+    
+    # This InputSplitter instance is used to transform completed cells before
+    # running them. It allows cell magics to contain blank lines.
+    input_transformer_manager = Instance('IPython.core.inputsplitter.IPythonInputSplitter',
+                                         (), {'line_input_checker': False})
+    
     logstart = CBool(False, config=True, help=
         """
         Start logging to the default log file.
@@ -2581,10 +2587,9 b' class InteractiveShell(SingletonConfigurable):'
         if silent:
             store_history = False
 
-        self.input_splitter.push(raw_cell)
+        self.input_transformer_manager.push(raw_cell)
+        cell = self.input_transformer_manager.source_reset()
 
-        cell = self.input_splitter.source_reset()
-        
         # Our own compiler remembers the __future__ environment. If we want to
         # run code with a separate __future__ environment, use the default
         # compiler

IPython/core/tests/test_inputsplitter.py

@@ -168,9 +168,6 b' class InputSplitterTestCase(unittest.TestCase):'
         self.assertEqual(isp.indent_spaces, 0)
 
     def test_indent2(self):
-        # In cell mode, inputs must be fed in whole blocks, so skip this test
-        if self.isp.input_mode == 'cell': return
-
         isp = self.isp
         isp.push('if 1:')
         self.assertEqual(isp.indent_spaces, 4)
@@ -181,9 +178,6 b' class InputSplitterTestCase(unittest.TestCase):'
         self.assertEqual(isp.indent_spaces, 4)
 
     def test_indent3(self):
-        # In cell mode, inputs must be fed in whole blocks, so skip this test
-        if self.isp.input_mode == 'cell': return
-
         isp = self.isp
         # When a multiline statement contains parens or multiline strings, we
         # shouldn't get confused.
@@ -192,9 +186,6 b' class InputSplitterTestCase(unittest.TestCase):'
         self.assertEqual(isp.indent_spaces, 4)
 
     def test_indent4(self):
-        # In cell mode, inputs must be fed in whole blocks, so skip this test
-        if self.isp.input_mode == 'cell': return
-
         isp = self.isp
         # whitespace after ':' should not screw up indent level
         isp.push('if 1: \n    x=1')
@@ -279,23 +270,12 b' class InputSplitterTestCase(unittest.TestCase):'
         isp.push('  a = 1')
         self.assertFalse(isp.push('b = [1,'))
 
-    def test_replace_mode(self):
-        isp = self.isp
-        isp.input_mode = 'cell'
-        isp.push('x=1')
-        self.assertEqual(isp.source, 'x=1\n')
-        isp.push('x=2')
-        self.assertEqual(isp.source, 'x=2\n')
-
     def test_push_accepts_more(self):
         isp = self.isp
         isp.push('x=1')
         self.assertFalse(isp.push_accepts_more())
 
     def test_push_accepts_more2(self):
-        # In cell mode, inputs must be fed in whole blocks, so skip this test
-        if self.isp.input_mode == 'cell': return
-
         isp = self.isp
         isp.push('if 1:')
         self.assertTrue(isp.push_accepts_more())
@@ -310,9 +290,6 b' class InputSplitterTestCase(unittest.TestCase):'
         self.assertFalse(isp.push_accepts_more())
 
     def test_push_accepts_more4(self):
-        # In cell mode, inputs must be fed in whole blocks, so skip this test
-        if self.isp.input_mode == 'cell': return
-
         isp = self.isp
         # When a multiline statement contains parens or multiline strings, we
         # shouldn't get confused.
@@ -331,14 +308,13 b' class InputSplitterTestCase(unittest.TestCase):'
         self.assertFalse(isp.push_accepts_more())
 
     def test_push_accepts_more5(self):
-        # In cell mode, inputs must be fed in whole blocks, so skip this test
-        if self.isp.input_mode == 'cell': return
-
         isp = self.isp
         isp.push('try:')
         isp.push('    a = 5')
         isp.push('except:')
         isp.push('    raise')
+        # We want to be able to add an else: block at this point, so it should
+        # wait for a blank line.
         self.assertTrue(isp.push_accepts_more())
 
     def test_continuation(self):
@@ -431,7 +407,7 b' class IPythonInputTestCase(InputSplitterTestCase):'
     """
 
     def setUp(self):
-        self.isp = isp.IPythonInputSplitter(input_mode='line')
+        self.isp = isp.IPythonInputSplitter()
 
     def test_syntax(self):
         """Call all single-line syntax tests from the main object"""
@@ -467,32 +443,6 b' class IPythonInputTestCase(InputSplitterTestCase):'
                 self.assertEqual(out.rstrip(), out_t)
                 self.assertEqual(out_raw.rstrip(), raw)
 
-
-class BlockIPythonInputTestCase(IPythonInputTestCase):
-
-    # Deactivate tests that don't make sense for the block mode
-    test_push3 = test_split = lambda s: None
-
-    def setUp(self):
-        self.isp = isp.IPythonInputSplitter(input_mode='cell')
-
-    def test_syntax_multiline(self):
-        isp = self.isp
-        for example in syntax_ml.itervalues():
-            raw_parts = []
-            out_t_parts = []
-            for line_pairs in example:
-                raw_parts, out_t_parts = zip(*line_pairs)
-
-                raw = '\n'.join(r for r in raw_parts if r is not None)
-                out_t = '\n'.join(o for o in out_t_parts if o is not None)
-
-                isp.push(raw)
-                out, out_raw = isp.source_raw_reset()
-                # Match ignoring trailing whitespace
-                self.assertEqual(out.rstrip(), out_t.rstrip())
-                self.assertEqual(out_raw.rstrip(), raw.rstrip())
-
     def test_syntax_multiline_cell(self):
         isp = self.isp
         for example in syntax_ml.itervalues():
@@ -588,39 +538,35 b' class CellMagicsCommon(object):'
         out = sp.source_reset()
         ref = u"get_ipython().run_cell_magic({u}'cellm', {u}'line', {u}'body')\n"
         nt.assert_equal(out, py3compat.u_format(ref))
+    
+    def test_cellmagic_help(self):
+        self.sp.push('%%cellm?')
+        nt.assert_false(self.sp.push_accepts_more())
 
     def tearDown(self):
         self.sp.reset()
 
 
 class CellModeCellMagics(CellMagicsCommon, unittest.TestCase):
-    sp = isp.IPythonInputSplitter(input_mode='cell')
+    sp = isp.IPythonInputSplitter(line_input_checker=False)
 
     def test_incremental(self):
         sp = self.sp
-        src = '%%cellm line2\n'
-        sp.push(src)
+        sp.push('%%cellm firstline\n')
         nt.assert_true(sp.push_accepts_more()) #1
-        src += '\n'
-        sp.push(src)
-        # Note: if we ever change the logic to allow full blank lines (see
-        # _handle_cell_magic), then the following test should change to true
-        nt.assert_false(sp.push_accepts_more()) #2
-        # By now, even with full blanks allowed, a second blank should signal
-        # the end.  For now this test is only a redundancy safety, but don't
-        # delete it in case we change our mind and the previous one goes to
-        # true.
-        src += '\n'
-        sp.push(src)
-        nt.assert_false(sp.push_accepts_more()) #3
-
+        sp.push('line2\n')
+        nt.assert_true(sp.push_accepts_more()) #2
+        sp.push('\n')
+        # This should accept a blank line and carry on until the cell is reset
+        nt.assert_true(sp.push_accepts_more()) #3
 
 class LineModeCellMagics(CellMagicsCommon, unittest.TestCase):
-    sp = isp.IPythonInputSplitter(input_mode='line')
+    sp = isp.IPythonInputSplitter(line_input_checker=True)
 
     def test_incremental(self):
         sp = self.sp
         sp.push('%%cellm line2\n')
         nt.assert_true(sp.push_accepts_more()) #1
         sp.push('\n')
+        # In this case, a blank line should end the cell magic
         nt.assert_false(sp.push_accepts_more()) #2

IPython/core/tests/test_inputtransformer.py

@@ -19,9 +19,9 b' def transform_and_reset(transformer):'
     return transform
 
 # Transformer tests
-def transform_checker(tests, transformer):
+def transform_checker(tests, transformer, **kwargs):
     """Utility to loop over test inputs"""
-    transformer = transformer()
+    transformer = transformer(**kwargs)
     try:
         for inp, tr in tests:
             if inp is None:
@@ -223,7 +223,7 b' syntax_ml = \\'
           ],
          [(u'%%bar 123', None),
           (u'hello', None),
-          (u'', u_fmt("get_ipython().run_cell_magic({u}'bar', {u}'123', {u}'hello')")),
+          (None , u_fmt("get_ipython().run_cell_magic({u}'bar', {u}'123', {u}'hello')")),
           ],
        ],
        
@@ -348,6 +348,12 b' def test_escaped_paren():'
 def test_cellmagic():
     for example in syntax_ml['cellmagic']:
         transform_checker(example, ipt.cellmagic)
+    
+    line_example = [(u'%%bar 123', None),
+                    (u'hello', None),
+                    (u'' , u_fmt("get_ipython().run_cell_magic({u}'bar', {u}'123', {u}'hello')")),
+                   ]
+    transform_checker(line_example, ipt.cellmagic, end_on_blank_line=True)
 
 def test_has_comment():
     tests = [('text', False),

IPython/frontend/qt/console/frontend_widget.py

@@ -146,7 +146,7 b' class FrontendWidget(HistoryConsoleWidget, BaseFrontendMixin):'
         self._copy_raw_action = QtGui.QAction('Copy (Raw Text)', None)
         self._hidden = False
         self._highlighter = FrontendHighlighter(self)
-        self._input_splitter = self._input_splitter_class(input_mode='cell')
+        self._input_splitter = self._input_splitter_class()
         self._kernel_manager = None
         self._request_info = {}
         self._request_info['execute'] = {};
@@ -204,6 +204,7 b' class FrontendWidget(HistoryConsoleWidget, BaseFrontendMixin):'
             prompt created. When triggered by an Enter/Return key press,
             'interactive' is True; otherwise, it is False.
         """
+        self._input_splitter.reset()
         complete = self._input_splitter.push(source)
         if interactive:
             complete = not self._input_splitter.push_accepts_more()

IPython/frontend/terminal/interactiveshell.py

@@ -82,7 +82,7 b' def get_pasted_lines(sentinel, l_input=py3compat.input):'
 class TerminalMagics(Magics):
     def __init__(self, shell):
         super(TerminalMagics, self).__init__(shell)
-        self.input_splitter = IPythonInputSplitter(input_mode='line')
+        self.input_splitter = IPythonInputSplitter()
 
     def cleanup_input(self, block):
         """Apply all possible IPython cleanups to an input block.

docs/source/config/inputtransforms.txt

@@ -35,6 +35,14 b' in attributes of :class:`~IPython.core.inputsplitter.IPythonInputSplitter`:'
   passed to these, but note that function and class *definitions* are still a
   series of separate statements. IPython does not use any of these by default.
 
+An InteractiveShell instance actually has two
+:class:`~IPython.core.inputsplitter.IPythonInputSplitter` instances, as the
+attributes :attr:`~IPython.core.interactiveshell.InteractiveShell.input_splitter`,
+to tell when a block of input is complete, and
+:attr:`~IPython.core.interactiveshell.InteractiveShell.input_transformer_manager`,
+to transform complete cells. If you add a transformer, you should make sure that
+it gets added to both.
+
 Stateless transformations
 -------------------------