Add experimental support for cell-based execution.

For now the implementation is a bit hackish, but it does already allow
pasting very complex examples such as:

http://matplotlib.sourceforge.net/examples/pylab_examples/demo_agg_filter.html

which before couldn't be executed.  We'll need to test it further.

IPython/core/inputsplitter.py

@@ -159,6 +159,88 b' def get_input_encoding():'
 # Classes and functions for normal Python syntax handling
 #-----------------------------------------------------------------------------
 
+# HACK!  This implementation, written by Robert K a while ago using the
+# compiler module, is more robust than the other one below, but it expects its
+# input to be pure python (no ipython syntax).  For now we're using it as a
+# second-pass splitter after the first pass transforms the input to pure
+# python.
+
+def split_blocks(python):
+    """ Split multiple lines of code into discrete commands that can be
+    executed singly.
+
+    Parameters
+    ----------
+    python : str
+        Pure, exec'able Python code.
+
+    Returns
+    -------
+    commands : list of str
+        Separate commands that can be exec'ed independently.
+    """
+
+    import compiler
+    
+    # compiler.parse treats trailing spaces after a newline as a
+    # SyntaxError.  This is different than codeop.CommandCompiler, which
+    # will compile the trailng spaces just fine.  We simply strip any
+    # trailing whitespace off.  Passing a string with trailing whitespace
+    # to exec will fail however.  There seems to be some inconsistency in
+    # how trailing whitespace is handled, but this seems to work.
+    python_ori = python # save original in case we bail on error
+    python = python.strip()
+
+    # The compiler module does not like unicode. We need to convert
+    # it encode it:
+    if isinstance(python, unicode):
+        # Use the utf-8-sig BOM so the compiler detects this a UTF-8
+        # encode string.
+        python = '\xef\xbb\xbf' + python.encode('utf-8')
+
+    # The compiler module will parse the code into an abstract syntax tree.
+    # This has a bug with str("a\nb"), but not str("""a\nb""")!!!
+    try:
+        ast = compiler.parse(python)
+    except:
+        return [python_ori]
+
+    # Uncomment to help debug the ast tree
+    # for n in ast.node:
+    #     print n.lineno,'->',n
+
+    # Each separate command is available by iterating over ast.node. The
+    # lineno attribute is the line number (1-indexed) beginning the commands
+    # suite.
+    # lines ending with ";" yield a Discard Node that doesn't have a lineno
+    # attribute.  These nodes can and should be discarded.  But there are
+    # other situations that cause Discard nodes that shouldn't be discarded.
+    # We might eventually discover other cases where lineno is None and have
+    # to put in a more sophisticated test.
+    linenos = [x.lineno-1 for x in ast.node if x.lineno is not None]
+
+    # When we finally get the slices, we will need to slice all the way to
+    # the end even though we don't have a line number for it. Fortunately,
+    # None does the job nicely.
+    linenos.append(None)
+
+    # Same problem at the other end: sometimes the ast tree has its
+    # first complete statement not starting on line 0. In this case
+    # we might miss part of it.  This fixes ticket 266993.  Thanks Gael!
+    linenos[0] = 0
+
+    lines = python.splitlines()
+
+    # Create a list of atomic commands.
+    cmds = []
+    for i, j in zip(linenos[:-1], linenos[1:]):
+        cmd = lines[i:j]
+        if cmd:
+            cmds.append('\n'.join(cmd)+'\n')
+
+    return cmds
+
+
 class InputSplitter(object):
     """An object that can split Python source input in executable blocks.
 
@@ -431,7 +513,11 b' class InputSplitter(object):'
             # Form the new block with the current source input
             blocks.append(self.source_reset())
             
-        return blocks
+        #return blocks
+        # HACK!!! Now that our input is in blocks but guaranteed to be pure
+        # python syntax, feed it back a second time through the AST-based
+        # splitter, which is more accurate than ours.
+        return split_blocks(''.join(blocks))
 
     #------------------------------------------------------------------------
     # Private interface

IPython/core/interactiveshell.py

@@ -46,6 +46,7 b' from IPython.core.error import TryNext, UsageError'
 from IPython.core.extensions import ExtensionManager
 from IPython.core.fakemodule import FakeModule, init_fakemod_dict
 from IPython.core.inputlist import InputList
+from IPython.core.inputsplitter import IPythonInputSplitter
 from IPython.core.logger import Logger
 from IPython.core.magic import Magic
 from IPython.core.payload import PayloadManager
@@ -154,6 +155,7 b' class InteractiveShell(Configurable, Magic):'
     exit_now = CBool(False)
     filename = Str("<ipython console>")
     ipython_dir= Unicode('', config=True) # Set to get_ipython_dir() in __init__
+    input_splitter = Instance('IPython.core.inputsplitter.IPythonInputSplitter')
     logstart = CBool(False, config=True)
     logfile = Str('', config=True)
     logappend = Str('', config=True)
@@ -212,7 +214,7 b' class InteractiveShell(Configurable, Magic):'
 
     def __init__(self, config=None, ipython_dir=None,
                  user_ns=None, user_global_ns=None,
-                 custom_exceptions=((),None)):
+                 custom_exceptions=((), None)):
 
         # This is where traits with a config_key argument are updated
         # from the values on config.
@@ -252,7 +254,7 b' class InteractiveShell(Configurable, Magic):'
         # pre_config_initialization
         self.init_shadow_hist()
 
-        # The next section should contain averything that was in ipmaker.
+        # The next section should contain everything that was in ipmaker.
         self.init_logstart()
 
         # The following was in post_config_initialization
@@ -386,6 +388,10 b' class InteractiveShell(Configurable, Magic):'
         # Indentation management
         self.indent_current_nsp = 0
 
+        # Input splitter, to split entire cells of input into either individual
+        # interactive statements or whole blocks.
+        self.input_splitter = IPythonInputSplitter()
+
     def init_encoding(self):
         # Get system encoding at startup time.  Certain terminals (like Emacs
         # under Win32 have it set to None, and we need to have a known valid
@@ -2061,6 +2067,46 b' class InteractiveShell(Configurable, Magic):'
                 self.showtraceback()
                 warn('Unknown failure executing file: <%s>' % fname)
 
+    def run_cell(self, cell):
+        """Run the contents of an entire multiline 'cell' of code.
+
+        The cell is split into separate blocks which can be executed
+        individually.  Then, based on how many blocks there are, they are
+        executed as follows:
+
+        - A single block: 'single' mode.
+
+        If there's more than one block, it depends:
+
+        - if the last one is a single line long, run all but the last in
+        'exec' mode and the very last one in 'single' mode.  This makes it
+        easy to type simple expressions at the end to see computed values.
+        - otherwise (last one is also multiline), run all in 'exec' mode
+
+        When code is executed in 'single' mode, :func:`sys.displayhook` fires,
+        results are displayed and output prompts are computed.  In 'exec' mode,
+        no results are displayed unless :func:`print` is called explicitly;
+        this mode is more akin to running a script.
+
+        Parameters
+        ----------
+        cell : str
+          A single or multiline string.
+        """
+        blocks = self.input_splitter.split_blocks(cell)
+        if not blocks:
+            return
+
+        if len(blocks) == 1:
+            self.runlines(blocks[0])
+
+        last = blocks[-1]
+        if len(last.splitlines()) < 2:
+            map(self.runcode, blocks[:-1])
+            self.runlines(last)
+        else:
+            map(self.runcode, blocks)
+
     def runlines(self, lines, clean=False):
         """Run a string of one or more lines of source.
 
@@ -2166,7 +2212,7 b' class InteractiveShell(Configurable, Magic):'
         else:
             return None
 
-    def runcode(self,code_obj):
+    def runcode(self, code_obj):
         """Execute a code object.
 
         When an exception occurs, self.showtraceback() is called to display a

IPython/core/tests/test_inputsplitter.py

@@ -278,8 +278,8 b' class InputSplitterTestCase(unittest.TestCase):'
                        [['x=1'],
                         ['y=2']],
 
-                       [['x=1'],
-                        ['# a comment'],
+                       [['x=1',
+                         '# a comment'],
                         ['y=11']],
 
                        [['if 1:',
@@ -322,11 +322,11 b' class InputSplitterTestCase(unittest.TestCase):'
         # Block splitting with invalid syntax
         all_blocks = [ [['a syntax error']],
             
-                       [['x=1'],
-                        ['a syntax error']],
+                       [['x=1',
+                         'another syntax error']],
 
                        [['for i in range(10):'
-                         '  an error']],
+                         '  yet another error']],
                        
                        ]
         for block_lines in all_blocks:

IPython/zmq/ipkernel.py

@@ -179,7 +179,12 b' class Kernel(Configurable):'
             else:
                 # FIXME: runlines calls the exception handler itself.
                 shell._reply_content = None
-                shell.runlines(code)
+
+                # Experimental: cell mode!  Test more before turning into
+                # default and removing the hacks around runlines.
+                shell.run_cell(code)
+                # For now leave this here until we're sure we can stop using it
+                #shell.runlines(code)
         except:
             status = u'error'
             # FIXME: this code right now isn't being used yet by default,

IPython/zmq/zmqshell.py

@@ -26,12 +26,12 b' from IPython.core.interactiveshell import ('
 )
 from IPython.core.displayhook import DisplayHook
 from IPython.core.macro import Macro
+from IPython.core.payloadpage import install_payload_page
 from IPython.utils.path import get_py_filename
 from IPython.utils.text import StringTypes
 from IPython.utils.traitlets import Instance, Type, Dict
 from IPython.utils.warn import warn
 from IPython.zmq.session import extract_header
-from IPython.core.payloadpage import install_payload_page
 from session import Session
 
 #-----------------------------------------------------------------------------