Merge branch 'ast-splitter' of https://github.com/takluyver/ipython into takluyver-ast-splitter

IPython/core/compilerop.py

@@ -7,6 +7,7 b' Authors'
 -------
 * Robert Kern
 * Fernando Perez
+* Thomas Kluyver
 """
 
 # Note: though it might be more natural to name this module 'compiler', that
@@ -51,12 +52,12 b' def code_name(code, number=0):'
 # Classes and functions
 #-----------------------------------------------------------------------------
 
-class CachingCompiler(object):
+class CachingCompiler(codeop.Compile):
     """A compiler that caches code compiled from interactive statements.
     """
 
     def __init__(self):
-        self._compiler = codeop.CommandCompiler()
+        codeop.Compile.__init__(self)
         
         # This is ugly, but it must be done this way to allow multiple
         # simultaneous ipython instances to coexist.  Since Python itself
@@ -81,35 +82,30 b' class CachingCompiler(object):'
     def compiler_flags(self):
         """Flags currently active in the compilation process.
         """
-        return self._compiler.compiler.flags
+        return self.flags
+        
+    def cache(self, code, number=0):
+        """Make a name for a block of code, and cache the code.
         
-    def __call__(self, code, symbol, number=0):
-        """Compile some code while caching its contents such that the inspect
-        module can find it later.
-
         Parameters
         ----------
         code : str
-          Source code to be compiled, one or more lines.
-
-        symbol : str
-          One of 'single', 'exec' or 'eval' (see the builtin ``compile``
-          documentation for further details on these fields).
-
-        number : int, optional
-          An integer argument identifying the code, useful for informational
-          purposes in tracebacks (typically it will be the IPython prompt
-          number).
+          The Python source code to cache.
+        number : int
+          A number which forms part of the code's name. Used for the execution
+          counter.
+          
+        Returns
+        -------
+        The name of the cached code (as a string). Pass this as the filename
+        argument to compilation, so that tracebacks are correctly hooked up.
         """
         name = code_name(code, number)
-        code_obj = self._compiler(code, name, symbol)
         entry = (len(code), time.time(),
                  [line+'\n' for line in code.splitlines()], name)
-        # Cache the info both in the linecache (a global cache used internally
-        # by most of Python's inspect/traceback machinery), and in our cache
         linecache.cache[name] = entry
         linecache._ipython_cache[name] = entry
-        return code_obj
+        return name
 
     def check_cache(self, *args):
         """Call linecache.checkcache() safely protecting our cached values.

IPython/core/inputsplitter.py

@@ -166,78 +166,6 b' def get_input_encoding():'
 # Classes and functions for normal Python syntax handling
 #-----------------------------------------------------------------------------
 
-# HACK!  This implementation, written by Robert K a while ago using the
-# compiler module, is more robust than the other one below, but it expects its
-# input to be pure python (no ipython syntax).  For now we're using it as a
-# second-pass splitter after the first pass transforms the input to pure
-# python.
-
-def split_blocks(python):
-    """ Split multiple lines of code into discrete commands that can be
-    executed singly.
-
-    Parameters
-    ----------
-    python : str
-        Pure, exec'able Python code.
-
-    Returns
-    -------
-    commands : list of str
-        Separate commands that can be exec'ed independently.
-    """
-    # compiler.parse treats trailing spaces after a newline as a
-    # SyntaxError.  This is different than codeop.CommandCompiler, which
-    # will compile the trailng spaces just fine.  We simply strip any
-    # trailing whitespace off.  Passing a string with trailing whitespace
-    # to exec will fail however.  There seems to be some inconsistency in
-    # how trailing whitespace is handled, but this seems to work.
-    python_ori = python # save original in case we bail on error
-    python = python.strip()
-
-    # The compiler module will parse the code into an abstract syntax tree.
-    # This has a bug with str("a\nb"), but not str("""a\nb""")!!!
-    try:
-        code_ast = ast.parse(python)
-    except:
-        return [python_ori]
-
-    # Uncomment to help debug the ast tree
-    # for n in code_ast.body:
-    #     print n.lineno,'->',n
-
-    # Each separate command is available by iterating over ast.node. The
-    # lineno attribute is the line number (1-indexed) beginning the commands
-    # suite.
-    # lines ending with ";" yield a Discard Node that doesn't have a lineno
-    # attribute.  These nodes can and should be discarded.  But there are
-    # other situations that cause Discard nodes that shouldn't be discarded.
-    # We might eventually discover other cases where lineno is None and have
-    # to put in a more sophisticated test.
-    linenos = [x.lineno-1 for x in code_ast.body if x.lineno is not None]
-
-    # When we finally get the slices, we will need to slice all the way to
-    # the end even though we don't have a line number for it. Fortunately,
-    # None does the job nicely.
-    linenos.append(None)
-
-    # Same problem at the other end: sometimes the ast tree has its
-    # first complete statement not starting on line 0. In this case
-    # we might miss part of it.  This fixes ticket 266993.  Thanks Gael!
-    linenos[0] = 0
-
-    lines = python.splitlines()
-
-    # Create a list of atomic commands.
-    cmds = []
-    for i, j in zip(linenos[:-1], linenos[1:]):
-        cmd = lines[i:j]
-        if cmd:
-            cmds.append('\n'.join(cmd)+'\n')
-
-    return cmds
-
-
 class InputSplitter(object):
     """An object that can split Python source input in executable blocks.
 
@@ -445,96 +373,18 b' class InputSplitter(object):'
                 if not self._full_dedent:
                     return False
             else:
-                nblocks = len(split_blocks(''.join(self._buffer)))
-                if nblocks==1:
+                try:
+                    code_ast = ast.parse(u''.join(self._buffer))
+                except Exception:
                     return False
+                else:
+                    if len(code_ast.body) == 1:
+                        return False
 
         # When input is complete, then termination is marked by an extra blank
         # line at the end.
         last_line = self.source.splitlines()[-1]
         return bool(last_line and not last_line.isspace())
-        
-    def split_blocks(self, lines):
-        """Split a multiline string into multiple input blocks.
-
-        Note: this method starts by performing a full reset().
-        
-        Parameters
-        ----------
-        lines : str
-          A possibly multiline string.
-
-        Returns
-        -------
-        blocks : list
-          A list of strings, each possibly multiline.  Each string corresponds
-          to a single block that can be compiled in 'single' mode (unless it
-          has a syntax error)."""
-
-        # This code is fairly delicate.  If you make any changes here, make
-        # absolutely sure that you do run the full test suite and ALL tests
-        # pass.
-
-        self.reset()
-        blocks = []
-        
-        # Reversed copy so we can use pop() efficiently and consume the input
-        # as a stack
-        lines = lines.splitlines()[::-1]
-        # Outer loop over all input
-        while lines:
-            #print 'Current lines:', lines  # dbg
-            # Inner loop to build each block
-            while True:
-                # Safety exit from inner loop
-                if not lines:
-                    break
-                # Grab next line but don't push it yet
-                next_line = lines.pop()
-                # Blank/empty lines are pushed as-is
-                if not next_line or next_line.isspace():
-                    self.push(next_line)
-                    continue
-
-                # Check indentation changes caused by the *next* line
-                indent_spaces, _full_dedent = self._find_indent(next_line)
-
-                # If the next line causes a dedent, it can be for two differnt
-                # reasons: either an explicit de-dent by the user or a
-                # return/raise/pass statement.  These MUST be handled
-                # separately:
-                #
-                # 1. the first case is only detected when the actual explicit
-                # dedent happens, and that would be the *first* line of a *new*
-                # block.  Thus, we must put the line back into the input buffer
-                # so that it starts a new block on the next pass.
-                #
-                # 2. the second case is detected in the line before the actual
-                # dedent happens, so , we consume the line and we can break out
-                # to start a new block.
-
-                # Case 1, explicit dedent causes a break.
-                # Note: check that we weren't on the very last line, else we'll
-                # enter an infinite loop adding/removing the last line.
-                if  _full_dedent and lines and not next_line.startswith(' '):
-                    lines.append(next_line)
-                    break
-                
-                # Otherwise any line is pushed
-                self.push(next_line)
-
-                # Case 2, full dedent with full block ready:
-                if _full_dedent or \
-                       self.indent_spaces==0 and not self.push_accepts_more():
-                    break
-            # Form the new block with the current source input
-            blocks.append(self.source_reset())
-            
-        #return blocks
-        # HACK!!! Now that our input is in blocks but guaranteed to be pure
-        # python syntax, feed it back a second time through the AST-based
-        # splitter, which is more accurate than ours.
-        return split_blocks(''.join(blocks))
 
     #------------------------------------------------------------------------
     # Private interface

IPython/core/interactiveshell.py

@@ -20,6 +20,7 b' from __future__ import absolute_import'
 import __builtin__
 import __future__
 import abc
+import ast
 import atexit
 import codeop
 import inspect
@@ -2098,115 +2099,95 b' class InteractiveShell(Configurable, Magic):'
             except:
                 self.showtraceback()
                 warn('Unknown failure executing file: <%s>' % fname)
-
+                
     def run_cell(self, cell, store_history=True):
-        """Run the contents of an entire multiline 'cell' of code, and store it
-        in the history.
-
-        The cell is split into separate blocks which can be executed
-        individually.  Then, based on how many blocks there are, they are
-        executed as follows:
-
-        - A single block: 'single' mode. If it is also a single line, dynamic
-        transformations, including automagic and macros, will be applied.
-
-        If there's more than one block, it depends:
-
-        - if the last one is no more than two lines long, run all but the last
-        in 'exec' mode and the very last one in 'single' mode.  This makes it
-        easy to type simple expressions at the end to see computed values.  -
-        otherwise (last one is also multiline), run all in 'exec' mode
-
-        When code is executed in 'single' mode, :func:`sys.displayhook` fires,
-        results are displayed and output prompts are computed.  In 'exec' mode,
-        no results are displayed unless :func:`print` is called explicitly;
-        this mode is more akin to running a script.
-
+        """Run a complete IPython cell.
+        
         Parameters
         ----------
         cell : str
-          A single or multiline string.
+          The code (including IPython code such as %magic functions) to run.
+        store_history : bool
+          If True, the raw and translated cell will be stored in IPython's
+          history. For user code calling back into IPython's machinery, this
+          should be set to False.
         """
-        # Store the untransformed code
         raw_cell = cell
-        
-        # Code transformation and execution must take place with our
-        # modifications to builtins.
         with self.builtin_trap:
+            cell = self.prefilter_manager.prefilter_lines(cell)
             
-            # We need to break up the input into executable blocks that can
-            # be runin 'single' mode, to provide comfortable user behavior.
-            blocks = self.input_splitter.split_blocks(cell)
-            
-            if not blocks:   # Blank cell
-                return
-            
-            # We only do dynamic transforms on a single line. But a macro
-            # can be expanded to several lines, so we need to split it
-            # into input blocks again.
-            if len(cell.splitlines()) <= 1:
-                cell = self.prefilter_manager.prefilter_line(blocks[0])
-                blocks = self.input_splitter.split_blocks(cell)
-
-            # Store the 'ipython' version of the cell as well, since
-            # that's what needs to go into the translated history and get
-            # executed (the original cell may contain non-python syntax).
-            cell = ''.join(blocks)
-
             # Store raw and processed history
             if store_history:
                 self.history_manager.store_inputs(self.execution_count, 
                                                   cell, raw_cell)
 
             self.logger.log(cell, raw_cell)
-
-            # All user code execution should take place with our
-            # modified displayhook.
+            
+            cell_name = self.compile.cache(cell, self.execution_count)
+            
             with self.display_trap:
-                # Single-block input should behave like an interactive prompt
-                if len(blocks) == 1:
-                    out = self.run_source(blocks[0])
-                    # Write output to the database. Does nothing unless
-                    # history output logging is enabled.
-                    if store_history:
-                        self.history_manager.store_output(self.execution_count)
-                        # Since we return here, we need to update the
-                        # execution count
-                        self.execution_count += 1
-                    return out
-
-                # In multi-block input, if the last block is a simple (one-two
-                # lines) expression, run it in single mode so it produces output.
-                # Otherwise just run it all in 'exec' mode.  This seems like a
-                # reasonable usability design.
-                last = blocks[-1]
-                last_nlines = len(last.splitlines())
+                try:
+                    code_ast = ast.parse(cell, filename=cell_name)
+                except (OverflowError, SyntaxError, ValueError, TypeError, MemoryError):
+                    # Case 1
+                    self.showsyntaxerror()
+                    self.execution_count += 1
+                    return None
+                    
+                interactivity = 'last'      # Last node to be run interactive
+                if len(cell.splitlines()) == 1:
+                    interactivity = 'all'   # Single line; run fully interactive
+
+                self.run_ast_nodes(code_ast.body, cell_name, interactivity)
                 
-                if last_nlines < 2:
-                    # Here we consider the cell split between 'body' and 'last',
-                    # store all history and execute 'body', and if successful, then
-                    # proceed to execute 'last'.
-
-                    # Get the main body to run as a cell
-                    ipy_body = ''.join(blocks[:-1])
-                    retcode = self.run_source(ipy_body, symbol='exec',
-                                              post_execute=False)
-                    if retcode==0:
-                        # Last expression compiled as 'single' so it
-                        # produces output
-                        self.run_source(last)
-                else:
-                    # Run the whole cell as one entity, storing both raw and
-                    # processed input in history
-                    self.run_source(cell, symbol='exec')
-
-        # Write output to the database. Does nothing unless
-        # history output logging is enabled.
         if store_history:
+            # Write output to the database. Does nothing unless
+            # history output logging is enabled.
             self.history_manager.store_output(self.execution_count)
             # Each cell is a *single* input, regardless of how many lines it has
             self.execution_count += 1
-
+            
+    def run_ast_nodes(self, nodelist, cell_name, interactivity='last'):
+        """Run a sequence of AST nodes. The execution mode depends on the
+        interactivity parameter.
+        
+        Parameters
+        ----------
+        nodelist : list
+          A sequence of AST nodes to run.
+        cell_name : str
+          Will be passed to the compiler as the filename of the cell. Typically
+          the value returned by ip.compile.cache(cell).
+        interactivity : str
+          'all', 'last' or 'none', specifying which nodes should be run
+          interactively (displaying output from expressions). Other values for
+          this parameter will raise a ValueError.
+        """
+        if not nodelist:
+            return
+        
+        if interactivity == 'none':
+            to_run_exec, to_run_interactive = nodelist, []
+        elif interactivity == 'last':
+            to_run_exec, to_run_interactive = nodelist[:-1], nodelist[-1:]
+        elif interactivity == 'all':
+            to_run_exec, to_run_interactive = [], nodelist
+        else:
+            raise ValueError("Interactivity was %r" % interactivity)
+            
+        exec_count = self.execution_count
+        if to_run_exec:
+            mod = ast.Module(to_run_exec)
+            self.code_to_run = code = self.compile(mod, cell_name, "exec")
+            if self.run_code(code) == 1:
+                return
+                
+        if to_run_interactive:
+            mod = ast.Interactive(to_run_interactive)
+            self.code_to_run = code = self.compile(mod, cell_name, "single")
+            return self.run_code(code)
+    
+    
     # PENDING REMOVAL: this method is slated for deletion, once our new
     # input logic has been 100% moved to frontends and is stable.
     def runlines(self, lines, clean=False):
@@ -2296,7 +2277,8 b' class InteractiveShell(Configurable, Magic):'
             print 'encoding', self.stdin_encoding  # dbg
         
         try:
-            code = self.compile(usource, symbol, self.execution_count)
+            code_name = self.compile.cache(usource, self.execution_count)
+            code = self.compile(usource, code_name, symbol)
         except (OverflowError, SyntaxError, ValueError, TypeError, MemoryError):
             # Case 1
             self.showsyntaxerror(filename)

IPython/core/tests/test_compilerop.py

@@ -40,12 +40,12 b' def test_code_name2():'
     nt.assert_true(name.startswith('<ipython-input-9'))
 
 
-def test_compiler():
+def test_cache():
     """Test the compiler correctly compiles and caches inputs
     """
     cp = compilerop.CachingCompiler()
     ncache = len(linecache.cache)
-    cp('x=1', 'single')
+    cp.cache('x=1')
     nt.assert_true(len(linecache.cache) > ncache)
 
 def setUp():
@@ -53,10 +53,10 b' def setUp():'
     # as GTK, can change the default encoding, which can hide bugs.)
     nt.assert_equal(sys.getdefaultencoding(), "ascii")
 
-def test_compiler_unicode():
+def test_cache_unicode():
     cp = compilerop.CachingCompiler()
     ncache = len(linecache.cache)
-    cp(u"t = 'žćčšđ'", "single")
+    cp.cache(u"t = 'žćčšđ'")
     nt.assert_true(len(linecache.cache) > ncache)
 
 def test_compiler_check_cache():
@@ -64,7 +64,7 b' def test_compiler_check_cache():'
     """
     # Rather simple-minded tests that just exercise the API
     cp = compilerop.CachingCompiler()
-    cp('x=1', 'single', 99)
+    cp.cache('x=1', 99)
     # Ensure now that after clearing the cache, our entries survive
     cp.check_cache()
     for k in linecache.cache:

IPython/core/tests/test_handlers.py

@@ -43,9 +43,7 b' def run(tests):'
     for pre, post in tests:
         global num_tests
         num_tests += 1        
-        ip.runlines(pre)
-        ip.runlines('_i')  # Not sure why I need this...
-        actual = ip.user_ns['_i']
+        actual = ip.prefilter_manager.prefilter_lines(pre)
         if actual != None:
             actual = actual.rstrip('\n')
         if actual != post:

IPython/core/tests/test_inputsplitter.py

@@ -286,92 +286,6 b' class InputSplitterTestCase(unittest.TestCase):'
         isp.push('run foo')
         self.assertFalse(isp.push_accepts_more())
 
-    def check_split(self, block_lines, compile=True):
-        blocks = assemble(block_lines)
-        lines = ''.join(blocks)
-        oblock = self.isp.split_blocks(lines)
-        self.assertEqual(oblock, blocks)
-        if compile:
-            for block in blocks:
-                self.isp._compile(block)
-
-    def test_split(self):
-        # All blocks of input we want to test in a list.  The format for each
-        # block is a list of lists, with each inner lists consisting of all the
-        # lines (as single-lines) that should make up a sub-block.
-
-        # Note: do NOT put here sub-blocks that don't compile, as the
-        # check_split() routine makes a final verification pass to check that
-        # each sub_block, as returned by split_blocks(), does compile
-        # correctly.
-        all_blocks = [ [['x=1']],
-
-                       [['x=1'],
-                        ['y=2']],
-
-                       [['x=1',
-                         '# a comment'],
-                        ['y=11']],
-
-                       [['if 1:',
-                         '  x=1'],
-                        ['y=3']],
-
-                       [['def f(x):',
-                         '  return x'],
-                        ['x=1']],
-
-                       [['def f(x):',
-                         '  x+=1',
-                         '  ',
-                         '  return x'],
-                        ['x=1']],
-
-                       [['def f(x):',
-                         '  if x>0:',
-                         '    y=1',
-                         '  # a comment',
-                         '  else:',
-                         '    y=4',
-                         ' ',
-                         '  return y'],
-                        ['x=1'],
-                        ['if 1:',
-                         '  y=11'] ],
-                       
-                       [['for i in range(10):'
-                         '  x=i**2']],
-
-                       [['for i in range(10):'
-                         '  x=i**2'],
-                        ['z = 1']],
-
-                       [['"asdf"']],
-
-                       [['"asdf"'],
-                        ['10'],
-                       ],
-
-                       [['"""foo',
-                         'bar"""']],
-                       ]
-        for block_lines in all_blocks:
-            self.check_split(block_lines)
-        
-    def test_split_syntax_errors(self):
-        # Block splitting with invalid syntax
-        all_blocks = [ [['a syntax error']],
-            
-                       [['x=1',
-                         'another syntax error']],
-
-                       [['for i in range(10):'
-                         '  yet another error']],
-                       
-                       ]
-        for block_lines in all_blocks:
-            self.check_split(block_lines, compile=False)
-
     def test_unicode(self):
         self.isp.push(u"Pérez")
         self.isp.push(u'\xc3\xa9')

IPython/core/tests/test_interactiveshell.py

@@ -56,7 +56,6 b' class InteractiveShellTestCase(unittest.TestCase):'
         self.assertEquals(ip.user_ns['x'], 2)
         self.assertEquals(ip.user_ns['y'], 3)
 
-    @dec.skip_known_failure
     def test_multiline_string_cells(self):
         "Code sprinkled with multiline strings should execute (GH-306)"
         ip = get_ipython()

IPython/lib/tests/test_irunner.py

@@ -97,10 +97,10 b' In [7]: autocall 0'
 Automatic calling is: OFF
 
 In [8]: cos pi
-   File "<ipython-input-8-6bd7313dd9a9>", line 1
+   File "<ipython-input-8-586f1104ea44>", line 1
      cos pi
           ^
-SyntaxError: invalid syntax
+SyntaxError: unexpected EOF while parsing
 
 
 In [9]: cos(pi)