Merge branch 'blockbreaker' of git://github.com/fperez/ipython into qtfrontend

Conflicts:
	IPython/core/blockbreaker.py

IPython/core/tests/test_inputsplitter.py

@@ -0,0 +1,346 b''
+"""Tests for the inputsplitter module.
+"""
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2010  The IPython Development Team
+#
+#  Distributed under the terms of the BSD License.  The full license is in
+#  the file COPYING, distributed as part of this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+# stdlib
+import unittest
+
+# Third party
+import nose.tools as nt
+
+# Our own
+from IPython.core import inputsplitter as isp
+
+#-----------------------------------------------------------------------------
+# Semi-complete examples (also used as tests)
+#-----------------------------------------------------------------------------
+def mini_interactive_loop(raw_input):
+    """Minimal example of the logic of an interactive interpreter loop.
+
+    This serves as an example, and it is used by the test system with a fake
+    raw_input that simulates interactive input."""
+
+    from IPython.core.inputsplitter import InputSplitter
+    
+    isp = InputSplitter()
+    # In practice, this input loop would be wrapped in an outside loop to read
+    # input indefinitely, until some exit/quit command was issued.  Here we
+    # only illustrate the basic inner loop.
+    while isp.push_accepts_more():
+        indent = ' '*isp.indent_spaces
+        prompt = '>>> ' + indent
+        line = indent + raw_input(prompt)
+        isp.push(line)
+
+    # Here we just return input so we can use it in a test suite, but a real
+    # interpreter would instead send it for execution somewhere.
+    src = isp.source_reset()
+    print 'Input source was:\n', src
+    return src
+
+#-----------------------------------------------------------------------------
+# Test utilities, just for local use
+#-----------------------------------------------------------------------------
+
+def assemble(block):
+    """Assemble a block into multi-line sub-blocks."""
+    return ['\n'.join(sub_block)+'\n' for sub_block in block]
+
+
+def pseudo_input(lines):
+    """Return a function that acts like raw_input but feeds the input list."""
+    ilines = iter(lines)
+    def raw_in(prompt):
+        try:
+            return next(ilines)
+        except StopIteration:
+            return ''
+    return raw_in
+
+#-----------------------------------------------------------------------------
+# Tests
+#-----------------------------------------------------------------------------
+def test_spaces():
+    tests = [('', 0),
+             (' ', 1),
+             ('\n', 0),
+             (' \n', 1),
+             ('x', 0),
+             (' x', 1),
+             ('  x',2),
+             ('    x',4),
+             # Note: tabs are counted as a single whitespace!
+             ('\tx', 1),
+             ('\t x', 2),
+             ]
+    
+    for s, nsp in tests:
+        nt.assert_equal(isp.num_ini_spaces(s), nsp)
+
+
+def test_remove_comments():
+    tests = [('text', 'text'),
+             ('text # comment', 'text '),
+             ('text # comment\n', 'text \n'),
+             ('text # comment \n', 'text \n'),
+             ('line # c \nline\n','line \nline\n'),
+             ('line # c \nline#c2  \nline\nline #c\n\n',
+              'line \nline\nline\nline \n\n'),
+             ]
+
+    for inp, out in tests:
+        nt.assert_equal(isp.remove_comments(inp), out)
+
+
+def test_get_input_encoding():
+    encoding = isp.get_input_encoding()
+    nt.assert_true(isinstance(encoding, basestring))
+    # simple-minded check that at least encoding a simple string works with the
+    # encoding we got.
+    nt.assert_equal('test'.encode(encoding), 'test')
+
+
+class InputSplitterTestCase(unittest.TestCase):
+    def setUp(self):
+        self.isp = isp.InputSplitter()
+
+    def test_reset(self):
+        isp = self.isp
+        isp.push('x=1')
+        isp.reset()
+        self.assertEqual(isp._buffer, [])
+        self.assertEqual(isp.indent_spaces, 0)
+        self.assertEqual(isp.source, '')
+        self.assertEqual(isp.code, None)
+        self.assertEqual(isp._is_complete, False)
+
+    def test_source(self):
+        self.isp._store('1')
+        self.isp._store('2')
+        self.assertEqual(self.isp.source, '1\n2\n')
+        self.assertTrue(len(self.isp._buffer)>0)
+        self.assertEqual(self.isp.source_reset(), '1\n2\n')
+        self.assertEqual(self.isp._buffer, [])
+        self.assertEqual(self.isp.source, '')
+        
+    def test_indent(self):
+        isp = self.isp # shorthand
+        isp.push('x=1')
+        self.assertEqual(isp.indent_spaces, 0)
+        isp.push('if 1:\n    x=1')
+        self.assertEqual(isp.indent_spaces, 4)
+        isp.push('y=2\n')
+        self.assertEqual(isp.indent_spaces, 0)
+        isp.push('if 1:')
+        self.assertEqual(isp.indent_spaces, 4)
+        isp.push('    x=1')
+        self.assertEqual(isp.indent_spaces, 4)
+        # Blank lines shouldn't change the indent level
+        isp.push(' '*2)
+        self.assertEqual(isp.indent_spaces, 4)
+
+    def test_indent2(self):
+        isp = self.isp
+        # When a multiline statement contains parens or multiline strings, we
+        # shouldn't get confused.
+        isp.push("if 1:")
+        isp.push("    x = (1+\n    2)")
+        self.assertEqual(isp.indent_spaces, 4)
+
+    def test_dedent(self):
+        isp = self.isp # shorthand
+        isp.push('if 1:')
+        self.assertEqual(isp.indent_spaces, 4)
+        isp.push('    pass')
+        self.assertEqual(isp.indent_spaces, 0)
+        
+    def test_push(self):
+        isp = self.isp
+        self.assertTrue(isp.push('x=1'))
+
+    def test_push2(self):
+        isp = self.isp
+        self.assertFalse(isp.push('if 1:'))
+        for line in ['  x=1', '# a comment', '  y=2']:
+            self.assertTrue(isp.push(line))
+            
+    def test_push3(self):
+        """Test input with leading whitespace"""
+        isp = self.isp
+        isp.push('  x=1')
+        isp.push('  y=2')
+        self.assertEqual(isp.source, 'if 1:\n  x=1\n  y=2\n')
+
+    def test_replace_mode(self):
+        isp = self.isp
+        isp.input_mode = 'replace'
+        isp.push('x=1')
+        self.assertEqual(isp.source, 'x=1\n')
+        isp.push('x=2')
+        self.assertEqual(isp.source, 'x=2\n')
+
+    def test_push_accepts_more(self):
+        isp = self.isp
+        isp.push('x=1')
+        self.assertFalse(isp.push_accepts_more())
+
+    def test_push_accepts_more2(self):
+        isp = self.isp
+        isp.push('if 1:')
+        self.assertTrue(isp.push_accepts_more())
+        isp.push('  x=1')
+        self.assertTrue(isp.push_accepts_more())
+        isp.push('')
+        self.assertFalse(isp.push_accepts_more())
+        
+    def test_push_accepts_more3(self):
+        isp = self.isp
+        isp.push("x = (2+\n3)")
+        self.assertFalse(isp.push_accepts_more())
+
+    def test_push_accepts_more4(self):
+        isp = self.isp
+        # When a multiline statement contains parens or multiline strings, we
+        # shouldn't get confused.
+        # FIXME: we should be able to better handle de-dents in statements like
+        # multiline strings and multiline expressions (continued with \ or
+        # parens).  Right now we aren't handling the indentation tracking quite
+        # correctly with this, though in practice it may not be too much of a
+        # problem.  We'll need to see.
+        isp.push("if 1:")
+        isp.push("    x = (2+")
+        isp.push("    3)")
+        self.assertTrue(isp.push_accepts_more())
+        isp.push("    y = 3")
+        self.assertTrue(isp.push_accepts_more())
+        isp.push('')
+        self.assertFalse(isp.push_accepts_more())
+
+    def test_syntax_error(self):
+        isp = self.isp
+        # Syntax errors immediately produce a 'ready' block, so the invalid
+        # Python can be sent to the kernel for evaluation with possible ipython
+        # special-syntax conversion.
+        isp.push('run foo')
+        self.assertFalse(isp.push_accepts_more())
+
+    def check_split(self, block_lines, compile=True):
+        blocks = assemble(block_lines)
+        lines = ''.join(blocks)
+        oblock = self.isp.split_blocks(lines)
+        self.assertEqual(oblock, blocks)
+        if compile:
+            for block in blocks:
+                self.isp._compile(block)
+
+    def test_split(self):
+        # All blocks of input we want to test in a list.  The format for each
+        # block is a list of lists, with each inner lists consisting of all the
+        # lines (as single-lines) that should make up a sub-block.
+
+        # Note: do NOT put here sub-blocks that don't compile, as the
+        # check_split() routine makes a final verification pass to check that
+        # each sub_block, as returned by split_blocks(), does compile
+        # correctly.
+        all_blocks = [ [['x=1']],
+
+                       [['x=1'],
+                        ['y=2']],
+
+                       [['x=1'],
+                        ['# a comment'],
+                        ['y=11']],
+
+                       [['if 1:',
+                         '  x=1'],
+                        ['y=3']],
+
+                       [['def f(x):',
+                         '  return x'],
+                        ['x=1']],
+
+                       [['def f(x):',
+                         '  x+=1',
+                         '  ',
+                         '  return x'],
+                        ['x=1']],
+
+                       [['def f(x):',
+                         '  if x>0:',
+                         '    y=1',
+                         '  # a comment',
+                         '  else:',
+                         '    y=4',
+                         ' ',
+                         '  return y'],
+                        ['x=1'],
+                        ['if 1:',
+                         '  y=11'] ],
+                       
+                       [['for i in range(10):'
+                         '  x=i**2']],
+
+                       [['for i in range(10):'
+                         '  x=i**2'],
+                        ['z = 1']],
+                       ]
+        for block_lines in all_blocks:
+            self.check_split(block_lines)
+        
+    def test_split_syntax_errors(self):
+        # Block splitting with invalid syntax
+        all_blocks = [ [['a syntax error']],
+            
+                       [['x=1'],
+                        ['a syntax error']],
+
+                       [['for i in range(10):'
+                         '  an error']],
+                       
+                       ]
+        for block_lines in all_blocks:
+            self.check_split(block_lines, compile=False)
+
+
+class InteractiveLoopTestCase(unittest.TestCase):
+    """Tests for an interactive loop like a python shell.
+    """
+    def check_ns(self, lines, ns):
+        """Validate that the given input lines produce the resulting namespace.
+
+        Note: the input lines are given exactly as they would be typed in an
+        auto-indenting environment, as mini_interactive_loop above already does
+        auto-indenting and prepends spaces to the input.
+        """
+        src = mini_interactive_loop(pseudo_input(lines))
+        test_ns = {}
+        exec src in test_ns
+        # We can't check that the provided ns is identical to the test_ns,
+        # because Python fills test_ns with extra keys (copyright, etc).  But
+        # we can check that the given dict is *contained* in test_ns
+        for k,v in ns.items():
+            self.assertEqual(test_ns[k], v)
+        
+    def test_simple(self):
+        self.check_ns(['x=1'], dict(x=1))
+
+    def test_simple2(self):
+        self.check_ns(['if 1:', 'x=2'], dict(x=2))
+
+    def test_xy(self):
+        self.check_ns(['x=1; y=2'], dict(x=1, y=2))
+
+    def test_abc(self):
+        self.check_ns(['if 1:','a=1','b=2','c=3'], dict(a=1, b=2, c=3))
+    
+    def test_multi(self):
+        self.check_ns(['x =(1+','1+','2)'], dict(x=4))
+    

IPython/core/inputsplitter.py

@@ -1,12 +1,11 b''
 """Analysis of text input into executable blocks.
 
-This is a simple example of how an interactive terminal-based client can use
-this tool::
+The main class in this module, :class:`InputSplitter`, is designed to break
+input from either interactive, line-by-line environments or block-based ones,
+into standalone blocks that can be executed by Python as 'single' statements
+(thus triggering sys.displayhook).
 
-    bb = BlockBreaker()
-    while not bb.interactive_block_ready():
-        bb.push(raw_input('>>> '))
-    print 'Input source was:\n', bb.source,
+For more details, see the class docstring below.
 """
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2010  The IPython Development Team
@@ -43,6 +42,10 b' def num_ini_spaces(s):'
     Parameters
     ----------
     s : string
+
+    Returns
+    -------
+    n : int
     """
 
     ini_spaces = ini_spaces_re.match(s)
@@ -72,7 +75,6 b' def remove_comments(src):'
 
 def get_input_encoding():
     """Return the default standard input encoding."""
-    
     # There are strange environments for which sys.stdin.encoding is None. We
     # ensure that a valid encoding is returned.
     encoding = getattr(sys.stdin, 'encoding', None)
@@ -84,29 +86,64 b' def get_input_encoding():'
 # Classes and functions
 #-----------------------------------------------------------------------------
 
-class BlockBreaker(object):
-    # Command compiler
-    compile = None
-    # Number of spaces of indentation
+class InputSplitter(object):
+    """An object that can split Python source input in executable blocks.
+
+    This object is designed to be used in one of two basic modes:
+
+    1. By feeding it python source line-by-line, using :meth:`push`.  In this
+       mode, it will return on each push whether the currently pushed code
+       could be executed already.  In addition, it provides a method called
+       :meth:`push_accepts_more` that can be used to query whether more input
+       can be pushed into a single interactive block.
+
+    2. By calling :meth:`split_blocks` with a single, multiline Python string,
+       that is then split into blocks each of which can be executed
+       interactively as a single statement.
+
+    This is a simple example of how an interactive terminal-based client can use
+    this tool::
+
+        isp = InputSplitter()
+        while isp.push_accepts_more():
+            indent = ' '*isp.indent_spaces
+            prompt = '>>> ' + indent
+            line = indent + raw_input(prompt)
+            isp.push(line)
+        print 'Input source was:\n', isp.source_reset(),
+    """
+    # Number of spaces of indentation computed from input that has been pushed
+    # so far.  This is the attributes callers should query to get the current
+    # indentation level, in order to provide auto-indent facilities.
     indent_spaces = 0
-    # String, indicating the default input encoding
+    # String, indicating the default input encoding.  It is computed by default
+    # at initialization time via get_input_encoding(), but it can be reset by a
+    # client with specific knowledge of the encoding.
     encoding = ''
-    # String where the current full source input is stored, properly encoded
+    # String where the current full source input is stored, properly encoded.
+    # Reading this attribute is the normal way of querying the currently pushed
+    # source code, that has been properly encoded.
     source = ''
-    # Code object corresponding to the current source
+    # Code object corresponding to the current source.  It is automatically
+    # synced to the source, so it can be queried at any time to obtain the code
+    # object; it will be None if the source doesn't compile to valid Python.
     code = None
-    # Boolean indicating whether the current block is complete
-    is_complete = None
     # Input mode
     input_mode = 'append'
     
     # Private attributes
     
-    # List
+    # List with lines of input accumulated so far
     _buffer = None
+    # Command compiler
+    _compile = None
+    # Mark when input has changed indentation all the way back to flush-left
+    _full_dedent = False
+    # Boolean indicating whether the current block is complete
+    _is_complete = None
     
     def __init__(self, input_mode=None):
-        """Create a new BlockBreaker instance.
+        """Create a new InputSplitter instance.
 
         Parameters
         ----------
@@ -122,9 +159,9 b' class BlockBreaker(object):'
           while block-oriented ones will want to use 'replace'.
         """
         self._buffer = []
-        self.compile = codeop.CommandCompiler()
+        self._compile = codeop.CommandCompiler()
         self.encoding = get_input_encoding()
-        self.input_mode = BlockBreaker.input_mode if input_mode is None \
+        self.input_mode = InputSplitter.input_mode if input_mode is None \
                           else input_mode
 
     def reset(self):
@@ -133,6 +170,8 b' class BlockBreaker(object):'
         self._buffer[:] = []
         self.source = ''
         self.code = None
+        self._is_complete = False
+        self._full_dedent = False
 
     def source_reset(self):
         """Return the input source and perform a full reset.
@@ -147,7 +186,8 b' class BlockBreaker(object):'
         This stores the given lines and returns a status code indicating
         whether the code forms a complete Python block or not.
 
-        Any exceptions generated in compilation are allowed to propagate.
+        Any exceptions generated in compilation are swallowed, but if an
+        exception was produced, the method returns True.
 
         Parameters
         ----------
@@ -159,8 +199,8 b' class BlockBreaker(object):'
         is_complete : boolean
           True if the current input source (the result of the current input
         plus prior inputs) forms a complete Python execution block.  Note that
-        this value is also stored as an attribute so it can be queried at any
-        time.
+        this value is also stored as a private attribute (_is_complete), so it
+        can be queried at any time.
         """
         if self.input_mode == 'replace':
             self.reset()
@@ -175,34 +215,37 b' class BlockBreaker(object):'
         self._store(lines)
         source = self.source
 
-        # Before calling compile(), reset the code object to None so that if an
+        # Before calling _compile(), reset the code object to None so that if an
         # exception is raised in compilation, we don't mislead by having
         # inconsistent code/source attributes.
-        self.code, self.is_complete = None, None
+        self.code, self._is_complete = None, None
+
+        self._update_indent(lines)
         try:
-            self.code = self.compile(source)
+            self.code = self._compile(source)
         # Invalid syntax can produce any of a number of different errors from
         # inside the compiler, so we have to catch them all.  Syntax errors
         # immediately produce a 'ready' block, so the invalid Python can be
         # sent to the kernel for evaluation with possible ipython
         # special-syntax conversion.
-        except (SyntaxError, OverflowError, ValueError, TypeError, MemoryError):
-            self.is_complete = True
+        except (SyntaxError, OverflowError, ValueError, TypeError,
+                MemoryError):
+            self._is_complete = True
         else:
             # Compilation didn't produce any exceptions (though it may not have
             # given a complete code object)
-            self.is_complete = self.code is not None
-            self._update_indent(lines)
+            self._is_complete = self.code is not None
 
-        return self.is_complete
+        return self._is_complete
 
-    def interactive_block_ready(self):
-        """Return whether a block of interactive input is ready for execution.
+    def push_accepts_more(self):
+        """Return whether a block of interactive input can accept more input.
 
         This method is meant to be used by line-oriented frontends, who need to
         guess whether a block is complete or not based solely on prior and
-        current input lines.  The BlockBreaker considers it has a complete
-        interactive block when *all* of the following are true:
+        current input lines.  The InputSplitter considers it has a complete
+        interactive block and will not accept more input only when either a
+        SyntaxError is raised, or *all* of the following are true:
 
         1. The input compiles to a complete statement.
         
@@ -218,40 +261,147 b' class BlockBreaker(object):'
 
         Block-oriented frontends that have a separate keyboard event to
         indicate execution should use the :meth:`split_blocks` method instead.
+
+        If the current input produces a syntax error, this method immediately
+        returns False but does *not* raise the syntax error exception, as
+        typically clients will want to send invalid syntax to an execution
+        backend which might convert the invalid syntax into valid Python via
+        one of the dynamic IPython mechanisms.
         """
-        if not self.is_complete:
-            return False
-        if self.indent_spaces==0:
-            return True
-        last_line = self.source.splitlines()[-1]
-        if not last_line or last_line.isspace():
+            
+        if not self._is_complete:
             return True
-        else:
-            return False
 
+        if self.indent_spaces==0:
+            return False
+        
+        last_line = self.source.splitlines()[-1]
+        return bool(last_line and not last_line.isspace())
+        
     def split_blocks(self, lines):
-        """Split a multiline string into multiple input blocks"""
-        raise NotImplementedError
+        """Split a multiline string into multiple input blocks.
+
+        Note: this method starts by performing a full reset().
+        
+        Parameters
+        ----------
+        lines : str
+          A possibly multiline string.
+
+        Returns
+        -------
+        blocks : list
+          A list of strings, each possibly multiline.  Each string corresponds
+          to a single block that can be compiled in 'single' mode (unless it
+          has a syntax error)."""
+
+        # This code is fairly delicate.  If you make any changes here, make
+        # absolutely sure that you do run the full test suite and ALL tests
+        # pass.
+
+        self.reset()
+        blocks = []
+        
+        # Reversed copy so we can use pop() efficiently and consume the input
+        # as a stack
+        lines = lines.splitlines()[::-1]
+        # Outer loop over all input
+        while lines:
+            # Inner loop to build each block
+            while True:
+                # Safety exit from inner loop
+                if not lines:
+                    break
+                # Grab next line but don't push it yet
+                next_line = lines.pop()
+                # Blank/empty lines are pushed as-is
+                if not next_line or next_line.isspace():
+                    self.push(next_line)
+                    continue
+
+                # Check indentation changes caused by the *next* line
+                indent_spaces, _full_dedent = self._find_indent(next_line)
+
+                # If the next line causes a dedent, it can be for two differnt
+                # reasons: either an explicit de-dent by the user or a
+                # return/raise/pass statement.  These MUST be handled
+                # separately:
+                #
+                # 1. the first case is only detected when the actual explicit
+                # dedent happens, and that would be the *first* line of a *new*
+                # block.  Thus, we must put the line back into the input buffer
+                # so that it starts a new block on the next pass.
+                #
+                # 2. the second case is detected in the line before the actual
+                # dedent happens, so , we consume the line and we can break out
+                # to start a new block.
+
+                # Case 1, explicit dedent causes a break
+                if _full_dedent and not next_line.startswith(' '):
+                    lines.append(next_line)
+                    break
+                
+                # Otherwise any line is pushed
+                self.push(next_line)
+
+                # Case 2, full dedent with full block ready:
+                if _full_dedent or \
+                       self.indent_spaces==0 and not self.push_accepts_more():
+                    break
+            # Form the new block with the current source input
+            blocks.append(self.source_reset())
+            
+        return blocks
 
     #------------------------------------------------------------------------
     # Private interface
     #------------------------------------------------------------------------
-    
-    def _update_indent(self, lines):
-        """Keep track of the indent level."""
 
-        for line in remove_comments(lines).splitlines():
+    def _find_indent(self, line):
+        """Compute the new indentation level for a single line.
+
+        Parameters
+        ----------
+        line : str
+          A single new line of non-whitespace, non-comment Python input.
+          
+        Returns
+        -------
+        indent_spaces : int
+          New value for the indent level (it may be equal to self.indent_spaces
+        if indentation doesn't change.
+
+        full_dedent : boolean
+          Whether the new line causes a full flush-left dedent.
+        """
+        indent_spaces = self.indent_spaces
+        full_dedent = self._full_dedent
+        
+        inisp = num_ini_spaces(line)
+        if inisp < indent_spaces:
+            indent_spaces = inisp
+            if indent_spaces <= 0:
+                #print 'Full dedent in text',self.source # dbg
+                full_dedent = True
+
+        if line[-1] == ':':
+            indent_spaces += 4
+        elif dedent_re.match(line):
+            indent_spaces -= 4
+            if indent_spaces <= 0:
+                full_dedent = True
+
+        # Safety
+        if indent_spaces < 0:
+            indent_spaces = 0
+            #print 'safety' # dbg
             
+        return indent_spaces, full_dedent
+        
+    def _update_indent(self, lines):
+        for line in remove_comments(lines).splitlines():
             if line and not line.isspace():
-                if self.code is not None:
-                    inisp = num_ini_spaces(line)
-                    if inisp < self.indent_spaces:
-                        self.indent_spaces = inisp
-
-                if line[-1] == ':':
-                    self.indent_spaces += 4
-                elif dedent_re.match(line):
-                    self.indent_spaces -= 4
+                self.indent_spaces, self._full_dedent = self._find_indent(line)
 
     def _store(self, lines):
         """Store one or more lines of input.
@@ -263,4 +413,7 b' class BlockBreaker(object):'
             self._buffer.append(lines)
         else:
             self._buffer.append(lines+'\n')
+        self._set_source()
+
+    def _set_source(self):
         self.source = ''.join(self._buffer).encode(self.encoding)