Simplify InputSplitter by stripping out input_mode distinction

IPython/core/inputsplitter.py

@@ -1,721 +1,661 b''
 """Analysis of text input into executable blocks.
 
 The main class in this module, :class:`InputSplitter`, is designed to break
 input from either interactive, line-by-line environments or block-based ones,
 into standalone blocks that can be executed by Python as 'single' statements
 (thus triggering sys.displayhook).
 
 A companion, :class:`IPythonInputSplitter`, provides the same functionality but
 with full support for the extended IPython syntax (magics, system calls, etc).
 
 For more details, see the class docstring below.
 
 Syntax Transformations
 ----------------------
 
 One of the main jobs of the code in this file is to apply all syntax
 transformations that make up 'the IPython language', i.e. magics, shell
 escapes, etc.  All transformations should be implemented as *fully stateless*
 entities, that simply take one line as their input and return a line.
 Internally for implementation purposes they may be a normal function or a
 callable object, but the only input they receive will be a single line and they
 should only return a line, without holding any data-dependent state between
 calls.
 
 As an example, the EscapedTransformer is a class so we can more clearly group
 together the functionality of dispatching to individual functions based on the
 starting escape character, but the only method for public use is its call
 method.
 
 
 ToDo
 ----
 
 - Should we make push() actually raise an exception once push_accepts_more()
   returns False?
 
 - Naming cleanups.  The tr_* names aren't the most elegant, though now they are
   at least just attributes of a class so not really very exposed.
 
 - Think about the best way to support dynamic things: automagic, autocall,
   macros, etc.
 
 - Think of a better heuristic for the application of the transforms in
   IPythonInputSplitter.push() than looking at the buffer ending in ':'.  Idea:
   track indentation change events (indent, dedent, nothing) and apply them only
   if the indentation went up, but not otherwise.
 
 - Think of the cleanest way for supporting user-specified transformations (the
   user prefilters we had before).
 
 Authors
 -------
 
 * Fernando Perez
 * Brian Granger
 """
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2010  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 # stdlib
 import ast
 import codeop
 import re
 import sys
 
 # IPython modules
 from IPython.core.splitinput import split_user_input, LineInfo
 from IPython.utils.py3compat import cast_unicode
 from IPython.core.inputtransformer import (leading_indent,
                                            classic_prompt,
                                            ipy_prompt,
                                            cellmagic,
                                            assemble_logical_lines,
                                            help_end,
                                            escaped_commands,
                                            assign_from_magic,
                                            assign_from_system,
                                            assemble_python_lines,
                                            )
 
 # Temporary!
 from IPython.core.inputtransformer import (ESC_SHELL, ESC_SH_CAP, ESC_HELP,
                                         ESC_HELP2, ESC_MAGIC, ESC_MAGIC2,
                                         ESC_QUOTE, ESC_QUOTE2, ESC_PAREN, ESC_SEQUENCES)
 
 #-----------------------------------------------------------------------------
 # Utilities
 #-----------------------------------------------------------------------------
 
 # FIXME: These are general-purpose utilities that later can be moved to the
 # general ward.  Kept here for now because we're being very strict about test
 # coverage with this code, and this lets us ensure that we keep 100% coverage
 # while developing.
 
 # compiled regexps for autoindent management
 dedent_re = re.compile('|'.join([
     r'^\s+raise(\s.*)?$', # raise statement (+ space + other stuff, maybe)
     r'^\s+raise\([^\)]*\).*$', # wacky raise with immediate open paren
     r'^\s+return(\s.*)?$', # normal return (+ space + other stuff, maybe)
     r'^\s+return\([^\)]*\).*$', # wacky return with immediate open paren
     r'^\s+pass\s*$', # pass (optionally followed by trailing spaces)
     r'^\s+break\s*$', # break (optionally followed by trailing spaces)
     r'^\s+continue\s*$', # continue (optionally followed by trailing spaces)
 ]))
 ini_spaces_re = re.compile(r'^([ \t\r\f\v]+)')
 
 # regexp to match pure comment lines so we don't accidentally insert 'if 1:'
 # before pure comments
 comment_line_re = re.compile('^\s*\#')
 
 
 def num_ini_spaces(s):
     """Return the number of initial spaces in a string.
 
     Note that tabs are counted as a single space.  For now, we do *not* support
     mixing of tabs and spaces in the user's input.
 
     Parameters
     ----------
     s : string
 
     Returns
     -------
     n : int
     """
 
     ini_spaces = ini_spaces_re.match(s)
     if ini_spaces:
         return ini_spaces.end()
     else:
         return 0
 
 def last_blank(src):
     """Determine if the input source ends in a blank.
 
     A blank is either a newline or a line consisting of whitespace.
 
     Parameters
     ----------
     src : string
       A single or multiline string.
     """
     if not src: return False
     ll  = src.splitlines()[-1]
     return (ll == '') or ll.isspace()
 
 
 last_two_blanks_re = re.compile(r'\n\s*\n\s*$', re.MULTILINE)
 last_two_blanks_re2 = re.compile(r'.+\n\s*\n\s+$', re.MULTILINE)
 
 def last_two_blanks(src):
     """Determine if the input source ends in two blanks.
 
     A blank is either a newline or a line consisting of whitespace.
 
     Parameters
     ----------
     src : string
       A single or multiline string.
     """
     if not src: return False
     # The logic here is tricky: I couldn't get a regexp to work and pass all
     # the tests, so I took a different approach: split the source by lines,
     # grab the last two and prepend '###\n' as a stand-in for whatever was in
     # the body before the last two lines.  Then, with that structure, it's
     # possible to analyze with two regexps.  Not the most elegant solution, but
     # it works.  If anyone tries to change this logic, make sure to validate
     # the whole test suite first!
     new_src = '\n'.join(['###\n'] + src.splitlines()[-2:])
     return (bool(last_two_blanks_re.match(new_src)) or
             bool(last_two_blanks_re2.match(new_src)) )
 
 
 def remove_comments(src):
     """Remove all comments from input source.
 
     Note: comments are NOT recognized inside of strings!
 
     Parameters
     ----------
     src : string
       A single or multiline input string.
 
     Returns
     -------
     String with all Python comments removed.
     """
 
     return re.sub('#.*', '', src)
 
 
 def get_input_encoding():
     """Return the default standard input encoding.
 
     If sys.stdin has no encoding, 'ascii' is returned."""
     # There are strange environments for which sys.stdin.encoding is None. We
     # ensure that a valid encoding is returned.
     encoding = getattr(sys.stdin, 'encoding', None)
     if encoding is None:
         encoding = 'ascii'
     return encoding
 
 #-----------------------------------------------------------------------------
 # Classes and functions for normal Python syntax handling
 #-----------------------------------------------------------------------------
 
 class InputSplitter(object):
     """An object that can accumulate lines of Python source before execution.
 
     This object is designed to be fed python source line-by-line, using
        :meth:`push`. It will return on each push whether the currently pushed
        code could be executed already. In addition, it provides a method called
        :meth:`push_accepts_more` that can be used to query whether more input
        can be pushed into a single interactive block.
 
     This is a simple example of how an interactive terminal-based client can use
     this tool::
 
         isp = InputSplitter()
         while isp.push_accepts_more():
             indent = ' '*isp.indent_spaces
             prompt = '>>> ' + indent
             line = indent + raw_input(prompt)
             isp.push(line)
         print 'Input source was:\n', isp.source_reset(),
     """
     # Number of spaces of indentation computed from input that has been pushed
     # so far.  This is the attributes callers should query to get the current
     # indentation level, in order to provide auto-indent facilities.
     indent_spaces = 0
     # String, indicating the default input encoding.  It is computed by default
     # at initialization time via get_input_encoding(), but it can be reset by a
     # client with specific knowledge of the encoding.
     encoding = ''
     # String where the current full source input is stored, properly encoded.
     # Reading this attribute is the normal way of querying the currently pushed
     # source code, that has been properly encoded.
     source = ''
     # Code object corresponding to the current source.  It is automatically
     # synced to the source, so it can be queried at any time to obtain the code
     # object; it will be None if the source doesn't compile to valid Python.
     code = None
-    # Input mode
-    input_mode = 'line'
 
     # Private attributes
 
     # List with lines of input accumulated so far
     _buffer = None
     # Command compiler
     _compile = None
     # Mark when input has changed indentation all the way back to flush-left
     _full_dedent = False
     # Boolean indicating whether the current block is complete
     _is_complete = None
 
-    def __init__(self, input_mode=None):
+    def __init__(self):
         """Create a new InputSplitter instance.
-
-        Parameters
-        ----------
-        input_mode : str
-
-          One of ['line', 'cell']; default is 'line'.
-
-       The input_mode parameter controls how new inputs are used when fed via
-       the :meth:`push` method:
-
-       - 'line': meant for line-oriented clients, inputs are appended one at a
-         time to the internal buffer and the whole buffer is compiled.
-
-       - 'cell': meant for clients that can edit multi-line 'cells' of text at
-          a time.  A cell can contain one or more blocks that can be compile in
-          'single' mode by Python.  In this mode, each new input new input
-          completely replaces all prior inputs.  Cell mode is thus equivalent
-          to prepending a full reset() to every push() call.
         """
         self._buffer = []
         self._compile = codeop.CommandCompiler()
         self.encoding = get_input_encoding()
-        self.input_mode = InputSplitter.input_mode if input_mode is None \
-                          else input_mode
 
     def reset(self):
         """Reset the input buffer and associated state."""
         self.indent_spaces = 0
         self._buffer[:] = []
         self.source = ''
         self.code = None
         self._is_complete = False
         self._full_dedent = False
 
     def source_reset(self):
         """Return the input source and perform a full reset.
         """
         out = self.source
         self.reset()
         return out
 
     def push(self, lines):
         """Push one or more lines of input.
 
         This stores the given lines and returns a status code indicating
         whether the code forms a complete Python block or not.
 
         Any exceptions generated in compilation are swallowed, but if an
         exception was produced, the method returns True.
 
         Parameters
         ----------
         lines : string
           One or more lines of Python input.
 
         Returns
         -------
         is_complete : boolean
           True if the current input source (the result of the current input
           plus prior inputs) forms a complete Python execution block.  Note that
           this value is also stored as a private attribute (``_is_complete``), so it
           can be queried at any time.
         """
-        if self.input_mode == 'cell':
-            self.reset()
-
         self._store(lines)
         source = self.source
 
         # Before calling _compile(), reset the code object to None so that if an
         # exception is raised in compilation, we don't mislead by having
         # inconsistent code/source attributes.
         self.code, self._is_complete = None, None
 
         # Honor termination lines properly
         if source.endswith('\\\n'):
             return False
 
         self._update_indent(lines)
         try:
             self.code = self._compile(source, symbol="exec")
         # Invalid syntax can produce any of a number of different errors from
         # inside the compiler, so we have to catch them all.  Syntax errors
         # immediately produce a 'ready' block, so the invalid Python can be
         # sent to the kernel for evaluation with possible ipython
         # special-syntax conversion.
         except (SyntaxError, OverflowError, ValueError, TypeError,
                 MemoryError):
             self._is_complete = True
         else:
             # Compilation didn't produce any exceptions (though it may not have
             # given a complete code object)
             self._is_complete = self.code is not None
 
         return self._is_complete
 
     def push_accepts_more(self):
         """Return whether a block of interactive input can accept more input.
 
         This method is meant to be used by line-oriented frontends, who need to
         guess whether a block is complete or not based solely on prior and
         current input lines.  The InputSplitter considers it has a complete
         interactive block and will not accept more input only when either a
         SyntaxError is raised, or *all* of the following are true:
 
         1. The input compiles to a complete statement.
 
         2. The indentation level is flush-left (because if we are indented,
            like inside a function definition or for loop, we need to keep
            reading new input).
 
         3. There is one extra line consisting only of whitespace.
 
         Because of condition #3, this method should be used only by
         *line-oriented* frontends, since it means that intermediate blank lines
         are not allowed in function definitions (or any other indented block).
 
         If the current input produces a syntax error, this method immediately
         returns False but does *not* raise the syntax error exception, as
         typically clients will want to send invalid syntax to an execution
         backend which might convert the invalid syntax into valid Python via
         one of the dynamic IPython mechanisms.
         """
 
         # With incomplete input, unconditionally accept more
+        # A syntax error also sets _is_complete to True - see push()
         if not self._is_complete:
+            #print("Not complete")  # debug
             return True
-
-        # If we already have complete input and we're flush left, the answer
-        # depends.  In line mode, if there hasn't been any indentation,
-        # that's it. If we've come back from some indentation, we need
-        # the blank final line to finish.
-        # In cell mode, we need to check how many blocks the input so far
-        # compiles into, because if there's already more than one full
-        # independent block of input, then the client has entered full
-        # 'cell' mode and is feeding lines that each is complete.  In this
-        # case we should then keep accepting. The Qt terminal-like console
-        # does precisely this, to provide the convenience of terminal-like
-        # input of single expressions, but allowing the user (with a
-        # separate keystroke) to switch to 'cell' mode and type multiple
-        # expressions in one shot.
+        
+        # The user can make any (complete) input execute by leaving a blank line
+        last_line = self.source.splitlines()[-1]
+        if (not last_line) or last_line.isspace():
+            #print("Blank line")  # debug
+            return False
+        
+        # If there's just a single AST node, and we're flush left, as is the 
+        # case after a simple statement such as 'a=1', we want to execute it
+        # straight away.
         if self.indent_spaces==0:
-            if self.input_mode=='line':
-                if not self._full_dedent:
-                    return False
+            try:
+                code_ast = ast.parse(u''.join(self._buffer))
+            except Exception:
+                #print("Can't parse AST")  # debug
+                return False
             else:
-                try:
-                    code_ast = ast.parse(u''.join(self._buffer))
-                except Exception:
+                if len(code_ast.body) == 1 and \
+                                    not hasattr(code_ast.body[0], 'body'):
+                    #print("Simple statement")  # debug
                     return False
-                else:
-                    if len(code_ast.body) == 1:
-                        return False
 
-        # When input is complete, then termination is marked by an extra blank
-        # line at the end.
-        last_line = self.source.splitlines()[-1]
-        return bool(last_line and not last_line.isspace())
+        # General fallback - accept more code
+        return True
 
     #------------------------------------------------------------------------
     # Private interface
     #------------------------------------------------------------------------
 
     def _find_indent(self, line):
         """Compute the new indentation level for a single line.
 
         Parameters
         ----------
         line : str
           A single new line of non-whitespace, non-comment Python input.
 
         Returns
         -------
         indent_spaces : int
           New value for the indent level (it may be equal to self.indent_spaces
         if indentation doesn't change.
 
         full_dedent : boolean
           Whether the new line causes a full flush-left dedent.
         """
         indent_spaces = self.indent_spaces
         full_dedent = self._full_dedent
 
         inisp = num_ini_spaces(line)
         if inisp < indent_spaces:
             indent_spaces = inisp
             if indent_spaces <= 0:
                 #print 'Full dedent in text',self.source # dbg
                 full_dedent = True
 
         if line.rstrip()[-1] == ':':
             indent_spaces += 4
         elif dedent_re.match(line):
             indent_spaces -= 4
             if indent_spaces <= 0:
                 full_dedent = True
 
         # Safety
         if indent_spaces < 0:
             indent_spaces = 0
             #print 'safety' # dbg
 
         return indent_spaces, full_dedent
 
     def _update_indent(self, lines):
         for line in remove_comments(lines).splitlines():
             if line and not line.isspace():
                 self.indent_spaces, self._full_dedent = self._find_indent(line)
 
     def _store(self, lines, buffer=None, store='source'):
         """Store one or more lines of input.
 
         If input lines are not newline-terminated, a newline is automatically
         appended."""
 
         if buffer is None:
             buffer = self._buffer
 
         if lines.endswith('\n'):
             buffer.append(lines)
         else:
             buffer.append(lines+'\n')
         setattr(self, store, self._set_source(buffer))
 
     def _set_source(self, buffer):
         return u''.join(buffer)
 
 
 class IPythonInputSplitter(InputSplitter):
     """An input splitter that recognizes all of IPython's special syntax."""
 
     # String with raw, untransformed input.
     source_raw = ''
     
     # Flag to track when a transformer has stored input that it hasn't given
     # back yet.
     transformer_accumulating = False
     
     # Flag to track when assemble_python_lines has stored input that it hasn't
     # given back yet.
     within_python_line = False
 
     # Private attributes
 
     # List with lines of raw input accumulated so far.
     _buffer_raw = None
 
-    def __init__(self, input_mode=None, physical_line_transforms=None,
+    def __init__(self, physical_line_transforms=None,
                     logical_line_transforms=None, python_line_transforms=None):
-        super(IPythonInputSplitter, self).__init__(input_mode)
+        super(IPythonInputSplitter, self).__init__()
         self._buffer_raw = []
         self._validate = True
         
         if physical_line_transforms is not None:
             self.physical_line_transforms = physical_line_transforms
         else:
             self.physical_line_transforms = [leading_indent(),
                                              classic_prompt(),
                                              ipy_prompt(),
                                             ]
         
         self.assemble_logical_lines = assemble_logical_lines()
         if logical_line_transforms is not None:
             self.logical_line_transforms = logical_line_transforms
         else:
             self.logical_line_transforms = [cellmagic(),
                                             help_end(),
                                             escaped_commands(),
                                             assign_from_magic(),
                                             assign_from_system(),
                                            ]
         
         self.assemble_python_lines = assemble_python_lines()
         if python_line_transforms is not None:
             self.python_line_transforms = python_line_transforms
         else:
             # We don't use any of these at present
             self.python_line_transforms = []
     
     @property
     def transforms(self):
         "Quick access to all transformers."
         return self.physical_line_transforms + \
             [self.assemble_logical_lines] + self.logical_line_transforms + \
             [self.assemble_python_lines]  + self.python_line_transforms
     
     @property
     def transforms_in_use(self):
         """Transformers, excluding logical line transformers if we're in a
         Python line."""
         t = self.physical_line_transforms[:]
         if not self.within_python_line:
             t += [self.assemble_logical_lines] + self.logical_line_transforms
         return t + [self.assemble_python_lines] + self.python_line_transforms
 
     def reset(self):
         """Reset the input buffer and associated state."""
         super(IPythonInputSplitter, self).reset()
         self._buffer_raw[:] = []
         self.source_raw = ''
         self.transformer_accumulating = False
         self.within_python_line = False
         for t in self.transforms:
             t.reset()
     
     def flush_transformers(self):
         def _flush(transform, out):
             if out is not None:
                 tmp = transform.push(out)
                 return tmp or transform.reset() or None
             else:
                 return transform.reset() or None
         
         out = None
         for t in self.transforms_in_use:
             out = _flush(t, out)
         
         if out is not None:
             self._store(out)
 
     def source_raw_reset(self):
         """Return input and raw source and perform a full reset.
         """
         self.flush_transformers()
         out = self.source
         out_r = self.source_raw
         self.reset()
         return out, out_r
     
     def source_reset(self):
         self.flush_transformers()
         return super(IPythonInputSplitter, self).source_reset()
 
     def push_accepts_more(self):
         if self.transformer_accumulating:
             return True
         else:
             return super(IPythonInputSplitter, self).push_accepts_more()
 
     def transform_cell(self, cell):
         """Process and translate a cell of input.
         """
         self.reset()
         self.push(cell)
         return self.source_reset()
 
     def push(self, lines):
         """Push one or more lines of IPython input.
 
         This stores the given lines and returns a status code indicating
         whether the code forms a complete Python block or not, after processing
         all input lines for special IPython syntax.
 
         Any exceptions generated in compilation are swallowed, but if an
         exception was produced, the method returns True.
 
         Parameters
         ----------
         lines : string
           One or more lines of Python input.
 
         Returns
         -------
         is_complete : boolean
           True if the current input source (the result of the current input
         plus prior inputs) forms a complete Python execution block.  Note that
         this value is also stored as a private attribute (_is_complete), so it
         can be queried at any time.
         """
 
         # We must ensure all input is pure unicode
         lines = cast_unicode(lines, self.encoding)
         
         # ''.splitlines() --> [], but we need to push the empty line to transformers
         lines_list = lines.splitlines()
         if not lines_list:
             lines_list = ['']
 
-        # Transform logic
-        #
-        # We only apply the line transformers to the input if we have either no
-        # input yet, or complete input, or if the last line of the buffer ends
-        # with ':' (opening an indented block).  This prevents the accidental
-        # transformation of escapes inside multiline expressions like
-        # triple-quoted strings or parenthesized expressions.
-        #
-        # The last heuristic, while ugly, ensures that the first line of an
-        # indented block is correctly transformed.
-        #
-        # FIXME: try to find a cleaner approach for this last bit.
-
-        # If we were in 'block' mode, since we're going to pump the parent
-        # class by hand line by line, we need to temporarily switch out to
-        # 'line' mode, do a single manual reset and then feed the lines one
-        # by one.  Note that this only matters if the input has more than one
-        # line.
-        changed_input_mode = False
-
-        if self.input_mode == 'cell':
-            self.reset()
-            changed_input_mode = True
-            saved_input_mode = 'cell'
-            self.input_mode = 'line'
-
         # Store raw source before applying any transformations to it.  Note
         # that this must be done *after* the reset() call that would otherwise
         # flush the buffer.
         self._store(lines, self._buffer_raw, 'source_raw')
 
-        try:
-            for line in lines_list:
-                out = self.push_line(line)
-        finally:
-            if changed_input_mode:
-                self.input_mode = saved_input_mode
-        
+        for line in lines_list:
+            out = self.push_line(line)
+
         return out
     
     def push_line(self, line):
         buf = self._buffer
         
         def _accumulating(dbg):
             #print(dbg)
             self.transformer_accumulating = True
             return False
         
         for transformer in self.physical_line_transforms:
             line = transformer.push(line)
             if line is None:
                 return _accumulating(transformer)
         
         if not self.within_python_line:
             line = self.assemble_logical_lines.push(line)
             if line is None:
                 return _accumulating('acc logical line')        
         
             for transformer in self.logical_line_transforms:
                 line = transformer.push(line)
                 if line is None:
                     return _accumulating(transformer)
         
         line = self.assemble_python_lines.push(line)
         if line is None:
             self.within_python_line = True
             return _accumulating('acc python line')
         else:
             self.within_python_line = False
         
         for transformer in self.python_line_transforms:
             line = transformer.push(line)
             if line is None:
                 return _accumulating(transformer)
 
         #print("transformers clear") #debug
         self.transformer_accumulating = False
         return super(IPythonInputSplitter, self).push(line)

IPython/core/tests/test_inputsplitter.py

@@ -1,626 +1,576 b''
 # -*- coding: utf-8 -*-
 """Tests for the inputsplitter module.
 
 Authors
 -------
 * Fernando Perez
 * Robert Kern
 """
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2010-2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 # stdlib
 import unittest
 import sys
 
 # Third party
 import nose.tools as nt
 
 # Our own
 from IPython.core import inputsplitter as isp
 from IPython.core.tests.test_inputtransformer import syntax, syntax_ml
 from IPython.testing import tools as tt
 from IPython.utils import py3compat
 
 #-----------------------------------------------------------------------------
 # Semi-complete examples (also used as tests)
 #-----------------------------------------------------------------------------
 
 # Note: at the bottom, there's a slightly more complete version of this that
 # can be useful during development of code here.
 
 def mini_interactive_loop(input_func):
     """Minimal example of the logic of an interactive interpreter loop.
 
     This serves as an example, and it is used by the test system with a fake
     raw_input that simulates interactive input."""
 
     from IPython.core.inputsplitter import InputSplitter
 
     isp = InputSplitter()
     # In practice, this input loop would be wrapped in an outside loop to read
     # input indefinitely, until some exit/quit command was issued.  Here we
     # only illustrate the basic inner loop.
     while isp.push_accepts_more():
         indent = ' '*isp.indent_spaces
         prompt = '>>> ' + indent
         line = indent + input_func(prompt)
         isp.push(line)
 
     # Here we just return input so we can use it in a test suite, but a real
     # interpreter would instead send it for execution somewhere.
     src = isp.source_reset()
     #print 'Input source was:\n', src  # dbg
     return src
 
 #-----------------------------------------------------------------------------
 # Test utilities, just for local use
 #-----------------------------------------------------------------------------
 
 def assemble(block):
     """Assemble a block into multi-line sub-blocks."""
     return ['\n'.join(sub_block)+'\n' for sub_block in block]
 
 
 def pseudo_input(lines):
     """Return a function that acts like raw_input but feeds the input list."""
     ilines = iter(lines)
     def raw_in(prompt):
         try:
             return next(ilines)
         except StopIteration:
             return ''
     return raw_in
 
 #-----------------------------------------------------------------------------
 # Tests
 #-----------------------------------------------------------------------------
 def test_spaces():
     tests = [('', 0),
              (' ', 1),
              ('\n', 0),
              (' \n', 1),
              ('x', 0),
              (' x', 1),
              ('  x',2),
              ('    x',4),
              # Note: tabs are counted as a single whitespace!
              ('\tx', 1),
              ('\t x', 2),
              ]
     tt.check_pairs(isp.num_ini_spaces, tests)
 
 
 def test_remove_comments():
     tests = [('text', 'text'),
              ('text # comment', 'text '),
              ('text # comment\n', 'text \n'),
              ('text # comment \n', 'text \n'),
              ('line # c \nline\n','line \nline\n'),
              ('line # c \nline#c2  \nline\nline #c\n\n',
               'line \nline\nline\nline \n\n'),
              ]
     tt.check_pairs(isp.remove_comments, tests)
 
 
 def test_get_input_encoding():
     encoding = isp.get_input_encoding()
     nt.assert_true(isinstance(encoding, basestring))
     # simple-minded check that at least encoding a simple string works with the
     # encoding we got.
     nt.assert_equal(u'test'.encode(encoding), b'test')
 
 
 class NoInputEncodingTestCase(unittest.TestCase):
     def setUp(self):
         self.old_stdin = sys.stdin
         class X: pass
         fake_stdin = X()
         sys.stdin = fake_stdin
 
     def test(self):
         # Verify that if sys.stdin has no 'encoding' attribute we do the right
         # thing
         enc = isp.get_input_encoding()
         self.assertEqual(enc, 'ascii')
 
     def tearDown(self):
         sys.stdin = self.old_stdin
 
 
 class InputSplitterTestCase(unittest.TestCase):
     def setUp(self):
         self.isp = isp.InputSplitter()
 
     def test_reset(self):
         isp = self.isp
         isp.push('x=1')
         isp.reset()
         self.assertEqual(isp._buffer, [])
         self.assertEqual(isp.indent_spaces, 0)
         self.assertEqual(isp.source, '')
         self.assertEqual(isp.code, None)
         self.assertEqual(isp._is_complete, False)
 
     def test_source(self):
         self.isp._store('1')
         self.isp._store('2')
         self.assertEqual(self.isp.source, '1\n2\n')
         self.assertTrue(len(self.isp._buffer)>0)
         self.assertEqual(self.isp.source_reset(), '1\n2\n')
         self.assertEqual(self.isp._buffer, [])
         self.assertEqual(self.isp.source, '')
 
     def test_indent(self):
         isp = self.isp # shorthand
         isp.push('x=1')
         self.assertEqual(isp.indent_spaces, 0)
         isp.push('if 1:\n    x=1')
         self.assertEqual(isp.indent_spaces, 4)
         isp.push('y=2\n')
         self.assertEqual(isp.indent_spaces, 0)
 
     def test_indent2(self):
-        # In cell mode, inputs must be fed in whole blocks, so skip this test
-        if self.isp.input_mode == 'cell': return
-
         isp = self.isp
         isp.push('if 1:')
         self.assertEqual(isp.indent_spaces, 4)
         isp.push('    x=1')
         self.assertEqual(isp.indent_spaces, 4)
         # Blank lines shouldn't change the indent level
         isp.push(' '*2)
         self.assertEqual(isp.indent_spaces, 4)
 
     def test_indent3(self):
-        # In cell mode, inputs must be fed in whole blocks, so skip this test
-        if self.isp.input_mode == 'cell': return
-
         isp = self.isp
         # When a multiline statement contains parens or multiline strings, we
         # shouldn't get confused.
         isp.push("if 1:")
         isp.push("    x = (1+\n    2)")
         self.assertEqual(isp.indent_spaces, 4)
 
     def test_indent4(self):
-        # In cell mode, inputs must be fed in whole blocks, so skip this test
-        if self.isp.input_mode == 'cell': return
-
         isp = self.isp
         # whitespace after ':' should not screw up indent level
         isp.push('if 1: \n    x=1')
         self.assertEqual(isp.indent_spaces, 4)
         isp.push('y=2\n')
         self.assertEqual(isp.indent_spaces, 0)
         isp.push('if 1:\t\n    x=1')
         self.assertEqual(isp.indent_spaces, 4)
         isp.push('y=2\n')
         self.assertEqual(isp.indent_spaces, 0)
 
     def test_dedent_pass(self):
         isp = self.isp # shorthand
         # should NOT cause dedent
         isp.push('if 1:\n    passes = 5')
         self.assertEqual(isp.indent_spaces, 4)
         isp.push('if 1:\n     pass')
         self.assertEqual(isp.indent_spaces, 0)
         isp.push('if 1:\n     pass   ')
         self.assertEqual(isp.indent_spaces, 0)
 
     def test_dedent_break(self):
         isp = self.isp # shorthand
         # should NOT cause dedent
         isp.push('while 1:\n    breaks = 5')
         self.assertEqual(isp.indent_spaces, 4)
         isp.push('while 1:\n     break')
         self.assertEqual(isp.indent_spaces, 0)
         isp.push('while 1:\n     break   ')
         self.assertEqual(isp.indent_spaces, 0)
 
     def test_dedent_continue(self):
         isp = self.isp # shorthand
         # should NOT cause dedent
         isp.push('while 1:\n    continues = 5')
         self.assertEqual(isp.indent_spaces, 4)
         isp.push('while 1:\n     continue')
         self.assertEqual(isp.indent_spaces, 0)
         isp.push('while 1:\n     continue   ')
         self.assertEqual(isp.indent_spaces, 0)
 
     def test_dedent_raise(self):
         isp = self.isp # shorthand
         # should NOT cause dedent
         isp.push('if 1:\n    raised = 4')
         self.assertEqual(isp.indent_spaces, 4)
         isp.push('if 1:\n     raise TypeError()')
         self.assertEqual(isp.indent_spaces, 0)
         isp.push('if 1:\n     raise')
         self.assertEqual(isp.indent_spaces, 0)
         isp.push('if 1:\n     raise      ')
         self.assertEqual(isp.indent_spaces, 0)
 
     def test_dedent_return(self):
         isp = self.isp # shorthand
         # should NOT cause dedent
         isp.push('if 1:\n    returning = 4')
         self.assertEqual(isp.indent_spaces, 4)
         isp.push('if 1:\n     return 5 + 493')
         self.assertEqual(isp.indent_spaces, 0)
         isp.push('if 1:\n     return')
         self.assertEqual(isp.indent_spaces, 0)
         isp.push('if 1:\n     return      ')
         self.assertEqual(isp.indent_spaces, 0)
         isp.push('if 1:\n     return(0)')
         self.assertEqual(isp.indent_spaces, 0)
 
     def test_push(self):
         isp = self.isp
         self.assertTrue(isp.push('x=1'))
 
     def test_push2(self):
         isp = self.isp
         self.assertFalse(isp.push('if 1:'))
         for line in ['  x=1', '# a comment', '  y=2']:
             print(line)
             self.assertTrue(isp.push(line))
 
     def test_push3(self):
         isp = self.isp
         isp.push('if True:')
         isp.push('  a = 1')
         self.assertFalse(isp.push('b = [1,'))
 
-    def test_replace_mode(self):
-        isp = self.isp
-        isp.input_mode = 'cell'
-        isp.push('x=1')
-        self.assertEqual(isp.source, 'x=1\n')
-        isp.push('x=2')
-        self.assertEqual(isp.source, 'x=2\n')
-
     def test_push_accepts_more(self):
         isp = self.isp
         isp.push('x=1')
         self.assertFalse(isp.push_accepts_more())
 
     def test_push_accepts_more2(self):
-        # In cell mode, inputs must be fed in whole blocks, so skip this test
-        if self.isp.input_mode == 'cell': return
-
         isp = self.isp
         isp.push('if 1:')
         self.assertTrue(isp.push_accepts_more())
         isp.push('  x=1')
         self.assertTrue(isp.push_accepts_more())
         isp.push('')
         self.assertFalse(isp.push_accepts_more())
 
     def test_push_accepts_more3(self):
         isp = self.isp
         isp.push("x = (2+\n3)")
         self.assertFalse(isp.push_accepts_more())
 
     def test_push_accepts_more4(self):
-        # In cell mode, inputs must be fed in whole blocks, so skip this test
-        if self.isp.input_mode == 'cell': return
-
         isp = self.isp
         # When a multiline statement contains parens or multiline strings, we
         # shouldn't get confused.
         # FIXME: we should be able to better handle de-dents in statements like
         # multiline strings and multiline expressions (continued with \ or
         # parens).  Right now we aren't handling the indentation tracking quite
         # correctly with this, though in practice it may not be too much of a
         # problem.  We'll need to see.
         isp.push("if 1:")
         isp.push("    x = (2+")
         isp.push("    3)")
         self.assertTrue(isp.push_accepts_more())
         isp.push("    y = 3")
         self.assertTrue(isp.push_accepts_more())
         isp.push('')
         self.assertFalse(isp.push_accepts_more())
 
     def test_push_accepts_more5(self):
-        # In cell mode, inputs must be fed in whole blocks, so skip this test
-        if self.isp.input_mode == 'cell': return
-
         isp = self.isp
         isp.push('try:')
         isp.push('    a = 5')
         isp.push('except:')
         isp.push('    raise')
+        # We want to be able to add an else: block at this point, so it should
+        # wait for a blank line.
         self.assertTrue(isp.push_accepts_more())
 
     def test_continuation(self):
         isp = self.isp
         isp.push("import os, \\")
         self.assertTrue(isp.push_accepts_more())
         isp.push("sys")
         self.assertFalse(isp.push_accepts_more())
 
     def test_syntax_error(self):
         isp = self.isp
         # Syntax errors immediately produce a 'ready' block, so the invalid
         # Python can be sent to the kernel for evaluation with possible ipython
         # special-syntax conversion.
         isp.push('run foo')
         self.assertFalse(isp.push_accepts_more())
 
     def test_unicode(self):
         self.isp.push(u"Pérez")
         self.isp.push(u'\xc3\xa9')
         self.isp.push(u"u'\xc3\xa9'")
 
     def test_line_continuation(self):
         """ Test issue #2108."""
         isp = self.isp
         # A blank line after a line continuation should not accept more
         isp.push("1 \\\n\n")
         self.assertFalse(isp.push_accepts_more())
         # Whitespace after a \ is a SyntaxError.  The only way to test that
         # here is to test that push doesn't accept more (as with
         # test_syntax_error() above).
         isp.push(r"1 \ ")
         self.assertFalse(isp.push_accepts_more())
         # Even if the line is continuable (c.f. the regular Python
         # interpreter)
         isp.push(r"(1 \ ")
         self.assertFalse(isp.push_accepts_more())
 
 class InteractiveLoopTestCase(unittest.TestCase):
     """Tests for an interactive loop like a python shell.
     """
     def check_ns(self, lines, ns):
         """Validate that the given input lines produce the resulting namespace.
 
         Note: the input lines are given exactly as they would be typed in an
         auto-indenting environment, as mini_interactive_loop above already does
         auto-indenting and prepends spaces to the input.
         """
         src = mini_interactive_loop(pseudo_input(lines))
         test_ns = {}
         exec src in test_ns
         # We can't check that the provided ns is identical to the test_ns,
         # because Python fills test_ns with extra keys (copyright, etc).  But
         # we can check that the given dict is *contained* in test_ns
         for k,v in ns.iteritems():
             self.assertEqual(test_ns[k], v)
 
     def test_simple(self):
         self.check_ns(['x=1'], dict(x=1))
 
     def test_simple2(self):
         self.check_ns(['if 1:', 'x=2'], dict(x=2))
 
     def test_xy(self):
         self.check_ns(['x=1; y=2'], dict(x=1, y=2))
 
     def test_abc(self):
         self.check_ns(['if 1:','a=1','b=2','c=3'], dict(a=1, b=2, c=3))
 
     def test_multi(self):
         self.check_ns(['x =(1+','1+','2)'], dict(x=4))
 
 
 def test_LineInfo():
     """Simple test for LineInfo construction and str()"""
     linfo = isp.LineInfo('  %cd /home')
     nt.assert_equal(str(linfo), 'LineInfo [  |%|cd|/home]')
 
 
 
 
 class IPythonInputTestCase(InputSplitterTestCase):
     """By just creating a new class whose .isp is a different instance, we
     re-run the same test battery on the new input splitter.
 
     In addition, this runs the tests over the syntax and syntax_ml dicts that
     were tested by individual functions, as part of the OO interface.
 
     It also makes some checks on the raw buffer storage.
     """
 
     def setUp(self):
-        self.isp = isp.IPythonInputSplitter(input_mode='line')
+        self.isp = isp.IPythonInputSplitter()
 
     def test_syntax(self):
         """Call all single-line syntax tests from the main object"""
         isp = self.isp
         for example in syntax.itervalues():
             for raw, out_t in example:
                 if raw.startswith(' '):
                     continue
 
                 isp.push(raw+'\n')
                 out, out_raw = isp.source_raw_reset()
                 self.assertEqual(out.rstrip(), out_t,
                         tt.pair_fail_msg.format("inputsplitter",raw, out_t, out))
                 self.assertEqual(out_raw.rstrip(), raw.rstrip())
 
     def test_syntax_multiline(self):
         isp = self.isp
         for example in syntax_ml.itervalues():
             for line_pairs in example:
                 out_t_parts = []
                 raw_parts = []
                 for lraw, out_t_part in line_pairs:
                     if out_t_part is not None:
                         out_t_parts.append(out_t_part)
                     
                     if lraw is not None:
                         isp.push(lraw)
                         raw_parts.append(lraw)
 
                 out, out_raw = isp.source_raw_reset()
                 out_t = '\n'.join(out_t_parts).rstrip()
                 raw = '\n'.join(raw_parts).rstrip()
                 self.assertEqual(out.rstrip(), out_t)
                 self.assertEqual(out_raw.rstrip(), raw)
 
-
-class BlockIPythonInputTestCase(IPythonInputTestCase):
-
-    # Deactivate tests that don't make sense for the block mode
-    test_push3 = test_split = lambda s: None
-
-    def setUp(self):
-        self.isp = isp.IPythonInputSplitter(input_mode='cell')
-
-    def test_syntax_multiline(self):
-        isp = self.isp
-        for example in syntax_ml.itervalues():
-            raw_parts = []
-            out_t_parts = []
-            for line_pairs in example:
-                raw_parts, out_t_parts = zip(*line_pairs)
-
-                raw = '\n'.join(r for r in raw_parts if r is not None)
-                out_t = '\n'.join(o for o in out_t_parts if o is not None)
-
-                isp.push(raw)
-                out, out_raw = isp.source_raw_reset()
-                # Match ignoring trailing whitespace
-                self.assertEqual(out.rstrip(), out_t.rstrip())
-                self.assertEqual(out_raw.rstrip(), raw.rstrip())
-
     def test_syntax_multiline_cell(self):
         isp = self.isp
         for example in syntax_ml.itervalues():
 
             out_t_parts = []
             for line_pairs in example:
                 raw = '\n'.join(r for r, _ in line_pairs if r is not None)
                 out_t = '\n'.join(t for _,t in line_pairs if t is not None)
                 out = isp.transform_cell(raw)
                 # Match ignoring trailing whitespace
                 self.assertEqual(out.rstrip(), out_t.rstrip())
 
 #-----------------------------------------------------------------------------
 # Main - use as a script, mostly for developer experiments
 #-----------------------------------------------------------------------------
 
 if __name__ == '__main__':
     # A simple demo for interactive experimentation.  This code will not get
     # picked up by any test suite.
     from IPython.core.inputsplitter import InputSplitter, IPythonInputSplitter
 
     # configure here the syntax to use, prompt and whether to autoindent
     #isp, start_prompt = InputSplitter(), '>>> '
     isp, start_prompt = IPythonInputSplitter(), 'In> '
 
     autoindent = True
     #autoindent = False
 
     try:
         while True:
             prompt = start_prompt
             while isp.push_accepts_more():
                 indent = ' '*isp.indent_spaces
                 if autoindent:
                     line = indent + raw_input(prompt+indent)
                 else:
                     line = raw_input(prompt)
                 isp.push(line)
                 prompt = '... '
 
             # Here we just return input so we can use it in a test suite, but a
             # real interpreter would instead send it for execution somewhere.
             #src = isp.source; raise EOFError # dbg
             src, raw = isp.source_raw_reset()
             print 'Input source was:\n', src
             print 'Raw source was:\n', raw
     except EOFError:
         print 'Bye'
 
 # Tests for cell magics support
 
 def test_last_blank():
     nt.assert_false(isp.last_blank(''))
     nt.assert_false(isp.last_blank('abc'))
     nt.assert_false(isp.last_blank('abc\n'))
     nt.assert_false(isp.last_blank('abc\na'))
 
     nt.assert_true(isp.last_blank('\n'))
     nt.assert_true(isp.last_blank('\n '))
     nt.assert_true(isp.last_blank('abc\n '))
     nt.assert_true(isp.last_blank('abc\n\n'))
     nt.assert_true(isp.last_blank('abc\nd\n\n'))
     nt.assert_true(isp.last_blank('abc\nd\ne\n\n'))
     nt.assert_true(isp.last_blank('abc \n \n \n\n'))
 
 
 def test_last_two_blanks():
     nt.assert_false(isp.last_two_blanks(''))
     nt.assert_false(isp.last_two_blanks('abc'))
     nt.assert_false(isp.last_two_blanks('abc\n'))
     nt.assert_false(isp.last_two_blanks('abc\n\na'))
     nt.assert_false(isp.last_two_blanks('abc\n \n'))
     nt.assert_false(isp.last_two_blanks('abc\n\n'))
 
     nt.assert_true(isp.last_two_blanks('\n\n'))
     nt.assert_true(isp.last_two_blanks('\n\n '))
     nt.assert_true(isp.last_two_blanks('\n \n'))
     nt.assert_true(isp.last_two_blanks('abc\n\n '))
     nt.assert_true(isp.last_two_blanks('abc\n\n\n'))
     nt.assert_true(isp.last_two_blanks('abc\n\n \n'))
     nt.assert_true(isp.last_two_blanks('abc\n\n \n '))
     nt.assert_true(isp.last_two_blanks('abc\n\n \n \n'))
     nt.assert_true(isp.last_two_blanks('abc\nd\n\n\n'))
     nt.assert_true(isp.last_two_blanks('abc\nd\ne\nf\n\n\n'))
 
 
 class CellMagicsCommon(object):
 
     def test_whole_cell(self):
         src = "%%cellm line\nbody\n"
         sp = self.sp
         sp.push(src)
         out = sp.source_reset()
         ref = u"get_ipython().run_cell_magic({u}'cellm', {u}'line', {u}'body')\n"
         nt.assert_equal(out, py3compat.u_format(ref))
 
     def tearDown(self):
         self.sp.reset()
 
 
 class CellModeCellMagics(CellMagicsCommon, unittest.TestCase):
     sp = isp.IPythonInputSplitter(input_mode='cell')
 
     def test_incremental(self):
         sp = self.sp
         src = '%%cellm line2\n'
         sp.push(src)
         nt.assert_true(sp.push_accepts_more()) #1
         src += '\n'
         sp.push(src)
         # Note: if we ever change the logic to allow full blank lines (see
         # _handle_cell_magic), then the following test should change to true
         nt.assert_false(sp.push_accepts_more()) #2
         # By now, even with full blanks allowed, a second blank should signal
         # the end.  For now this test is only a redundancy safety, but don't
         # delete it in case we change our mind and the previous one goes to
         # true.
         src += '\n'
         sp.push(src)
         nt.assert_false(sp.push_accepts_more()) #3
 
 
 class LineModeCellMagics(CellMagicsCommon, unittest.TestCase):
     sp = isp.IPythonInputSplitter(input_mode='line')
 
     def test_incremental(self):
         sp = self.sp
         sp.push('%%cellm line2\n')
         nt.assert_true(sp.push_accepts_more()) #1
         sp.push('\n')
         nt.assert_false(sp.push_accepts_more()) #2

IPython/frontend/qt/console/frontend_widget.py

@@ -1,770 +1,771 b''
 from __future__ import print_function
 
 # Standard library imports
 from collections import namedtuple
 import sys
 import time
 import uuid
 
 # System library imports
 from pygments.lexers import PythonLexer
 from IPython.external import qt
 from IPython.external.qt import QtCore, QtGui
 
 # Local imports
 from IPython.core.inputsplitter import InputSplitter, IPythonInputSplitter
 from IPython.core.inputtransformer import classic_prompt
 from IPython.core.oinspect import call_tip
 from IPython.frontend.qt.base_frontend_mixin import BaseFrontendMixin
 from IPython.utils.traitlets import Bool, Instance, Unicode
 from bracket_matcher import BracketMatcher
 from call_tip_widget import CallTipWidget
 from completion_lexer import CompletionLexer
 from history_console_widget import HistoryConsoleWidget
 from pygments_highlighter import PygmentsHighlighter
 
 
 class FrontendHighlighter(PygmentsHighlighter):
     """ A PygmentsHighlighter that understands and ignores prompts.
     """
 
     def __init__(self, frontend):
         super(FrontendHighlighter, self).__init__(frontend._control.document())
         self._current_offset = 0
         self._frontend = frontend
         self.highlighting_on = False
 
     def highlightBlock(self, string):
         """ Highlight a block of text. Reimplemented to highlight selectively.
         """
         if not self.highlighting_on:
             return
 
         # The input to this function is a unicode string that may contain
         # paragraph break characters, non-breaking spaces, etc. Here we acquire
         # the string as plain text so we can compare it.
         current_block = self.currentBlock()
         string = self._frontend._get_block_plain_text(current_block)
 
         # Decide whether to check for the regular or continuation prompt.
         if current_block.contains(self._frontend._prompt_pos):
             prompt = self._frontend._prompt
         else:
             prompt = self._frontend._continuation_prompt
 
         # Only highlight if we can identify a prompt, but make sure not to
         # highlight the prompt.
         if string.startswith(prompt):
             self._current_offset = len(prompt)
             string = string[len(prompt):]
             super(FrontendHighlighter, self).highlightBlock(string)
 
     def rehighlightBlock(self, block):
         """ Reimplemented to temporarily enable highlighting if disabled.
         """
         old = self.highlighting_on
         self.highlighting_on = True
         super(FrontendHighlighter, self).rehighlightBlock(block)
         self.highlighting_on = old
 
     def setFormat(self, start, count, format):
         """ Reimplemented to highlight selectively.
         """
         start += self._current_offset
         super(FrontendHighlighter, self).setFormat(start, count, format)
 
 
 class FrontendWidget(HistoryConsoleWidget, BaseFrontendMixin):
     """ A Qt frontend for a generic Python kernel.
     """
 
     # The text to show when the kernel is (re)started.
     banner = Unicode()
 
     # An option and corresponding signal for overriding the default kernel
     # interrupt behavior.
     custom_interrupt = Bool(False)
     custom_interrupt_requested = QtCore.Signal()
 
     # An option and corresponding signals for overriding the default kernel
     # restart behavior.
     custom_restart = Bool(False)
     custom_restart_kernel_died = QtCore.Signal(float)
     custom_restart_requested = QtCore.Signal()
 
     # Whether to automatically show calltips on open-parentheses.
     enable_calltips = Bool(True, config=True,
         help="Whether to draw information calltips on open-parentheses.")
 
     clear_on_kernel_restart = Bool(True, config=True,
         help="Whether to clear the console when the kernel is restarted")
 
     confirm_restart = Bool(True, config=True,
         help="Whether to ask for user confirmation when restarting kernel")
 
     # Emitted when a user visible 'execute_request' has been submitted to the
     # kernel from the FrontendWidget. Contains the code to be executed.
     executing = QtCore.Signal(object)
 
     # Emitted when a user-visible 'execute_reply' has been received from the
     # kernel and processed by the FrontendWidget. Contains the response message.
     executed = QtCore.Signal(object)
 
     # Emitted when an exit request has been received from the kernel.
     exit_requested = QtCore.Signal(object)
 
     # Protected class variables.
     _prompt_transformer = IPythonInputSplitter(physical_line_transforms=[classic_prompt()],
                                                logical_line_transforms=[],
                                                python_line_transforms=[],
                                               )
     _CallTipRequest = namedtuple('_CallTipRequest', ['id', 'pos'])
     _CompletionRequest = namedtuple('_CompletionRequest', ['id', 'pos'])
     _ExecutionRequest = namedtuple('_ExecutionRequest', ['id', 'kind'])
     _input_splitter_class = InputSplitter
     _local_kernel = False
     _highlighter = Instance(FrontendHighlighter)
 
     #---------------------------------------------------------------------------
     # 'object' interface
     #---------------------------------------------------------------------------
 
     def __init__(self, *args, **kw):
         super(FrontendWidget, self).__init__(*args, **kw)
         # FIXME: remove this when PySide min version is updated past 1.0.7
         # forcefully disable calltips if PySide is < 1.0.7, because they crash
         if qt.QT_API == qt.QT_API_PYSIDE:
             import PySide
             if PySide.__version_info__ < (1,0,7):
                 self.log.warn("PySide %s < 1.0.7 detected, disabling calltips" % PySide.__version__)
                 self.enable_calltips = False
 
         # FrontendWidget protected variables.
         self._bracket_matcher = BracketMatcher(self._control)
         self._call_tip_widget = CallTipWidget(self._control)
         self._completion_lexer = CompletionLexer(PythonLexer())
         self._copy_raw_action = QtGui.QAction('Copy (Raw Text)', None)
         self._hidden = False
         self._highlighter = FrontendHighlighter(self)
-        self._input_splitter = self._input_splitter_class(input_mode='cell')
+        self._input_splitter = self._input_splitter_class()
         self._kernel_manager = None
         self._request_info = {}
         self._request_info['execute'] = {};
         self._callback_dict = {}
 
         # Configure the ConsoleWidget.
         self.tab_width = 4
         self._set_continuation_prompt('... ')
 
         # Configure the CallTipWidget.
         self._call_tip_widget.setFont(self.font)
         self.font_changed.connect(self._call_tip_widget.setFont)
 
         # Configure actions.
         action = self._copy_raw_action
         key = QtCore.Qt.CTRL | QtCore.Qt.SHIFT | QtCore.Qt.Key_C
         action.setEnabled(False)
         action.setShortcut(QtGui.QKeySequence(key))
         action.setShortcutContext(QtCore.Qt.WidgetWithChildrenShortcut)
         action.triggered.connect(self.copy_raw)
         self.copy_available.connect(action.setEnabled)
         self.addAction(action)
 
         # Connect signal handlers.
         document = self._control.document()
         document.contentsChange.connect(self._document_contents_change)
 
         # Set flag for whether we are connected via localhost.
         self._local_kernel = kw.get('local_kernel',
                                     FrontendWidget._local_kernel)
 
     #---------------------------------------------------------------------------
     # 'ConsoleWidget' public interface
     #---------------------------------------------------------------------------
 
     def copy(self):
         """ Copy the currently selected text to the clipboard, removing prompts.
         """
         if self._page_control is not None and self._page_control.hasFocus():
             self._page_control.copy()
         elif self._control.hasFocus():
             text = self._control.textCursor().selection().toPlainText()
             if text:
                 text = self._prompt_transformer.transform_cell(text)
                 QtGui.QApplication.clipboard().setText(text)
         else:
             self.log.debug("frontend widget : unknown copy target")
 
     #---------------------------------------------------------------------------
     # 'ConsoleWidget' abstract interface
     #---------------------------------------------------------------------------
 
     def _is_complete(self, source, interactive):
         """ Returns whether 'source' can be completely processed and a new
             prompt created. When triggered by an Enter/Return key press,
             'interactive' is True; otherwise, it is False.
         """
+        self._input_splitter.reset()
         complete = self._input_splitter.push(source)
         if interactive:
             complete = not self._input_splitter.push_accepts_more()
         return complete
 
     def _execute(self, source, hidden):
         """ Execute 'source'. If 'hidden', do not show any output.
 
         See parent class :meth:`execute` docstring for full details.
         """
         msg_id = self.kernel_manager.shell_channel.execute(source, hidden)
         self._request_info['execute'][msg_id] = self._ExecutionRequest(msg_id, 'user')
         self._hidden = hidden
         if not hidden:
             self.executing.emit(source)
 
     def _prompt_started_hook(self):
         """ Called immediately after a new prompt is displayed.
         """
         if not self._reading:
             self._highlighter.highlighting_on = True
 
     def _prompt_finished_hook(self):
         """ Called immediately after a prompt is finished, i.e. when some input
             will be processed and a new prompt displayed.
         """
         # Flush all state from the input splitter so the next round of
         # reading input starts with a clean buffer.
         self._input_splitter.reset()
 
         if not self._reading:
             self._highlighter.highlighting_on = False
 
     def _tab_pressed(self):
         """ Called when the tab key is pressed. Returns whether to continue
             processing the event.
         """
         # Perform tab completion if:
         # 1) The cursor is in the input buffer.
         # 2) There is a non-whitespace character before the cursor.
         text = self._get_input_buffer_cursor_line()
         if text is None:
             return False
         complete = bool(text[:self._get_input_buffer_cursor_column()].strip())
         if complete:
             self._complete()
         return not complete
 
     #---------------------------------------------------------------------------
     # 'ConsoleWidget' protected interface
     #---------------------------------------------------------------------------
 
     def _context_menu_make(self, pos):
         """ Reimplemented to add an action for raw copy.
         """
         menu = super(FrontendWidget, self)._context_menu_make(pos)
         for before_action in menu.actions():
             if before_action.shortcut().matches(QtGui.QKeySequence.Paste) == \
                     QtGui.QKeySequence.ExactMatch:
                 menu.insertAction(before_action, self._copy_raw_action)
                 break
         return menu
 
     def request_interrupt_kernel(self):
         if self._executing:
             self.interrupt_kernel()
 
     def request_restart_kernel(self):
         message = 'Are you sure you want to restart the kernel?'
         self.restart_kernel(message, now=False)
 
     def _event_filter_console_keypress(self, event):
         """ Reimplemented for execution interruption and smart backspace.
         """
         key = event.key()
         if self._control_key_down(event.modifiers(), include_command=False):
 
             if key == QtCore.Qt.Key_C and self._executing:
                 self.request_interrupt_kernel()
                 return True
 
             elif key == QtCore.Qt.Key_Period:
                 self.request_restart_kernel()
                 return True
 
         elif not event.modifiers() & QtCore.Qt.AltModifier:
 
             # Smart backspace: remove four characters in one backspace if:
             # 1) everything left of the cursor is whitespace
             # 2) the four characters immediately left of the cursor are spaces
             if key == QtCore.Qt.Key_Backspace:
                 col = self._get_input_buffer_cursor_column()
                 cursor = self._control.textCursor()
                 if col > 3 and not cursor.hasSelection():
                     text = self._get_input_buffer_cursor_line()[:col]
                     if text.endswith('    ') and not text.strip():
                         cursor.movePosition(QtGui.QTextCursor.Left,
                                             QtGui.QTextCursor.KeepAnchor, 4)
                         cursor.removeSelectedText()
                         return True
 
         return super(FrontendWidget, self)._event_filter_console_keypress(event)
 
     def _insert_continuation_prompt(self, cursor):
         """ Reimplemented for auto-indentation.
         """
         super(FrontendWidget, self)._insert_continuation_prompt(cursor)
         cursor.insertText(' ' * self._input_splitter.indent_spaces)
 
     #---------------------------------------------------------------------------
     # 'BaseFrontendMixin' abstract interface
     #---------------------------------------------------------------------------
 
     def _handle_complete_reply(self, rep):
         """ Handle replies for tab completion.
         """
         self.log.debug("complete: %s", rep.get('content', ''))
         cursor = self._get_cursor()
         info = self._request_info.get('complete')
         if info and info.id == rep['parent_header']['msg_id'] and \
                 info.pos == cursor.position():
             text = '.'.join(self._get_context())
             cursor.movePosition(QtGui.QTextCursor.Left, n=len(text))
             self._complete_with_items(cursor, rep['content']['matches'])
 
     def _silent_exec_callback(self, expr, callback):
         """Silently execute `expr` in the kernel and call `callback` with reply
 
         the `expr` is evaluated silently in the kernel (without) output in
         the frontend. Call `callback` with the
         `repr <http://docs.python.org/library/functions.html#repr> `_ as first argument
 
         Parameters
         ----------
         expr : string
             valid string to be executed by the kernel.
         callback : function
             function accepting one argument, as a string. The string will be
             the `repr` of the result of evaluating `expr`
 
         The `callback` is called with the `repr()` of the result of `expr` as
         first argument. To get the object, do `eval()` on the passed value.
 
         See Also
         --------
         _handle_exec_callback : private method, deal with calling callback with reply
 
         """
 
         # generate uuid, which would be used as an indication of whether or
         # not the unique request originated from here (can use msg id ?)
         local_uuid = str(uuid.uuid1())
         msg_id = self.kernel_manager.shell_channel.execute('',
             silent=True, user_expressions={ local_uuid:expr })
         self._callback_dict[local_uuid] = callback
         self._request_info['execute'][msg_id] = self._ExecutionRequest(msg_id, 'silent_exec_callback')
 
     def _handle_exec_callback(self, msg):
         """Execute `callback` corresponding to `msg` reply, after ``_silent_exec_callback``
 
         Parameters
         ----------
         msg : raw message send by the kernel containing an `user_expressions`
                 and having a 'silent_exec_callback' kind.
 
         Notes
         -----
         This function will look for a `callback` associated with the
         corresponding message id. Association has been made by
         `_silent_exec_callback`. `callback` is then called with the `repr()`
         of the value of corresponding `user_expressions` as argument.
         `callback` is then removed from the known list so that any message
         coming again with the same id won't trigger it.
 
         """
 
         user_exp = msg['content'].get('user_expressions')
         if not user_exp:
             return
         for expression in user_exp:
             if expression in self._callback_dict:
                 self._callback_dict.pop(expression)(user_exp[expression])
 
     def _handle_execute_reply(self, msg):
         """ Handles replies for code execution.
         """
         self.log.debug("execute: %s", msg.get('content', ''))
         msg_id = msg['parent_header']['msg_id']
         info = self._request_info['execute'].get(msg_id)
         # unset reading flag, because if execute finished, raw_input can't
         # still be pending.
         self._reading = False
         if info and info.kind == 'user' and not self._hidden:
             # Make sure that all output from the SUB channel has been processed
             # before writing a new prompt.
             self.kernel_manager.iopub_channel.flush()
 
             # Reset the ANSI style information to prevent bad text in stdout
             # from messing up our colors. We're not a true terminal so we're
             # allowed to do this.
             if self.ansi_codes:
                 self._ansi_processor.reset_sgr()
 
             content = msg['content']
             status = content['status']
             if status == 'ok':
                 self._process_execute_ok(msg)
             elif status == 'error':
                 self._process_execute_error(msg)
             elif status == 'aborted':
                 self._process_execute_abort(msg)
 
             self._show_interpreter_prompt_for_reply(msg)
             self.executed.emit(msg)
             self._request_info['execute'].pop(msg_id)
         elif info and info.kind == 'silent_exec_callback' and not self._hidden:
             self._handle_exec_callback(msg)
             self._request_info['execute'].pop(msg_id)
         else:
             super(FrontendWidget, self)._handle_execute_reply(msg)
 
     def _handle_input_request(self, msg):
         """ Handle requests for raw_input.
         """
         self.log.debug("input: %s", msg.get('content', ''))
         if self._hidden:
             raise RuntimeError('Request for raw input during hidden execution.')
 
         # Make sure that all output from the SUB channel has been processed
         # before entering readline mode.
         self.kernel_manager.iopub_channel.flush()
 
         def callback(line):
             self.kernel_manager.stdin_channel.input(line)
         if self._reading:
             self.log.debug("Got second input request, assuming first was interrupted.")
             self._reading = False
         self._readline(msg['content']['prompt'], callback=callback)
 
     def _handle_kernel_died(self, since_last_heartbeat):
         """ Handle the kernel's death by asking if the user wants to restart.
         """
         self.log.debug("kernel died: %s", since_last_heartbeat)
         if self.custom_restart:
             self.custom_restart_kernel_died.emit(since_last_heartbeat)
         else:
             message = 'The kernel heartbeat has been inactive for %.2f ' \
                 'seconds. Do you want to restart the kernel? You may ' \
                 'first want to check the network connection.' % \
                 since_last_heartbeat
             self.restart_kernel(message, now=True)
 
     def _handle_object_info_reply(self, rep):
         """ Handle replies for call tips.
         """
         self.log.debug("oinfo: %s", rep.get('content', ''))
         cursor = self._get_cursor()
         info = self._request_info.get('call_tip')
         if info and info.id == rep['parent_header']['msg_id'] and \
                 info.pos == cursor.position():
             # Get the information for a call tip.  For now we format the call
             # line as string, later we can pass False to format_call and
             # syntax-highlight it ourselves for nicer formatting in the
             # calltip.
             content = rep['content']
             # if this is from pykernel, 'docstring' will be the only key
             if content.get('ismagic', False):
                 # Don't generate a call-tip for magics. Ideally, we should
                 # generate a tooltip, but not on ( like we do for actual
                 # callables.
                 call_info, doc = None, None
             else:
                 call_info, doc = call_tip(content, format_call=True)
             if call_info or doc:
                 self._call_tip_widget.show_call_info(call_info, doc)
 
     def _handle_pyout(self, msg):
         """ Handle display hook output.
         """
         self.log.debug("pyout: %s", msg.get('content', ''))
         if not self._hidden and self._is_from_this_session(msg):
             text = msg['content']['data']
             self._append_plain_text(text + '\n', before_prompt=True)
 
     def _handle_stream(self, msg):
         """ Handle stdout, stderr, and stdin.
         """
         self.log.debug("stream: %s", msg.get('content', ''))
         if not self._hidden and self._is_from_this_session(msg):
             # Most consoles treat tabs as being 8 space characters. Convert tabs
             # to spaces so that output looks as expected regardless of this
             # widget's tab width.
             text = msg['content']['data'].expandtabs(8)
 
             self._append_plain_text(text, before_prompt=True)
             self._control.moveCursor(QtGui.QTextCursor.End)
 
     def _handle_shutdown_reply(self, msg):
         """ Handle shutdown signal, only if from other console.
         """
         self.log.debug("shutdown: %s", msg.get('content', ''))
         if not self._hidden and not self._is_from_this_session(msg):
             if self._local_kernel:
                 if not msg['content']['restart']:
                     self.exit_requested.emit(self)
                 else:
                     # we just got notified of a restart!
                     time.sleep(0.25) # wait 1/4 sec to reset
                                      # lest the request for a new prompt
                                      # goes to the old kernel
                     self.reset()
             else: # remote kernel, prompt on Kernel shutdown/reset
                 title = self.window().windowTitle()
                 if not msg['content']['restart']:
                     reply = QtGui.QMessageBox.question(self, title,
                         "Kernel has been shutdown permanently. "
                         "Close the Console?",
                         QtGui.QMessageBox.Yes,QtGui.QMessageBox.No)
                     if reply == QtGui.QMessageBox.Yes:
                         self.exit_requested.emit(self)
                 else:
                     # XXX: remove message box in favor of using the
                     # clear_on_kernel_restart setting?
                     reply = QtGui.QMessageBox.question(self, title,
                         "Kernel has been reset. Clear the Console?",
                         QtGui.QMessageBox.Yes,QtGui.QMessageBox.No)
                     if reply == QtGui.QMessageBox.Yes:
                         time.sleep(0.25) # wait 1/4 sec to reset
                                          # lest the request for a new prompt
                                          # goes to the old kernel
                         self.reset()
 
     def _started_channels(self):
         """ Called when the KernelManager channels have started listening or
             when the frontend is assigned an already listening KernelManager.
         """
         self.reset(clear=True)
 
     #---------------------------------------------------------------------------
     # 'FrontendWidget' public interface
     #---------------------------------------------------------------------------
 
     def copy_raw(self):
         """ Copy the currently selected text to the clipboard without attempting
             to remove prompts or otherwise alter the text.
         """
         self._control.copy()
 
     def execute_file(self, path, hidden=False):
         """ Attempts to execute file with 'path'. If 'hidden', no output is
             shown.
         """
         self.execute('execfile(%r)' % path, hidden=hidden)
 
     def interrupt_kernel(self):
         """ Attempts to interrupt the running kernel.
         
         Also unsets _reading flag, to avoid runtime errors
         if raw_input is called again.
         """
         if self.custom_interrupt:
             self._reading = False
             self.custom_interrupt_requested.emit()
         elif self.kernel_manager.has_kernel:
             self._reading = False
             self.kernel_manager.interrupt_kernel()
         else:
             self._append_plain_text('Kernel process is either remote or '
                                     'unspecified. Cannot interrupt.\n')
 
     def reset(self, clear=False):
         """ Resets the widget to its initial state if ``clear`` parameter or
         ``clear_on_kernel_restart`` configuration setting is True, otherwise
         prints a visual indication of the fact that the kernel restarted, but
         does not clear the traces from previous usage of the kernel before it
         was restarted.  With ``clear=True``, it is similar to ``%clear``, but
         also re-writes the banner and aborts execution if necessary.
         """
         if self._executing:
             self._executing = False
             self._request_info['execute'] = {}
         self._reading = False
         self._highlighter.highlighting_on = False
 
         if self.clear_on_kernel_restart or clear:
             self._control.clear()
             self._append_plain_text(self.banner)
         else:
             self._append_plain_text("# restarting kernel...")
             self._append_html("<hr><br>")
             # XXX: Reprinting the full banner may be too much, but once #1680 is
             # addressed, that will mitigate it.
             #self._append_plain_text(self.banner)
         # update output marker for stdout/stderr, so that startup
         # messages appear after banner:
         self._append_before_prompt_pos = self._get_cursor().position()
         self._show_interpreter_prompt()
 
     def restart_kernel(self, message, now=False):
         """ Attempts to restart the running kernel.
         """
         # FIXME: now should be configurable via a checkbox in the dialog.  Right
         # now at least the heartbeat path sets it to True and the manual restart
         # to False.  But those should just be the pre-selected states of a
         # checkbox that the user could override if so desired.  But I don't know
         # enough Qt to go implementing the checkbox now.
 
         if self.custom_restart:
             self.custom_restart_requested.emit()
 
         elif self.kernel_manager.has_kernel:
             # Pause the heart beat channel to prevent further warnings.
             self.kernel_manager.hb_channel.pause()
 
             # Prompt the user to restart the kernel. Un-pause the heartbeat if
             # they decline. (If they accept, the heartbeat will be un-paused
             # automatically when the kernel is restarted.)
             if self.confirm_restart:
                 buttons = QtGui.QMessageBox.Yes | QtGui.QMessageBox.No
                 result = QtGui.QMessageBox.question(self, 'Restart kernel?',
                                                     message, buttons)
                 do_restart = result == QtGui.QMessageBox.Yes
             else:
                 # confirm_restart is False, so we don't need to ask user
                 # anything, just do the restart
                 do_restart = True
             if do_restart:
                 try:
                     self.kernel_manager.restart_kernel(now=now)
                 except RuntimeError:
                     self._append_plain_text('Kernel started externally. '
                                             'Cannot restart.\n',
                                             before_prompt=True
                                             )
                 else:
                     self.reset()
             else:
                 self.kernel_manager.hb_channel.unpause()
 
         else:
             self._append_plain_text('Kernel process is either remote or '
                                     'unspecified. Cannot restart.\n',
                                     before_prompt=True
                                     )
 
     #---------------------------------------------------------------------------
     # 'FrontendWidget' protected interface
     #---------------------------------------------------------------------------
 
     def _call_tip(self):
         """ Shows a call tip, if appropriate, at the current cursor location.
         """
         # Decide if it makes sense to show a call tip
         if not self.enable_calltips:
             return False
         cursor = self._get_cursor()
         cursor.movePosition(QtGui.QTextCursor.Left)
         if cursor.document().characterAt(cursor.position()) != '(':
             return False
         context = self._get_context(cursor)
         if not context:
             return False
 
         # Send the metadata request to the kernel
         name = '.'.join(context)
         msg_id = self.kernel_manager.shell_channel.object_info(name)
         pos = self._get_cursor().position()
         self._request_info['call_tip'] = self._CallTipRequest(msg_id, pos)
         return True
 
     def _complete(self):
         """ Performs completion at the current cursor location.
         """
         context = self._get_context()
         if context:
             # Send the completion request to the kernel
             msg_id = self.kernel_manager.shell_channel.complete(
                 '.'.join(context),                       # text
                 self._get_input_buffer_cursor_line(),    # line
                 self._get_input_buffer_cursor_column(),  # cursor_pos
                 self.input_buffer)                       # block
             pos = self._get_cursor().position()
             info = self._CompletionRequest(msg_id, pos)
             self._request_info['complete'] = info
 
     def _get_context(self, cursor=None):
         """ Gets the context for the specified cursor (or the current cursor
             if none is specified).
         """
         if cursor is None:
             cursor = self._get_cursor()
         cursor.movePosition(QtGui.QTextCursor.StartOfBlock,
                             QtGui.QTextCursor.KeepAnchor)
         text = cursor.selection().toPlainText()
         return self._completion_lexer.get_context(text)
 
     def _process_execute_abort(self, msg):
         """ Process a reply for an aborted execution request.
         """
         self._append_plain_text("ERROR: execution aborted\n")
 
     def _process_execute_error(self, msg):
         """ Process a reply for an execution request that resulted in an error.
         """
         content = msg['content']
         # If a SystemExit is passed along, this means exit() was called - also
         # all the ipython %exit magic syntax of '-k' to be used to keep
         # the kernel running
         if content['ename']=='SystemExit':
             keepkernel = content['evalue']=='-k' or content['evalue']=='True'
             self._keep_kernel_on_exit = keepkernel
             self.exit_requested.emit(self)
         else:
             traceback = ''.join(content['traceback'])
             self._append_plain_text(traceback)
 
     def _process_execute_ok(self, msg):
         """ Process a reply for a successful execution request.
         """
         payload = msg['content']['payload']
         for item in payload:
             if not self._process_execute_payload(item):
                 warning = 'Warning: received unknown payload of type %s'
                 print(warning % repr(item['source']))
 
     def _process_execute_payload(self, item):
         """ Process a single payload item from the list of payload items in an
             execution reply. Returns whether the payload was handled.
         """
         # The basic FrontendWidget doesn't handle payloads, as they are a
         # mechanism for going beyond the standard Python interpreter model.
         return False
 
     def _show_interpreter_prompt(self):
         """ Shows a prompt for the interpreter.
         """
         self._show_prompt('>>> ')
 
     def _show_interpreter_prompt_for_reply(self, msg):
         """ Shows a prompt for the interpreter given an 'execute_reply' message.
         """
         self._show_interpreter_prompt()
 
     #------ Signal handlers ----------------------------------------------------
 
     def _document_contents_change(self, position, removed, added):
         """ Called whenever the document's content changes. Display a call tip
             if appropriate.
         """
         # Calculate where the cursor should be *after* the change:
         position += added
 
         document = self._control.document()
         if position == self._get_cursor().position():
             self._call_tip()
 
     #------ Trait default initializers -----------------------------------------
 
     def _banner_default(self):
         """ Returns the standard Python banner.
         """
         banner = 'Python %s on %s\nType "help", "copyright", "credits" or ' \
             '"license" for more information.'
         return banner % (sys.version, sys.platform)

IPython/frontend/terminal/interactiveshell.py

@@ -1,752 +1,752 b''
 # -*- coding: utf-8 -*-
 """Subclass of InteractiveShell for terminal based frontends."""
 
 #-----------------------------------------------------------------------------
 #  Copyright (C) 2001 Janko Hauser <[e-mail]>
 #  Copyright (C) 2001-2007 Fernando Perez. <[e-mail]>
 #  Copyright (C) 2008-2011  The IPython Development Team
 #
 #  Distributed under the terms of the BSD License.  The full license is in
 #  the file COPYING, distributed as part of this software.
 #-----------------------------------------------------------------------------
 
 #-----------------------------------------------------------------------------
 # Imports
 #-----------------------------------------------------------------------------
 from __future__ import print_function
 
 import bdb
 import os
 import re
 import sys
 import textwrap
 
 # We need to use nested to support python 2.6, once we move to >=2.7, we can
 # use the with keyword's new builtin support for nested managers
 try:
     from contextlib import nested
 except:
     from IPython.utils.nested_context import nested
 
 from IPython.core.error import TryNext, UsageError
 from IPython.core.usage import interactive_usage, default_banner
 from IPython.core.inputsplitter import IPythonInputSplitter
 from IPython.core.interactiveshell import InteractiveShell, InteractiveShellABC
 from IPython.core.magic import Magics, magics_class, line_magic
 from IPython.testing.skipdoctest import skip_doctest
 from IPython.utils.encoding import get_stream_enc
 from IPython.utils import py3compat
 from IPython.utils.terminal import toggle_set_term_title, set_term_title
 from IPython.utils.process import abbrev_cwd
 from IPython.utils.warn import warn, error
 from IPython.utils.text import num_ini_spaces, SList, strip_email_quotes
 from IPython.utils.traitlets import Integer, CBool, Unicode
 
 #-----------------------------------------------------------------------------
 # Utilities
 #-----------------------------------------------------------------------------
 
 def get_default_editor():
     try:
         ed = os.environ['EDITOR']
     except KeyError:
         if os.name == 'posix':
             ed = 'vi'  # the only one guaranteed to be there!
         else:
             ed = 'notepad' # same in Windows!
     return ed
 
 
 def get_pasted_lines(sentinel, l_input=py3compat.input):
     """ Yield pasted lines until the user enters the given sentinel value.
     """
     print("Pasting code; enter '%s' alone on the line to stop or use Ctrl-D." \
           % sentinel)
     while True:
         try:
             l = l_input(':')
             if l == sentinel:
                 return
             else:
                 yield l
         except EOFError:
             print('<EOF>')
             return
 
 
 #------------------------------------------------------------------------
 # Terminal-specific magics
 #------------------------------------------------------------------------
 
 @magics_class
 class TerminalMagics(Magics):
     def __init__(self, shell):
         super(TerminalMagics, self).__init__(shell)
-        self.input_splitter = IPythonInputSplitter(input_mode='line')
+        self.input_splitter = IPythonInputSplitter()
 
     def cleanup_input(self, block):
         """Apply all possible IPython cleanups to an input block.
 
         This means:
 
         - remove any global leading whitespace (dedent)
         - remove any email quotes ('>') if they are present in *all* lines
         - apply all static inputsplitter transforms and break into sub-blocks
         - apply prefilter() to each sub-block that is a single line.
 
         Parameters
         ----------
         block : str
           A possibly multiline input string of code.
 
         Returns
         -------
         transformed block : str
           The input, with all transformations above applied.
         """
         # We have to effectively implement client-side the loop that is done by
         # the terminal frontend, and furthermore do it on a block that can
         # possibly contain multiple statments pasted in one go.
 
         # First, run the input through the block splitting code.  We should
         # eventually make this a self-contained method in the inputsplitter.
         isp = self.input_splitter
         isp.reset()
         b = textwrap.dedent(block)
 
         # Remove email quotes first.  These must be consistently applied to
         # *all* lines to be removed
         b = strip_email_quotes(b)
 
         # Split the input into independent sub-blocks so we can later do
         # prefiltering (which must be done *only* to single-line inputs)
         blocks = []
         last_block = []
         for line in b.splitlines():
             isp.push(line)
             last_block.append(line)
             if not isp.push_accepts_more():
                 blocks.append(isp.source_reset())
                 last_block = []
         if last_block:
             blocks.append('\n'.join(last_block))
 
         # Now, apply prefiltering to any one-line block to match the behavior
         # of the interactive terminal
         final_blocks = []
         for block in blocks:
             lines = block.splitlines()
             if len(lines) == 1:
                 final_blocks.append(self.shell.prefilter(lines[0]))
             else:
                 final_blocks.append(block)
 
         # We now have the final version of the input code as a list of blocks,
         # with all inputsplitter transformations applied and single-line blocks
         # run through prefilter.  For further processing, turn into a single
         # string as the rest of our apis use string inputs.
         return '\n'.join(final_blocks)
         
     def store_or_execute(self, block, name):
         """ Execute a block, or store it in a variable, per the user's request.
         """
 
         b = self.cleanup_input(block)
         if name:
             # If storing it for further editing
             self.shell.user_ns[name] = SList(b.splitlines())
             print("Block assigned to '%s'" % name)
         else:
             self.shell.user_ns['pasted_block'] = b
             self.shell.using_paste_magics = True
             try:
                 self.shell.run_cell(b)
             finally:
                 self.shell.using_paste_magics = False
 
     def rerun_pasted(self, name='pasted_block'):
         """ Rerun a previously pasted command.
         """
         b = self.shell.user_ns.get(name)
 
         # Sanity checks
         if b is None:
             raise UsageError('No previous pasted block available')
         if not isinstance(b, basestring):
             raise UsageError(
                 "Variable 'pasted_block' is not a string, can't execute")
 
         print("Re-executing '%s...' (%d chars)"% (b.split('\n',1)[0], len(b)))
         self.shell.run_cell(b)
 
     @line_magic
     def autoindent(self, parameter_s = ''):
         """Toggle autoindent on/off (if available)."""
 
         self.shell.set_autoindent()
         print("Automatic indentation is:",['OFF','ON'][self.shell.autoindent])
 
     @skip_doctest
     @line_magic
     def cpaste(self, parameter_s=''):
         """Paste & execute a pre-formatted code block from clipboard.
 
         You must terminate the block with '--' (two minus-signs) or Ctrl-D
         alone on the line. You can also provide your own sentinel with '%paste
         -s %%' ('%%' is the new sentinel for this operation)
 
         The block is dedented prior to execution to enable execution of method
         definitions. '>' and '+' characters at the beginning of a line are
         ignored, to allow pasting directly from e-mails, diff files and
         doctests (the '...' continuation prompt is also stripped).  The
         executed block is also assigned to variable named 'pasted_block' for
         later editing with '%edit pasted_block'.
 
         You can also pass a variable name as an argument, e.g. '%cpaste foo'.
         This assigns the pasted block to variable 'foo' as string, without
         dedenting or executing it (preceding >>> and + is still stripped)
 
         '%cpaste -r' re-executes the block previously entered by cpaste.
 
         Do not be alarmed by garbled output on Windows (it's a readline bug).
         Just press enter and type -- (and press enter again) and the block
         will be what was just pasted.
 
         IPython statements (magics, shell escapes) are not supported (yet).
 
         See also
         --------
         paste: automatically pull code from clipboard.
 
         Examples
         --------
         ::
 
           In [8]: %cpaste
           Pasting code; enter '--' alone on the line to stop.
           :>>> a = ["world!", "Hello"]
           :>>> print " ".join(sorted(a))
           :--
           Hello world!
         """
         opts, name = self.parse_options(parameter_s, 'rs:', mode='string')
         if 'r' in opts:
             self.rerun_pasted()
             return
 
         sentinel = opts.get('s', '--')
         block = '\n'.join(get_pasted_lines(sentinel))
         self.store_or_execute(block, name)
 
     @line_magic
     def paste(self, parameter_s=''):
         """Paste & execute a pre-formatted code block from clipboard.
 
         The text is pulled directly from the clipboard without user
         intervention and printed back on the screen before execution (unless
         the -q flag is given to force quiet mode).
 
         The block is dedented prior to execution to enable execution of method
         definitions. '>' and '+' characters at the beginning of a line are
         ignored, to allow pasting directly from e-mails, diff files and
         doctests (the '...' continuation prompt is also stripped).  The
         executed block is also assigned to variable named 'pasted_block' for
         later editing with '%edit pasted_block'.
 
         You can also pass a variable name as an argument, e.g. '%paste foo'.
         This assigns the pasted block to variable 'foo' as string, without
         executing it (preceding >>> and + is still stripped).
 
         Options
         -------
 
           -r: re-executes the block previously entered by cpaste.
 
           -q: quiet mode: do not echo the pasted text back to the terminal.
 
         IPython statements (magics, shell escapes) are not supported (yet).
 
         See also
         --------
         cpaste: manually paste code into terminal until you mark its end.
         """
         opts, name = self.parse_options(parameter_s, 'rq', mode='string')
         if 'r' in opts:
             self.rerun_pasted()
             return
         try:
             block = self.shell.hooks.clipboard_get()
         except TryNext as clipboard_exc:
             message = getattr(clipboard_exc, 'args')
             if message:
                 error(message[0])
             else:
                 error('Could not get text from the clipboard.')
             return
 
         # By default, echo back to terminal unless quiet mode is requested
         if 'q' not in opts:
             write = self.shell.write
             write(self.shell.pycolorize(block))
             if not block.endswith('\n'):
                 write('\n')
             write("## -- End pasted text --\n")
 
         self.store_or_execute(block, name)
 
     # Class-level: add a '%cls' magic only on Windows
     if sys.platform == 'win32':
         @line_magic
         def cls(self, s):
             """Clear screen.
             """
             os.system("cls")
 
 #-----------------------------------------------------------------------------
 # Main class
 #-----------------------------------------------------------------------------
 
 class TerminalInteractiveShell(InteractiveShell):
 
     autoedit_syntax = CBool(False, config=True,
         help="auto editing of files with syntax errors.")
     banner = Unicode('')
     banner1 = Unicode(default_banner, config=True,
         help="""The part of the banner to be printed before the profile"""
     )
     banner2 = Unicode('', config=True,
         help="""The part of the banner to be printed after the profile"""
     )
     confirm_exit = CBool(True, config=True,
         help="""
         Set to confirm when you try to exit IPython with an EOF (Control-D
         in Unix, Control-Z/Enter in Windows). By typing 'exit' or 'quit',
         you can force a direct exit without any confirmation.""",
     )
     # This display_banner only controls whether or not self.show_banner()
     # is called when mainloop/interact are called.  The default is False
     # because for the terminal based application, the banner behavior
     # is controlled by Global.display_banner, which IPythonApp looks at
     # to determine if *it* should call show_banner() by hand or not.
     display_banner = CBool(False) # This isn't configurable!
     embedded = CBool(False)
     embedded_active = CBool(False)
     editor = Unicode(get_default_editor(), config=True,
         help="Set the editor used by IPython (default to $EDITOR/vi/notepad)."
     )
     pager = Unicode('less', config=True,
         help="The shell program to be used for paging.")
 
     screen_length = Integer(0, config=True,
         help=
         """Number of lines of your screen, used to control printing of very
         long strings.  Strings longer than this number of lines will be sent
         through a pager instead of directly printed.  The default value for
         this is 0, which means IPython will auto-detect your screen size every
         time it needs to print certain potentially long strings (this doesn't
         change the behavior of the 'print' keyword, it's only triggered
         internally). If for some reason this isn't working well (it needs
         curses support), specify it yourself. Otherwise don't change the
         default.""",
     )
     term_title = CBool(False, config=True,
         help="Enable auto setting the terminal title."
     )
     
     # This `using_paste_magics` is used to detect whether the code is being
     # executed via paste magics functions
     using_paste_magics = CBool(False)
 
     # In the terminal, GUI control is done via PyOS_InputHook
     @staticmethod
     def enable_gui(gui=None, app=None):
         """Switch amongst GUI input hooks by name.
         """
         # Deferred import
         from IPython.lib.inputhook import enable_gui as real_enable_gui
         return real_enable_gui(gui, app)
     
     def __init__(self, config=None, ipython_dir=None, profile_dir=None,
                  user_ns=None, user_module=None, custom_exceptions=((),None),
                  usage=None, banner1=None, banner2=None, display_banner=None):
 
         super(TerminalInteractiveShell, self).__init__(
             config=config, ipython_dir=ipython_dir, profile_dir=profile_dir, user_ns=user_ns,
             user_module=user_module, custom_exceptions=custom_exceptions
         )
         # use os.system instead of utils.process.system by default,
         # because piped system doesn't make sense in the Terminal:
         self.system = self.system_raw
 
         self.init_term_title()
         self.init_usage(usage)
         self.init_banner(banner1, banner2, display_banner)
 
     #-------------------------------------------------------------------------
     # Overrides of init stages
     #-------------------------------------------------------------------------
 
     def init_display_formatter(self):
         super(TerminalInteractiveShell, self).init_display_formatter()
         # terminal only supports plaintext
         self.display_formatter.active_types = ['text/plain']
 
     #-------------------------------------------------------------------------
     # Things related to the terminal
     #-------------------------------------------------------------------------
 
     @property
     def usable_screen_length(self):
         if self.screen_length == 0:
             return 0
         else:
             num_lines_bot = self.separate_in.count('\n')+1
             return self.screen_length - num_lines_bot
 
     def init_term_title(self):
         # Enable or disable the terminal title.
         if self.term_title:
             toggle_set_term_title(True)
             set_term_title('IPython: ' + abbrev_cwd())
         else:
             toggle_set_term_title(False)
 
     #-------------------------------------------------------------------------
     # Things related to aliases
     #-------------------------------------------------------------------------
 
     def init_alias(self):
         # The parent class defines aliases that can be safely used with any
         # frontend.
         super(TerminalInteractiveShell, self).init_alias()
 
         # Now define aliases that only make sense on the terminal, because they
         # need direct access to the console in a way that we can't emulate in
         # GUI or web frontend
         if os.name == 'posix':
             aliases = [('clear', 'clear'), ('more', 'more'), ('less', 'less'),
                        ('man', 'man')]
         elif os.name == 'nt':
             aliases = [('cls', 'cls')]
 
 
         for name, cmd in aliases:
             self.alias_manager.define_alias(name, cmd)
 
     #-------------------------------------------------------------------------
     # Things related to the banner and usage
     #-------------------------------------------------------------------------
 
     def _banner1_changed(self):
         self.compute_banner()
 
     def _banner2_changed(self):
         self.compute_banner()
 
     def _term_title_changed(self, name, new_value):
         self.init_term_title()
 
     def init_banner(self, banner1, banner2, display_banner):
         if banner1 is not None:
             self.banner1 = banner1
         if banner2 is not None:
             self.banner2 = banner2
         if display_banner is not None:
             self.display_banner = display_banner
         self.compute_banner()
 
     def show_banner(self, banner=None):
         if banner is None:
             banner = self.banner
         self.write(banner)
 
     def compute_banner(self):
         self.banner = self.banner1
         if self.profile and self.profile != 'default':
             self.banner += '\nIPython profile: %s\n' % self.profile
         if self.banner2:
             self.banner += '\n' + self.banner2
 
     def init_usage(self, usage=None):
         if usage is None:
             self.usage = interactive_usage
         else:
             self.usage = usage
 
     #-------------------------------------------------------------------------
     # Mainloop and code execution logic
     #-------------------------------------------------------------------------
 
     def mainloop(self, display_banner=None):
         """Start the mainloop.
 
         If an optional banner argument is given, it will override the
         internally created default banner.
         """
 
         with nested(self.builtin_trap, self.display_trap):
 
             while 1:
                 try:
                     self.interact(display_banner=display_banner)
                     #self.interact_with_readline()
                     # XXX for testing of a readline-decoupled repl loop, call
                     # interact_with_readline above
                     break
                 except KeyboardInterrupt:
                     # this should not be necessary, but KeyboardInterrupt
                     # handling seems rather unpredictable...
                     self.write("\nKeyboardInterrupt in interact()\n")
 
     def _replace_rlhist_multiline(self, source_raw, hlen_before_cell):
         """Store multiple lines as a single entry in history"""
 
         # do nothing without readline or disabled multiline
         if not self.has_readline or not self.multiline_history:
             return hlen_before_cell
 
         # windows rl has no remove_history_item
         if not hasattr(self.readline, "remove_history_item"):
             return hlen_before_cell
 
         # skip empty cells
         if not source_raw.rstrip():
             return hlen_before_cell
 
         # nothing changed do nothing, e.g. when rl removes consecutive dups
         hlen = self.readline.get_current_history_length()
         if hlen == hlen_before_cell:
             return hlen_before_cell
 
         for i in range(hlen - hlen_before_cell):
             self.readline.remove_history_item(hlen - i - 1)
         stdin_encoding = get_stream_enc(sys.stdin, 'utf-8')
         self.readline.add_history(py3compat.unicode_to_str(source_raw.rstrip(),
                                     stdin_encoding))
         return self.readline.get_current_history_length()
 
     def interact(self, display_banner=None):
         """Closely emulate the interactive Python console."""
 
         # batch run -> do not interact
         if self.exit_now:
             return
 
         if display_banner is None:
             display_banner = self.display_banner
 
         if isinstance(display_banner, basestring):
             self.show_banner(display_banner)
         elif display_banner:
             self.show_banner()
 
         more = False
 
         if self.has_readline:
             self.readline_startup_hook(self.pre_readline)
             hlen_b4_cell = self.readline.get_current_history_length()
         else:
             hlen_b4_cell = 0
         # exit_now is set by a call to %Exit or %Quit, through the
         # ask_exit callback.
 
         while not self.exit_now:
             self.hooks.pre_prompt_hook()
             if more:
                 try:
                     prompt = self.prompt_manager.render('in2')
                 except:
                     self.showtraceback()
                 if self.autoindent:
                     self.rl_do_indent = True
 
             else:
                 try:
                     prompt = self.separate_in + self.prompt_manager.render('in')
                 except:
                     self.showtraceback()
             try:
                 line = self.raw_input(prompt)
                 if self.exit_now:
                     # quick exit on sys.std[in|out] close
                     break
                 if self.autoindent:
                     self.rl_do_indent = False
 
             except KeyboardInterrupt:
                 #double-guard against keyboardinterrupts during kbdint handling
                 try:
                     self.write('\nKeyboardInterrupt\n')
                     source_raw = self.input_splitter.source_raw_reset()[1]
                     hlen_b4_cell = \
                         self._replace_rlhist_multiline(source_raw, hlen_b4_cell)
                     more = False
                 except KeyboardInterrupt:
                     pass
             except EOFError:
                 if self.autoindent:
                     self.rl_do_indent = False
                     if self.has_readline:
                         self.readline_startup_hook(None)
                 self.write('\n')
                 self.exit()
             except bdb.BdbQuit:
                 warn('The Python debugger has exited with a BdbQuit exception.\n'
                      'Because of how pdb handles the stack, it is impossible\n'
                      'for IPython to properly format this particular exception.\n'
                      'IPython will resume normal operation.')
             except:
                 # exceptions here are VERY RARE, but they can be triggered
                 # asynchronously by signal handlers, for example.
                 self.showtraceback()
             else:
                 self.input_splitter.push(line)
                 more = self.input_splitter.push_accepts_more()
                 if (self.SyntaxTB.last_syntax_error and
                     self.autoedit_syntax):
                     self.edit_syntax_error()
                 if not more:
                     source_raw = self.input_splitter.source_raw_reset()[1]
                     self.run_cell(source_raw, store_history=True)
                     hlen_b4_cell = \
                         self._replace_rlhist_multiline(source_raw, hlen_b4_cell)
 
         # Turn off the exit flag, so the mainloop can be restarted if desired
         self.exit_now = False
 
     def raw_input(self, prompt=''):
         """Write a prompt and read a line.
 
         The returned line does not include the trailing newline.
         When the user enters the EOF key sequence, EOFError is raised.
 
         Optional inputs:
 
           - prompt(''): a string to be printed to prompt the user.
 
           - continue_prompt(False): whether this line is the first one or a
           continuation in a sequence of inputs.
         """
         # Code run by the user may have modified the readline completer state.
         # We must ensure that our completer is back in place.
 
         if self.has_readline:
             self.set_readline_completer()
         
         # raw_input expects str, but we pass it unicode sometimes
         prompt = py3compat.cast_bytes_py2(prompt)
 
         try:
             line = py3compat.str_to_unicode(self.raw_input_original(prompt))
         except ValueError:
             warn("\n********\nYou or a %run:ed script called sys.stdin.close()"
                  " or sys.stdout.close()!\nExiting IPython!\n")
             self.ask_exit()
             return ""
 
         # Try to be reasonably smart about not re-indenting pasted input more
         # than necessary.  We do this by trimming out the auto-indent initial
         # spaces, if the user's actual input started itself with whitespace.
         if self.autoindent:
             if num_ini_spaces(line) > self.indent_current_nsp:
                 line = line[self.indent_current_nsp:]
                 self.indent_current_nsp = 0
 
         return line
 
     #-------------------------------------------------------------------------
     # Methods to support auto-editing of SyntaxErrors.
     #-------------------------------------------------------------------------
 
     def edit_syntax_error(self):
         """The bottom half of the syntax error handler called in the main loop.
 
         Loop until syntax error is fixed or user cancels.
         """
 
         while self.SyntaxTB.last_syntax_error:
             # copy and clear last_syntax_error
             err = self.SyntaxTB.clear_err_state()
             if not self._should_recompile(err):
                 return
             try:
                 # may set last_syntax_error again if a SyntaxError is raised
                 self.safe_execfile(err.filename,self.user_ns)
             except:
                 self.showtraceback()
             else:
                 try:
                     f = open(err.filename)
                     try:
                         # This should be inside a display_trap block and I
                         # think it is.
                         sys.displayhook(f.read())
                     finally:
                         f.close()
                 except:
                     self.showtraceback()
 
     def _should_recompile(self,e):
         """Utility routine for edit_syntax_error"""
 
         if e.filename in ('<ipython console>','<input>','<string>',
                           '<console>','<BackgroundJob compilation>',
                           None):
 
             return False
         try:
             if (self.autoedit_syntax and
                 not self.ask_yes_no('Return to editor to correct syntax error? '
                               '[Y/n] ','y')):
                 return False
         except EOFError:
             return False
 
         def int0(x):
             try:
                 return int(x)
             except TypeError:
                 return 0
         # always pass integer line and offset values to editor hook
         try:
             self.hooks.fix_error_editor(e.filename,
                 int0(e.lineno),int0(e.offset),e.msg)
         except TryNext:
             warn('Could not open editor')
             return False
         return True
 
     #-------------------------------------------------------------------------
     # Things related to exiting
     #-------------------------------------------------------------------------
 
     def ask_exit(self):
         """ Ask the shell to exit. Can be overiden and used as a callback. """
         self.exit_now = True
 
     def exit(self):
         """Handle interactive exit.
 
         This method calls the ask_exit callback."""
         if self.confirm_exit:
             if self.ask_yes_no('Do you really want to exit ([y]/n)?','y'):
                 self.ask_exit()
         else:
             self.ask_exit()
 
     #-------------------------------------------------------------------------
     # Things related to magics
     #-------------------------------------------------------------------------
 
     def init_magics(self):
         super(TerminalInteractiveShell, self).init_magics()
         self.register_magics(TerminalMagics)
 
     def showindentationerror(self):
         super(TerminalInteractiveShell, self).showindentationerror()
         if not self.using_paste_magics:
             print("If you want to paste code into IPython, try the "
                 "%paste and %cpaste magic functions.")
 
 
 InteractiveShellABC.register(TerminalInteractiveShell)