Add support for accessing raw data to inputsplitter.

This is necessary to be able to use inputsplitter systematically
across all frontends, including plain terminal one.

IPython/core/history.py

@@ -61,13 +61,12 b' class HistoryManager(object):'
             histfname = 'history'
         self.hist_file = os.path.join(shell.ipython_dir, histfname)
 
-        # Fill the history zero entry, user counter starts at 1
-        self.input_hist.append('\n')
-        self.input_hist_raw.append('\n')
-
         # Objects related to shadow history management
         self._init_shadow_hist()
     
+        # Fill the history zero entry, user counter starts at 1
+        self.store_inputs('\n', '\n')
+
         # For backwards compatibility, we must put these back in the shell
         # object, until we've removed all direct uses of the history objects in
         # the shell itself.
@@ -154,10 +153,42 b' class HistoryManager(object):'
                 hist[i] = (input_hist[i], output_hist.get(i))
             else:
                 hist[i] = input_hist[i]
-        if len(hist)==0:
+        if not hist:
             raise IndexError('No history for range of indices: %r' % index)
         return hist
 
+    def store_inputs(self, source, source_raw=None):
+        """Store source and raw input in history.
+
+        Parameters
+        ----------
+        source : str
+          Python input.
+
+        source_raw : str, optional
+          If given, this is the raw input without any IPython transformations
+          applied to it.  If not given, ``source`` is used.
+        """
+        if source_raw is None:
+            source_raw = source
+        self.input_hist.append(source)
+        self.input_hist_raw.append(source_raw)
+        self.shadow_hist.add(source)
+
+    def sync_inputs(self):
+        """Ensure raw and translated histories have same length."""
+        if len(self.input_hist) != len (self.input_hist_raw):
+            self.input_hist_raw = InputList(self.input_hist)
+
+
+    def reset(self):
+        """Clear all histories managed by this object."""
+        self.input_hist[:] = []
+        self.input_hist_raw[:] = []
+        self.output_hist.clear()
+        # The directory history can't be completely empty
+        self.dir_hist[:] = [os.getcwd()]
+
 
 def magic_history(self, parameter_s = ''):
     """Print input history (_i<n> variables), with most recent last.

IPython/core/inputsplitter.py

@@ -600,20 +600,23 b' class InputSplitter(object):'
             if line and not line.isspace():
                 self.indent_spaces, self._full_dedent = self._find_indent(line)
 
-    def _store(self, lines):
+    def _store(self, lines, buffer=None, store='source'):
         """Store one or more lines of input.
 
         If input lines are not newline-terminated, a newline is automatically
         appended."""
 
+        if buffer is None:
+            buffer = self._buffer
+            
         if lines.endswith('\n'):
-            self._buffer.append(lines)
+            buffer.append(lines)
         else:
-            self._buffer.append(lines+'\n')
-        self._set_source()
+            buffer.append(lines+'\n')
+        setattr(self, store, self._set_source(buffer))
 
-    def _set_source(self):
-        self.source = ''.join(self._buffer).encode(self.encoding)
+    def _set_source(self, buffer):
+        return ''.join(buffer).encode(self.encoding)
 
 
 #-----------------------------------------------------------------------------
@@ -933,6 +936,32 b' transform_escaped = EscapedTransformer()'
 class IPythonInputSplitter(InputSplitter):
     """An input splitter that recognizes all of IPython's special syntax."""
 
+    # String with raw, untransformed input.
+    source_raw = ''
+
+    # Private attributes
+    
+    # List with lines of raw input accumulated so far.
+    _buffer_raw = None
+
+    def __init__(self, input_mode=None):
+        InputSplitter.__init__(self, input_mode)
+        self._buffer_raw = []
+        
+    def reset(self):
+        """Reset the input buffer and associated state."""
+        InputSplitter.reset(self)
+        self._buffer_raw[:] = []
+        self.source_raw = ''
+
+    def source_raw_reset(self):
+        """Return input and raw source and perform a full reset.
+        """
+        out = self.source
+        out_r = self.source_raw
+        self.reset()
+        return out, out_r
+
     def push(self, lines):
         """Push one or more lines of IPython input.
         """
@@ -964,13 +993,18 b' class IPythonInputSplitter(InputSplitter):'
         # by one.  Note that this only matters if the input has more than one
         # line.
         changed_input_mode = False
-        
-        if len(lines_list)>1 and self.input_mode == 'cell':
+
+        if self.input_mode == 'cell':
             self.reset()
             changed_input_mode = True
             saved_input_mode = 'cell'
             self.input_mode = 'line'
 
+        # Store raw source before applying any transformations to it.  Note
+        # that this must be done *after* the reset() call that would otherwise
+        # flush the buffer.
+        self._store(lines, self._buffer_raw, 'source_raw')
+        
         try:
             push = super(IPythonInputSplitter, self).push
             for line in lines_list:
@@ -983,5 +1017,4 b' class IPythonInputSplitter(InputSplitter):'
         finally:
             if changed_input_mode:
                 self.input_mode = saved_input_mode
-        
         return out

IPython/core/interactiveshell.py

@@ -218,7 +218,7 b' class InteractiveShell(Configurable, Magic):'
     plugin_manager = Instance('IPython.core.plugin.PluginManager')
     payload_manager = Instance('IPython.core.payload.PayloadManager')
     history_manager = Instance('IPython.core.history.HistoryManager')
-    
+
     # Private interface
     _post_execute = set()
 
@@ -369,8 +369,9 b' class InteractiveShell(Configurable, Magic):'
         # command compiler
         self.compile = codeop.CommandCompiler()
 
-        # User input buffer
+        # User input buffers
         self.buffer = []
+        self.buffer_raw = []
 
         # Make an empty namespace, which extension writers can rely on both
         # existing and NEVER being used by ipython itself.  This gives them a
@@ -965,30 +966,25 b' class InteractiveShell(Configurable, Magic):'
         # Finally, update the real user's namespace
         self.user_ns.update(ns)
 
-
     def reset(self):
         """Clear all internal namespaces.
 
         Note that this is much more aggressive than %reset, since it clears
         fully all namespaces, as well as all input/output lists.
         """
-        for ns in self.ns_refs_table:
-            ns.clear()
-
-        self.alias_manager.clear_aliases()
-
-        # Clear input and output histories
-        self.input_hist[:] = []
-        self.input_hist_raw[:] = []
-        self.output_hist.clear()
+        # Clear histories
+        self.history_manager.reset()
 
         # Reset counter used to index all histories
         self.execution_count = 0
         
         # Restore the user namespaces to minimal usability
+        for ns in self.ns_refs_table:
+            ns.clear()
         self.init_user_ns()
 
         # Restore the default and user aliases
+        self.alias_manager.clear_aliases()
         self.alias_manager.init_aliases()
 
     def reset_selective(self, regex=None):
@@ -2103,9 +2099,7 b' class InteractiveShell(Configurable, Magic):'
         self.execution_count += 1
 
         # Store raw and processed history
-        self.input_hist_raw.append(cell)
-        self.input_hist.append(ipy_cell)
-
+        self.history_manager.store_inputs(ipy_cell, cell)
 
         # dbg code!!!
         def myapp(self, val):  # dbg
@@ -2194,7 +2188,13 b' class InteractiveShell(Configurable, Magic):'
         # interactive IPython session (via a magic, for example).
         self.resetbuffer()
         lines = lines.splitlines()
-        more = 0
+
+        # Since we will prefilter all lines, store the user's raw input too
+        # before we apply any transformations
+        self.buffer_raw[:] = [ l+'\n' for l in lines]
+        
+        more = False
+        prefilter_lines = self.prefilter_manager.prefilter_lines
         with nested(self.builtin_trap, self.display_trap):
             for line in lines:
                 # skip blank lines so we don't mess up the prompt counter, but
@@ -2202,19 +2202,13 b' class InteractiveShell(Configurable, Magic):'
                 # is true)
             
                 if line or more:
-                    # push to raw history, so hist line numbers stay in sync
-                    self.input_hist_raw.append(line + '\n')
-                    prefiltered = self.prefilter_manager.prefilter_lines(line,
-                                                                         more)
-                    more = self.push_line(prefiltered)
+                    more = self.push_line(prefilter_lines(line, more))
                     # IPython's runsource returns None if there was an error
                     # compiling the code.  This allows us to stop processing
                     # right away, so the user gets the error message at the
                     # right place.
                     if more is None:
                         break
-                else:
-                    self.input_hist_raw.append("\n")
             # final newline in case the input didn't have it, so that the code
             # actually does get executed
             if more:
@@ -2370,8 +2364,11 b' class InteractiveShell(Configurable, Magic):'
         for subline in line.splitlines():
             self._autoindent_update(subline)
         self.buffer.append(line)
-        more = self.runsource('\n'.join(self.buffer), self.filename)
+        full_source = '\n'.join(self.buffer)
+        more = self.runsource(full_source, self.filename)
         if not more:
+            self.history_manager.store_inputs('\n'.join(self.buffer_raw),
+                                              full_source)
             self.resetbuffer()
             self.execution_count += 1
         return more
@@ -2379,6 +2376,7 b' class InteractiveShell(Configurable, Magic):'
     def resetbuffer(self):
         """Reset the input buffer."""
         self.buffer[:] = []
+        self.buffer_raw[:] = []
 
     def _is_secondary_block_start(self, s):
         if not s.endswith(':'):

IPython/core/logger.py

@@ -194,7 +194,8 b' which already exists. But you must first start the logging process with'
         # add blank lines if the input cache fell out of sync.
         if out_cache.do_full_cache and \
             out_cache.prompt_count +1 > len(input_hist):
-            input_hist.extend(['\n'] * (out_cache.prompt_count - len(input_hist)))
+            pass
+            #input_hist.extend(['\n'] * (out_cache.prompt_count - len(input_hist)))
             
         if not continuation and line_mod:
             self._iii = self._ii
@@ -203,7 +204,7 b' which already exists. But you must first start the logging process with'
             # put back the final \n of every input line
             self._i00 = line_mod+'\n'
             #print 'Logging input:<%s>' % line_mod  # dbg
-            input_hist.append(self._i00)
+            #input_hist.append(self._i00)
         #print '---[%s]' % (len(input_hist)-1,) # dbg
 
         # hackish access to top-level namespace to create _i1,_i2... dynamically
@@ -222,7 +223,7 b' which already exists. But you must first start the logging process with'
             new_i = '_i%s' % in_num
             if continuation:
                 self._i00 = '%s%s\n' % (self.shell.user_ns[new_i],line_mod)
-                input_hist[in_num] = self._i00
+                #input_hist[in_num] = self._i00
             to_main[new_i] = self._i00
         self.shell.user_ns.update(to_main)
 

IPython/core/prefilter.py

@@ -428,7 +428,7 b' class PrefilterManager(Configurable):'
         which is the case when the user goes back to a multiline history
         entry and presses enter.
         """
-        llines = lines.rstrip('\n').split('\n')
+        llines = lines.rstrip('\n').splitlines()
         # We can get multiple lines in one shot, where multiline input 'blends'
         # into one line, in cases like recalling from the readline history
         # buffer.  We need to make sure that in such cases, we correctly

IPython/core/tests/test_inputsplitter.py

@@ -563,6 +563,8 b' class IPythonInputTestCase(InputSplitterTestCase):'
 
     In addition, this runs the tests over the syntax and syntax_ml dicts that
     were tested by individual functions, as part of the OO interface.
+
+    It also makes some checks on the raw buffer storage.
     """
 
     def setUp(self):
@@ -577,21 +579,26 b' class IPythonInputTestCase(InputSplitterTestCase):'
                     continue
                 
                 isp.push(raw)
-                out = isp.source_reset().rstrip()
-                self.assertEqual(out, out_t)
+                out, out_raw = isp.source_raw_reset()
+                self.assertEqual(out.rstrip(), out_t)
+                self.assertEqual(out_raw.rstrip(), raw.rstrip())
 
     def test_syntax_multiline(self):
         isp = self.isp
         for example in syntax_ml.itervalues():
             out_t_parts = []
+            raw_parts = []
             for line_pairs in example:
-                for raw, out_t_part in line_pairs:
-                    isp.push(raw)
+                for lraw, out_t_part in line_pairs:
+                    isp.push(lraw)
                     out_t_parts.append(out_t_part)
+                    raw_parts.append(lraw)
 
-                out = isp.source_reset().rstrip()
+                out, out_raw = isp.source_raw_reset()
                 out_t = '\n'.join(out_t_parts).rstrip()
-                self.assertEqual(out, out_t)
+                raw = '\n'.join(raw_parts).rstrip()
+                self.assertEqual(out.rstrip(), out_t)
+                self.assertEqual(out_raw.rstrip(), raw)
                 
 
 class BlockIPythonInputTestCase(IPythonInputTestCase):
@@ -616,9 +623,10 b' class BlockIPythonInputTestCase(IPythonInputTestCase):'
                 out_t = '\n'.join(out_t_parts)
 
                 isp.push(raw)
-                out = isp.source_reset()
+                out, out_raw = isp.source_raw_reset()
                 # Match ignoring trailing whitespace
                 self.assertEqual(out.rstrip(), out_t.rstrip())
+                self.assertEqual(out_raw.rstrip(), raw.rstrip())
     
 
 #-----------------------------------------------------------------------------
@@ -652,7 +660,8 b" if __name__ == '__main__':"
             # Here we just return input so we can use it in a test suite, but a
             # real interpreter would instead send it for execution somewhere.
             #src = isp.source; raise EOFError # dbg
-            src = isp.source_reset()
+            src, raw = isp.source_raw_reset()
             print 'Input source was:\n', src
+            print 'Raw source was:\n', raw
     except EOFError:
         print 'Bye'

IPython/frontend/terminal/interactiveshell.py

@@ -191,8 +191,7 b' class TerminalInteractiveShell(InteractiveShell):'
 
             # if you run stuff with -c <cmd>, raw hist is not updated
             # ensure that it's in sync
-            if len(self.input_hist) != len (self.input_hist_raw):
-                self.input_hist_raw = InputList(self.input_hist)
+            self.history_manager.sync_inputs()
 
             while 1:
                 try:
@@ -218,7 +217,7 b' class TerminalInteractiveShell(InteractiveShell):'
         if display_banner:
             self.show_banner()
 
-        more = 0
+        more = False
         
         # Mark activity in the builtins
         __builtin__.__dict__['__IPYTHON__active'] += 1
@@ -231,7 +230,7 b' class TerminalInteractiveShell(InteractiveShell):'
         # Before showing any prompts, if the counter is at zero, we execute an
         # empty line to ensure the user only sees prompts starting at one.
         if self.execution_count == 0:
-            self.push_line('\n')
+            self.execution_count += 1
         
         while not self.exit_now:
             self.hooks.pre_prompt_hook()
@@ -249,7 +248,7 b' class TerminalInteractiveShell(InteractiveShell):'
                 except:
                     self.showtraceback()
             try:
-                line = self.raw_input(prompt, more)
+                line = self.raw_input(prompt)
                 if self.exit_now:
                     # quick exit on sys.std[in|out] close
                     break
@@ -266,7 +265,7 b' class TerminalInteractiveShell(InteractiveShell):'
     
                     if self.autoindent:
                         self.indent_current_nsp = 0
-                    more = 0
+                    more = False
                 except KeyboardInterrupt:
                     pass
             except EOFError:
@@ -286,18 +285,22 b' class TerminalInteractiveShell(InteractiveShell):'
                 # asynchronously by signal handlers, for example.
                 self.showtraceback()
             else:
-                more = self.push_line(line)
+                #more = self.push_line(line)
+                self.input_splitter.push(line)
+                more = self.input_splitter.push_accepts_more()
                 if (self.SyntaxTB.last_syntax_error and
                     self.autoedit_syntax):
                     self.edit_syntax_error()
-
+                if not more:
+                    pass
+                
         # We are off again...
         __builtin__.__dict__['__IPYTHON__active'] -= 1
 
         # Turn off the exit flag, so the mainloop can be restarted if desired
         self.exit_now = False
 
-    def raw_input(self,prompt='',continue_prompt=False):
+    def raw_input(self, prompt='', continue_prompt=False):
         """Write a prompt and read a line.
 
         The returned line does not include the trailing newline.
@@ -310,8 +313,6 b' class TerminalInteractiveShell(InteractiveShell):'
           - continue_prompt(False): whether this line is the first one or a
           continuation in a sequence of inputs.
         """
-        # growl.notify("raw_input: ", "prompt = %r\ncontinue_prompt = %s" % (prompt, continue_prompt))
-
         # Code run by the user may have modified the readline completer state.
         # We must ensure that our completer is back in place.
 
@@ -329,8 +330,6 b' class TerminalInteractiveShell(InteractiveShell):'
         # Try to be reasonably smart about not re-indenting pasted input more
         # than necessary.  We do this by trimming out the auto-indent initial
         # spaces, if the user's actual input started itself with whitespace.
-        #debugx('self.buffer[-1]')
-
         if self.autoindent:
             if num_ini_spaces(line) > self.indent_current_nsp:
                 line = line[self.indent_current_nsp:]
@@ -340,22 +339,15 b' class TerminalInteractiveShell(InteractiveShell):'
         # it.
         if line.strip():
             if continue_prompt:
-                self.input_hist_raw[-1] += '%s\n' % line
                 if self.has_readline and self.readline_use:
-                    try:
-                        histlen = self.readline.get_current_history_length()
-                        if histlen > 1:
-                            newhist = self.input_hist_raw[-1].rstrip()
-                            self.readline.remove_history_item(histlen-1)
-                            self.readline.replace_history_item(histlen-2,
-                                            newhist.encode(self.stdin_encoding))
-                    except AttributeError:
-                        pass # re{move,place}_history_item are new in 2.4.                
+                    histlen = self.readline.get_current_history_length()
+                    if histlen > 1:
+                        newhist = self.input_hist_raw[-1].rstrip()
+                        self.readline.remove_history_item(histlen-1)
+                        self.readline.replace_history_item(histlen-2,
+                                        newhist.encode(self.stdin_encoding))
             else:
                 self.input_hist_raw.append('%s\n' % line)                
-            # only entries starting at first column go to shadow history
-            if line.lstrip() == line:
-                self.shadowhist.add(line.strip())
         elif not continue_prompt:
             self.input_hist_raw.append('\n')
         try:
@@ -368,6 +360,45 b' class TerminalInteractiveShell(InteractiveShell):'
         else:
             return lineout
 
+
+    def raw_input(self, prompt=''):
+        """Write a prompt and read a line.
+
+        The returned line does not include the trailing newline.
+        When the user enters the EOF key sequence, EOFError is raised.
+
+        Optional inputs:
+
+          - prompt(''): a string to be printed to prompt the user.
+
+          - continue_prompt(False): whether this line is the first one or a
+          continuation in a sequence of inputs.
+        """
+        # Code run by the user may have modified the readline completer state.
+        # We must ensure that our completer is back in place.
+
+        if self.has_readline:
+            self.set_readline_completer()
+        
+        try:
+            line = raw_input_original(prompt).decode(self.stdin_encoding)
+        except ValueError:
+            warn("\n********\nYou or a %run:ed script called sys.stdin.close()"
+                 " or sys.stdout.close()!\nExiting IPython!")
+            self.ask_exit()
+            return ""
+
+        # Try to be reasonably smart about not re-indenting pasted input more
+        # than necessary.  We do this by trimming out the auto-indent initial
+        # spaces, if the user's actual input started itself with whitespace.
+        if self.autoindent:
+            if num_ini_spaces(line) > self.indent_current_nsp:
+                line = line[self.indent_current_nsp:]
+                self.indent_current_nsp = 0
+        
+        return line
+
+
     # TODO: The following three methods are an early attempt to refactor
     # the main code execution logic. We don't use them, but they may be
     # helpful when we refactor the code execution logic further.