Merge branch 'execution-refactor' of http://github.com/fperez/ipython into fperez-execution-refactor

IPython/core/displayhook.py

@@ -54,8 +54,9 b' class DisplayHook(Configurable):'
     """
 
     shell = Instance('IPython.core.interactiveshell.InteractiveShellABC')
+
     # Each call to the In[] prompt raises it by 1, even the first.
-    prompt_count = Int(0)
+    #prompt_count = Int(0)
 
     def __init__(self, shell=None, cache_size=1000,
                  colors='NoColor', input_sep='\n',
@@ -114,6 +115,10 b' class DisplayHook(Configurable):'
         to_user_ns = {'_':self._,'__':self.__,'___':self.___}
         self.shell.user_ns.update(to_user_ns)
 
+    @property
+    def prompt_count(self):
+        return self.shell.execution_count
+
     def _set_prompt_str(self,p_str,cache_def,no_cache_def):
         if p_str is None:
             if self.do_full_cache:
@@ -249,7 +254,7 b' class DisplayHook(Configurable):'
     def log_output(self, result):
         """Log the output."""
         if self.shell.logger.log_output:
-            self.shell.logger.log_write(repr(result),'output')
+            self.shell.logger.log_write(repr(result), 'output')
 
     def finish_displayhook(self):
         """Finish up all displayhook activities."""

IPython/core/history.py

@@ -1,14 +1,238 b''
-# -*- coding: utf-8 -*-
 """ History related magics and functionality """
+#-----------------------------------------------------------------------------
+#  Copyright (C) 2010 The IPython Development Team.
+#
+#  Distributed under the terms of the BSD License.
+#
+#  The full license is in the file COPYING.txt, distributed with this software.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Imports
+#-----------------------------------------------------------------------------
+from __future__ import print_function
 
 # Stdlib imports
 import fnmatch
 import os
+import sys
 
+# Our own packages
 import IPython.utils.io
+
+from IPython.core import ipapi
+from IPython.core.inputlist import InputList
+from IPython.utils.pickleshare import PickleShareDB
 from IPython.utils.io import ask_yes_no
 from IPython.utils.warn import warn
-from IPython.core import ipapi
+
+#-----------------------------------------------------------------------------
+# Classes and functions
+#-----------------------------------------------------------------------------
+
+class HistoryManager(object):
+    """A class to organize all history-related functionality in one place.
+    """
+    # Public interface
+
+    # An instance of the IPython shell we are attached to
+    shell = None
+    # An InputList instance to hold processed history
+    input_hist = None
+    # An InputList instance to hold raw history (as typed by user)
+    input_hist_raw = None
+    # A list of directories visited during session
+    dir_hist = None
+    # A dict of output history, keyed with ints from the shell's execution count
+    output_hist = None
+    # String with path to the history file
+    hist_file = None
+    # PickleShareDB instance holding the raw data for the shadow history
+    shadow_db = None
+    # ShadowHist instance with the actual shadow history
+    shadow_hist = None
+    
+    # Private interface
+    # Variables used to store the three last inputs from the user.  On each new
+    # history update, we populate the user's namespace with these, shifted as
+    # necessary.
+    _i00, _i, _ii, _iii = '','','',''
+    
+    def __init__(self, shell):
+        """Create a new history manager associated with a shell instance.
+        """
+        # We need a pointer back to the shell for various tasks.
+        self.shell = shell
+        
+        # List of input with multi-line handling.
+        self.input_hist = InputList()
+        # This one will hold the 'raw' input history, without any
+        # pre-processing.  This will allow users to retrieve the input just as
+        # it was exactly typed in by the user, with %hist -r.
+        self.input_hist_raw = InputList()
+
+        # list of visited directories
+        try:
+            self.dir_hist = [os.getcwd()]
+        except OSError:
+            self.dir_hist = []
+
+        # dict of output history
+        self.output_hist = {}
+
+        # Now the history file
+        if shell.profile:
+            histfname = 'history-%s' % shell.profile
+        else:
+            histfname = 'history'
+        self.hist_file = os.path.join(shell.ipython_dir, histfname)
+
+        # Objects related to shadow history management
+        self._init_shadow_hist()
+    
+        self._i00, self._i, self._ii, self._iii = '','','',''
+
+        # Object is fully initialized, we can now call methods on it.
+        
+        # Fill the history zero entry, user counter starts at 1
+        self.store_inputs('\n', '\n')
+
+        # For backwards compatibility, we must put these back in the shell
+        # object, until we've removed all direct uses of the history objects in
+        # the shell itself.
+        shell.input_hist = self.input_hist
+        shell.input_hist_raw = self.input_hist_raw
+        shell.output_hist = self.output_hist
+        shell.dir_hist = self.dir_hist
+        shell.histfile = self.hist_file
+        shell.shadowhist = self.shadow_hist
+        shell.db = self.shadow_db
+
+    def _init_shadow_hist(self):
+        try:
+            self.shadow_db = PickleShareDB(os.path.join(
+                                           self.shell.ipython_dir, 'db'))
+        except UnicodeDecodeError:
+            print("Your ipython_dir can't be decoded to unicode!")
+            print("Please set HOME environment variable to something that")
+            print(r"only has ASCII characters, e.g. c:\home")
+            print("Now it is", self.ipython_dir)
+            sys.exit()
+        self.shadow_hist = ShadowHist(self.shadow_db)
+        
+    def save_hist(self):
+        """Save input history to a file (via readline library)."""
+
+        try:
+            self.shell.readline.write_history_file(self.hist_file)
+        except:
+            print('Unable to save IPython command history to file: ' + 
+                  `self.hist_file`)
+
+    def reload_hist(self):
+        """Reload the input history from disk file."""
+
+        try:
+            self.shell.readline.clear_history()
+            self.shell.readline.read_history_file(self.hist_file)
+        except AttributeError:
+            pass
+
+    def get_history(self, index=None, raw=False, output=True):
+        """Get the history list.
+
+        Get the input and output history.
+
+        Parameters
+        ----------
+        index : n or (n1, n2) or None
+            If n, then the last entries. If a tuple, then all in
+            range(n1, n2). If None, then all entries. Raises IndexError if
+            the format of index is incorrect.
+        raw : bool
+            If True, return the raw input.
+        output : bool
+            If True, then return the output as well.
+
+        Returns
+        -------
+        If output is True, then return a dict of tuples, keyed by the prompt
+        numbers and with values of (input, output). If output is False, then
+        a dict, keyed by the prompt number with the values of input. Raises
+        IndexError if no history is found.
+        """
+        if raw:
+            input_hist = self.input_hist_raw
+        else:
+            input_hist = self.input_hist
+        if output:
+            output_hist = self.output_hist
+        n = len(input_hist)
+        if index is None:
+            start=0; stop=n
+        elif isinstance(index, int):
+            start=n-index; stop=n
+        elif isinstance(index, tuple) and len(index) == 2:
+            start=index[0]; stop=index[1]
+        else:
+            raise IndexError('Not a valid index for the input history: %r'
+                             % index)
+        hist = {}
+        for i in range(start, stop):
+            if output:
+                hist[i] = (input_hist[i], output_hist.get(i))
+            else:
+                hist[i] = input_hist[i]
+        if not hist:
+            raise IndexError('No history for range of indices: %r' % index)
+        return hist
+
+    def store_inputs(self, source, source_raw=None):
+        """Store source and raw input in history and create input cache
+        variables _i*.
+        
+        Parameters
+        ----------
+        source : str
+          Python input.
+
+        source_raw : str, optional
+          If given, this is the raw input without any IPython transformations
+          applied to it.  If not given, ``source`` is used.
+        """
+        if source_raw is None:
+            source_raw = source
+        self.input_hist.append(source)
+        self.input_hist_raw.append(source_raw)
+        self.shadow_hist.add(source)
+
+        # update the auto _i variables
+        self._iii = self._ii
+        self._ii = self._i
+        self._i = self._i00
+        self._i00 = source_raw
+
+        # hackish access to user namespace to create _i1,_i2... dynamically
+        new_i = '_i%s' % self.shell.execution_count
+        to_main = {'_i': self._i,
+                   '_ii': self._ii,
+                   '_iii': self._iii,
+                   new_i : self._i00 }
+        self.shell.user_ns.update(to_main)
+
+    def sync_inputs(self):
+        """Ensure raw and translated histories have same length."""
+        if len(self.input_hist) != len (self.input_hist_raw):
+            self.input_hist_raw = InputList(self.input_hist)
+
+    def reset(self):
+        """Clear all histories managed by this object."""
+        self.input_hist[:] = []
+        self.input_hist_raw[:] = []
+        self.output_hist.clear()
+        # The directory history can't be completely empty
+        self.dir_hist[:] = [os.getcwd()]
+
 
 def magic_history(self, parameter_s = ''):
     """Print input history (_i<n> variables), with most recent last.
@@ -55,7 +279,7 b" def magic_history(self, parameter_s = ''):"
     """
 
     if not self.shell.displayhook.do_full_cache:
-        print 'This feature is only available if numbered prompts are in use.'
+        print('This feature is only available if numbered prompts are in use.')
         return
     opts,args = self.parse_options(parameter_s,'gnoptsrf:',mode='list')
 
@@ -69,7 +293,7 b" def magic_history(self, parameter_s = ''):"
     else:
         if os.path.exists(outfname):
             if not ask_yes_no("File %r exists. Overwrite?" % outfname): 
-                print 'Aborting.'
+                print('Aborting.')
                 return
 
         outfile = open(outfname,'w')
@@ -103,7 +327,7 b" def magic_history(self, parameter_s = ''):"
         init, final = map(int, args)
     else:
         warn('%hist takes 0, 1 or 2 arguments separated by spaces.')
-        print >> IPython.utils.io.Term.cout, self.magic_hist.__doc__
+        print(self.magic_hist.__doc__, file=IPython.utils.io.Term.cout)
         return
     
     width = len(str(final))
@@ -117,14 +341,14 b" def magic_history(self, parameter_s = ''):"
         sh = self.shell.shadowhist.all()
         for idx, s in sh:
             if fnmatch.fnmatch(s, pattern):
-                print >> outfile, "0%d: %s" %(idx, s.expandtabs(4))
+                print("0%d: %s" %(idx, s.expandtabs(4)), file=outfile)
                 found = True
     
     if found:
-        print >> outfile, "==="
-        print >> outfile, \
-              "shadow history ends, fetch by %rep <number> (must start with 0)"
-        print >> outfile, "=== start of normal history ==="
+        print("===", file=outfile)
+        print("shadow history ends, fetch by %rep <number> (must start with 0)",
+              file=outfile)
+        print("=== start of normal history ===", file=outfile)
         
     for in_num in range(init, final):
         # Print user history with tabs expanded to 4 spaces.  The GUI clients
@@ -137,22 +361,22 b" def magic_history(self, parameter_s = ''):"
             
         multiline = int(inline.count('\n') > 1)
         if print_nums:
-            print >> outfile, \
-                  '%s:%s' % (str(in_num).ljust(width), line_sep[multiline]),
+            print('%s:%s' % (str(in_num).ljust(width), line_sep[multiline]),
+                  file=outfile)
         if pyprompts:
-            print >> outfile, '>>>',
+            print('>>>', file=outfile)
             if multiline:
                 lines = inline.splitlines()
-                print >> outfile, '\n... '.join(lines)
-                print >> outfile, '... '
+                print('\n... '.join(lines), file=outfile)
+                print('... ', file=outfile)
             else:
-                print >> outfile, inline,
+                print(inline, end='', file=outfile)
         else:
-            print >> outfile, inline,
+            print(inline,end='', file=outfile)
         if print_outputs:
             output = self.shell.output_hist.get(in_num)
             if output is not None:
-                print >> outfile, repr(output)
+                print(repr(output), file=outfile)
 
     if close_at_end:
         outfile.close()
@@ -223,10 +447,10 b' def rep_f(self, arg):'
         
     try:
         lines = self.extract_input_slices(args, True)
-        print "lines",lines
-        self.runlines(lines)
+        print("lines", lines)
+        self.run_cell(lines)
     except ValueError:
-        print "Not found in recent history:", args
+        print("Not found in recent history:", args)
         
 
 _sentinel = object()
@@ -251,11 +475,11 b' class ShadowHist(object):'
             if old is not _sentinel:
                 return
             newidx = self.inc_idx()
-            #print "new",newidx # dbg
+            #print("new", newidx) # dbg
             self.db.hset('shadowhist',ent, newidx)
         except:
             ipapi.get().showtraceback()
-            print "WARNING: disabling shadow history"
+            print("WARNING: disabling shadow history")
             self.disabled = True
     
     def all(self):
@@ -268,7 +492,6 b' class ShadowHist(object):'
         all = self.all()
         
         for k, v in all:
-            #print k,v
             if k == idx:
                 return v
 

IPython/core/hooks.py

@@ -53,7 +53,7 b' import IPython.utils.io'
 __all__ = ['editor', 'fix_error_editor', 'synchronize_with_editor',
            'input_prefilter', 'shutdown_hook', 'late_startup_hook',
            'generate_prompt', 'show_in_pager','pre_prompt_hook',
-           'pre_runcode_hook', 'clipboard_get']
+           'pre_run_code_hook', 'clipboard_get']
 
 def editor(self,filename, linenum=None):
     """Open the default editor at the given filename and linenumber.
@@ -238,7 +238,7 b' def pre_prompt_hook(self):'
     return None
 
 
-def pre_runcode_hook(self):
+def pre_run_code_hook(self):
     """ Executed before running the (prefiltered) code in IPython """
     return None
 

IPython/core/inputsplitter.py

@@ -371,15 +371,6 b' class InputSplitter(object):'
         if self.input_mode == 'cell':
             self.reset()
         
-        # If the source code has leading blanks, add 'if 1:\n' to it
-        # this allows execution of indented pasted code. It is tempting
-        # to add '\n' at the end of source to run commands like ' a=1'
-        # directly, but this fails for more complicated scenarios
-
-        if not self._buffer and lines[:1] in [' ', '\t'] and \
-           not comment_line_re.match(lines):
-            lines = 'if 1:\n%s' % lines
-        
         self._store(lines)
         source = self.source
 
@@ -594,26 +585,29 b' class InputSplitter(object):'
             #print 'safety' # dbg
             
         return indent_spaces, full_dedent
-        
+    
     def _update_indent(self, lines):
         for line in remove_comments(lines).splitlines():
             if line and not line.isspace():
                 self.indent_spaces, self._full_dedent = self._find_indent(line)
 
-    def _store(self, lines):
+    def _store(self, lines, buffer=None, store='source'):
         """Store one or more lines of input.
 
         If input lines are not newline-terminated, a newline is automatically
         appended."""
 
+        if buffer is None:
+            buffer = self._buffer
+            
         if lines.endswith('\n'):
-            self._buffer.append(lines)
+            buffer.append(lines)
         else:
-            self._buffer.append(lines+'\n')
-        self._set_source()
+            buffer.append(lines+'\n')
+        setattr(self, store, self._set_source(buffer))
 
-    def _set_source(self):
-        self.source = ''.join(self._buffer).encode(self.encoding)
+    def _set_source(self, buffer):
+        return ''.join(buffer).encode(self.encoding)
 
 
 #-----------------------------------------------------------------------------
@@ -932,6 +926,32 b' transform_escaped = EscapedTransformer()'
 class IPythonInputSplitter(InputSplitter):
     """An input splitter that recognizes all of IPython's special syntax."""
 
+    # String with raw, untransformed input.
+    source_raw = ''
+
+    # Private attributes
+    
+    # List with lines of raw input accumulated so far.
+    _buffer_raw = None
+
+    def __init__(self, input_mode=None):
+        InputSplitter.__init__(self, input_mode)
+        self._buffer_raw = []
+        
+    def reset(self):
+        """Reset the input buffer and associated state."""
+        InputSplitter.reset(self)
+        self._buffer_raw[:] = []
+        self.source_raw = ''
+
+    def source_raw_reset(self):
+        """Return input and raw source and perform a full reset.
+        """
+        out = self.source
+        out_r = self.source_raw
+        self.reset()
+        return out, out_r
+
     def push(self, lines):
         """Push one or more lines of IPython input.
         """
@@ -963,13 +983,18 b' class IPythonInputSplitter(InputSplitter):'
         # by one.  Note that this only matters if the input has more than one
         # line.
         changed_input_mode = False
-        
-        if len(lines_list)>1 and self.input_mode == 'cell':
+
+        if self.input_mode == 'cell':
             self.reset()
             changed_input_mode = True
             saved_input_mode = 'cell'
             self.input_mode = 'line'
 
+        # Store raw source before applying any transformations to it.  Note
+        # that this must be done *after* the reset() call that would otherwise
+        # flush the buffer.
+        self._store(lines, self._buffer_raw, 'source_raw')
+        
         try:
             push = super(IPythonInputSplitter, self).push
             for line in lines_list:
@@ -982,5 +1007,4 b' class IPythonInputSplitter(InputSplitter):'
         finally:
             if changed_input_mode:
                 self.input_mode = saved_input_mode
-        
         return out

IPython/core/logger.py

@@ -1,6 +1,4 b''
-# -*- coding: utf-8 -*-
-"""
-Logger class for IPython's logging facilities.
+"""Logger class for IPython's logging facilities.
 """
 
 #*****************************************************************************
@@ -26,13 +24,12 b' import time'
 class Logger(object):
     """A Logfile class with different policies for file creation"""
 
-    def __init__(self,shell,logfname='Logger.log',loghead='',logmode='over'):
-
-        self._i00,self._i,self._ii,self._iii = '','','',''
+    def __init__(self, home_dir, logfname='Logger.log', loghead='',
+                 logmode='over'):
 
         # this is the full ipython instance, we need some attributes from it
         # which won't exist until later. What a mess, clean up later...
-        self.shell = shell
+        self.home_dir = home_dir
 
         self.logfname = logfname
         self.loghead = loghead
@@ -102,7 +99,7 b' class Logger(object):'
             self.logfile = open(self.logfname,'w')
 
         elif logmode == 'global':
-            self.logfname = os.path.join(self.shell.home_dir,self.logfname)
+            self.logfname = os.path.join(self.home_dir,self.logfname)
             self.logfile = open(self.logfname, 'a')
 
         elif logmode == 'over':
@@ -166,63 +163,18 b' which already exists. But you must first start the logging process with'
             print 'Timestamping   :',self.timestamp
             print 'State          :',state
 
-    def log(self,line_ori,line_mod,continuation=None):
-        """Write the line to a log and create input cache variables _i*.
+    def log(self, line_mod, line_ori):
+        """Write the sources to a log.
 
         Inputs:
 
-        - line_ori: unmodified input line from the user.  This is not
-        necessarily valid Python.
-
         - line_mod: possibly modified input, such as the transformations made
         by input prefilters or input handlers of various kinds.  This should
         always be valid Python.
 
-        - continuation: if True, indicates this is part of multi-line input."""
-
-        # update the auto _i tables
-        #print '***logging line',line_mod # dbg
-        #print '***cache_count', self.shell.displayhook.prompt_count # dbg
-        try:
-            input_hist = self.shell.user_ns['_ih']
-        except:
-            #print 'userns:',self.shell.user_ns.keys()  # dbg
-            return
-        
-        out_cache = self.shell.displayhook
-
-        # add blank lines if the input cache fell out of sync.
-        if out_cache.do_full_cache and \
-            out_cache.prompt_count +1 > len(input_hist):
-            input_hist.extend(['\n'] * (out_cache.prompt_count - len(input_hist)))
-            
-        if not continuation and line_mod:
-            self._iii = self._ii
-            self._ii = self._i
-            self._i = self._i00
-            # put back the final \n of every input line
-            self._i00 = line_mod+'\n'
-            #print 'Logging input:<%s>' % line_mod  # dbg
-            input_hist.append(self._i00)
-        #print '---[%s]' % (len(input_hist)-1,) # dbg
-
-        # hackish access to top-level namespace to create _i1,_i2... dynamically
-        to_main = {'_i':self._i,'_ii':self._ii,'_iii':self._iii}
-        if self.shell.displayhook.do_full_cache:
-            in_num = self.shell.displayhook.prompt_count
-
-            # but if the opposite is true (a macro can produce multiple inputs
-            # with no output display called), then bring the output counter in
-            # sync:
-            last_num = len(input_hist)-1
-            if in_num != last_num:
-                in_num = self.shell.displayhook.prompt_count = last_num
-            new_i = '_i%s' % in_num
-            if continuation:
-                self._i00 = '%s%s\n' % (self.shell.user_ns[new_i],line_mod)
-                input_hist[in_num] = self._i00
-            to_main[new_i] = self._i00
-        self.shell.user_ns.update(to_main)
+        - line_ori: unmodified input line from the user.  This is not
+        necessarily valid Python.
+        """
 
         # Write the log line, but decide which one according to the
         # log_raw_input flag, set when the log is started.
@@ -231,7 +183,7 b' which already exists. But you must first start the logging process with'
         else:
             self.log_write(line_mod)
 
-    def log_write(self,data,kind='input'):
+    def log_write(self, data, kind='input'):
         """Write data to the log file, if active"""
 
         #print 'data: %r' % data # dbg
@@ -241,10 +193,10 b' which already exists. But you must first start the logging process with'
                 if self.timestamp:
                     write(time.strftime('# %a, %d %b %Y %H:%M:%S\n',
                                         time.localtime()))
-                write('%s\n' % data)
+                write(data)
             elif kind=='output' and self.log_output:
                 odata = '\n'.join(['#[Out]# %s' % s
-                                   for s in data.split('\n')])
+                                   for s in data.splitlines()])
                 write('%s\n' % odata)
             self.logfile.flush()
 

IPython/core/macro.py

@@ -20,9 +20,7 b' class Macro(IPyAutocall):'
     """
 
     def __init__(self,data):
-
-        # store the macro value, as a single string which can be evaluated by
-        # runlines()
+        # store the macro value, as a single string which can be executed
         self.value = ''.join(data).rstrip()+'\n'
         
     def __str__(self):
@@ -34,7 +32,7 b' class Macro(IPyAutocall):'
     def __call__(self,*args):
         IPython.utils.io.Term.cout.flush()
         self._ip.user_ns['_margv'] = args
-        self._ip.runlines(self.value)
+        self._ip.run_cell(self.value)
     
     def __getstate__(self):
         """ needed for safe pickling via %store """

IPython/core/magic.py

@@ -41,11 +41,6 b' except ImportError:'
     except ImportError:
         profile = pstats = None
 
-# print_function was added to __future__ in Python2.6, remove this when we drop
-# 2.5 compatibility
-if not hasattr(__future__,'CO_FUTURE_PRINT_FUNCTION'):
-    __future__.CO_FUTURE_PRINT_FUNCTION = 65536
-
 import IPython
 from IPython.core import debugger, oinspect
 from IPython.core.error import TryNext
@@ -1556,7 +1551,7 b' Currently the magic system has the following functions:\\n"""'
         
         stats = None
         try:
-            self.shell.savehist()
+            self.shell.save_hist()
 
             if opts.has_key('p'):
                 stats = self.magic_prun('',0,opts,arg_lst,prog_ns)
@@ -1675,7 +1670,7 b' Currently the magic system has the following functions:\\n"""'
                 # contained therein.
                 del sys.modules[main_mod_name]
 
-            self.shell.reloadhist()
+            self.shell.reload_hist()
                 
         return stats
 
@@ -2331,7 +2326,7 b' Currently the magic system has the following functions:\\n"""'
         else:
             print 'done. Executing edited code...'
             if opts_r:
-                self.shell.runlines(file_read(filename))
+                self.shell.run_cell(file_read(filename))
             else:
                 self.shell.safe_execfile(filename,self.shell.user_ns,
                                          self.shell.user_ns)
@@ -2992,7 +2987,7 b' Defaulting color scheme to \'NoColor\'"""'
                    (input.startswith(start) or input.startswith(start_magic)):
                 #print 'match',`input`  # dbg
                 print 'Executing:',input,
-                self.shell.runlines(input)
+                self.shell.run_cell(input)
                 return
         print 'No previous input matching `%s` found.' % start
 

IPython/core/prefilter.py

@@ -373,10 +373,6 b' class PrefilterManager(Configurable):'
         # print "prefilter_line: ", line, continue_prompt
         # All handlers *must* return a value, even if it's blank ('').
 
-        # Lines are NOT logged here. Handlers should process the line as
-        # needed, update the cache AND log it (so that the input cache array
-        # stays synced).
-
         # save the line away in case we crash, so the post-mortem handler can
         # record it
         self.shell._last_input_line = line
@@ -792,7 +788,6 b' class PrefilterHandler(Configurable):'
             ):
             line = ''
 
-        self.shell.log(line, line, continue_prompt)
         return line
 
     def __str__(self):
@@ -811,7 +806,6 b' class AliasHandler(PrefilterHandler):'
         line_out = '%sget_ipython().system(%s)' % (line_info.pre_whitespace,
                                          make_quoted_expr(transformed))
         
-        self.shell.log(line_info.line, line_out, line_info.continue_prompt)
         return line_out
 
 
@@ -840,8 +834,6 b' class ShellEscapeHandler(PrefilterHandler):'
             cmd = line.lstrip().lstrip(ESC_SHELL)
             line_out = '%sget_ipython().system(%s)' % (line_info.pre_whitespace,
                                              make_quoted_expr(cmd))
-        # update cache/log and return
-        self.shell.log(line, line_out, line_info.continue_prompt)
         return line_out
 
 
@@ -856,7 +848,6 b' class MagicHandler(PrefilterHandler):'
         the_rest = line_info.the_rest
         cmd = '%sget_ipython().magic(%s)' % (line_info.pre_whitespace,
                                    make_quoted_expr(ifun + " " + the_rest))
-        self.shell.log(line_info.line, cmd, line_info.continue_prompt)
         return cmd
 
 
@@ -877,7 +868,6 b' class AutoHandler(PrefilterHandler):'
 
         # This should only be active for single-line input!
         if continue_prompt:
-            self.shell.log(line,line,continue_prompt)
             return line
 
         force_auto = isinstance(obj, IPyAutocall)
@@ -918,9 +908,6 b' class AutoHandler(PrefilterHandler):'
         if auto_rewrite:
             self.shell.auto_rewrite_input(newcmd)
             
-        # log what is now valid Python, not the actual user input (without the
-        # final newline)
-        self.shell.log(line,newcmd,continue_prompt)
         return newcmd
 
 
@@ -947,7 +934,6 b' class HelpHandler(PrefilterHandler):'
                 line = line[1:]
             elif line[-1]==ESC_HELP:
                 line = line[:-1]
-            self.shell.log(line, '#?'+line, line_info.continue_prompt)
             if line:
                 #print 'line:<%r>' % line  # dbg
                 self.shell.magic_pinfo(line)

IPython/core/prompts.py

@@ -372,15 +372,7 b' class Prompt1(BasePrompt):'
         self.col_p_ni = self.col_p.replace('\001','').replace('\002','') 
         self.col_norm_ni = Colors.normal        
 
-    def peek_next_prompt(self):
-        """Get the next prompt, but don't increment the counter."""
-        self.cache.prompt_count += 1
-        next_prompt = str_safe(self.p_str)
-        self.cache.prompt_count -= 1
-        return next_prompt
-
     def __str__(self):
-        self.cache.prompt_count += 1
         self.cache.last_prompt = str_safe(self.p_str_nocolor).split('\n')[-1]
         return str_safe(self.p_str)
 

IPython/core/tests/test_inputsplitter.py

@@ -162,6 +162,12 b' class InputSplitterTestCase(unittest.TestCase):'
         self.assertEqual(isp.indent_spaces, 4)
         isp.push('y=2\n')
         self.assertEqual(isp.indent_spaces, 0)
+
+    def test_indent2(self):
+        # In cell mode, inputs must be fed in whole blocks, so skip this test
+        if self.isp.input_mode == 'cell': return
+
+        isp = self.isp
         isp.push('if 1:')
         self.assertEqual(isp.indent_spaces, 4)
         isp.push('    x=1')
@@ -170,7 +176,10 b' class InputSplitterTestCase(unittest.TestCase):'
         isp.push(' '*2)
         self.assertEqual(isp.indent_spaces, 4)
 
-    def test_indent2(self):
+    def test_indent3(self):
+        # In cell mode, inputs must be fed in whole blocks, so skip this test
+        if self.isp.input_mode == 'cell': return
+
         isp = self.isp
         # When a multiline statement contains parens or multiline strings, we
         # shouldn't get confused.
@@ -195,13 +204,6 b' class InputSplitterTestCase(unittest.TestCase):'
         for line in ['  x=1', '# a comment', '  y=2']:
             self.assertTrue(isp.push(line))
             
-    def test_push3(self):
-        """Test input with leading whitespace"""
-        isp = self.isp
-        isp.push('  x=1')
-        isp.push('  y=2')
-        self.assertEqual(isp.source, 'if 1:\n  x=1\n  y=2\n')
-
     def test_replace_mode(self):
         isp = self.isp
         isp.input_mode = 'cell'
@@ -216,6 +218,9 b' class InputSplitterTestCase(unittest.TestCase):'
         self.assertFalse(isp.push_accepts_more())
 
     def test_push_accepts_more2(self):
+        # In cell mode, inputs must be fed in whole blocks, so skip this test
+        if self.isp.input_mode == 'cell': return
+
         isp = self.isp
         isp.push('if 1:')
         self.assertTrue(isp.push_accepts_more())
@@ -230,6 +235,9 b' class InputSplitterTestCase(unittest.TestCase):'
         self.assertFalse(isp.push_accepts_more())
 
     def test_push_accepts_more4(self):
+        # In cell mode, inputs must be fed in whole blocks, so skip this test
+        if self.isp.input_mode == 'cell': return
+
         isp = self.isp
         # When a multiline statement contains parens or multiline strings, we
         # shouldn't get confused.
@@ -563,6 +571,8 b' class IPythonInputTestCase(InputSplitterTestCase):'
 
     In addition, this runs the tests over the syntax and syntax_ml dicts that
     were tested by individual functions, as part of the OO interface.
+
+    It also makes some checks on the raw buffer storage.
     """
 
     def setUp(self):
@@ -577,21 +587,26 b' class IPythonInputTestCase(InputSplitterTestCase):'
                     continue
                 
                 isp.push(raw)
-                out = isp.source_reset().rstrip()
-                self.assertEqual(out, out_t)
+                out, out_raw = isp.source_raw_reset()
+                self.assertEqual(out.rstrip(), out_t)
+                self.assertEqual(out_raw.rstrip(), raw.rstrip())
 
     def test_syntax_multiline(self):
         isp = self.isp
         for example in syntax_ml.itervalues():
             out_t_parts = []
+            raw_parts = []
             for line_pairs in example:
-                for raw, out_t_part in line_pairs:
-                    isp.push(raw)
+                for lraw, out_t_part in line_pairs:
+                    isp.push(lraw)
                     out_t_parts.append(out_t_part)
+                    raw_parts.append(lraw)
 
-                out = isp.source_reset().rstrip()
+                out, out_raw = isp.source_raw_reset()
                 out_t = '\n'.join(out_t_parts).rstrip()
-                self.assertEqual(out, out_t)
+                raw = '\n'.join(raw_parts).rstrip()
+                self.assertEqual(out.rstrip(), out_t)
+                self.assertEqual(out_raw.rstrip(), raw)
                 
 
 class BlockIPythonInputTestCase(IPythonInputTestCase):
@@ -616,9 +631,10 b' class BlockIPythonInputTestCase(IPythonInputTestCase):'
                 out_t = '\n'.join(out_t_parts)
 
                 isp.push(raw)
-                out = isp.source_reset()
+                out, out_raw = isp.source_raw_reset()
                 # Match ignoring trailing whitespace
                 self.assertEqual(out.rstrip(), out_t.rstrip())
+                self.assertEqual(out_raw.rstrip(), raw.rstrip())
     
 
 #-----------------------------------------------------------------------------
@@ -652,7 +668,8 b" if __name__ == '__main__':"
             # Here we just return input so we can use it in a test suite, but a
             # real interpreter would instead send it for execution somewhere.
             #src = isp.source; raise EOFError # dbg
-            src = isp.source_reset()
+            src, raw = isp.source_raw_reset()
             print 'Input source was:\n', src
+            print 'Raw source was:\n', raw
     except EOFError:
         print 'Bye'

IPython/extensions/parallelmagic.py

@@ -137,29 +137,30 b' class ParalleMagic(Plugin):'
             self._enable_autopx()
 
     def _enable_autopx(self):
-        """Enable %autopx mode by saving the original runsource and installing 
-        pxrunsource.
+        """Enable %autopx mode by saving the original run_source and installing 
+        pxrun_source.
         """
         if self.active_multiengine_client is None:
             print NO_ACTIVE_MULTIENGINE_CLIENT
             return
 
-        self._original_runsource = self.shell.runsource
-        self.shell.runsource = new.instancemethod(
-            self.pxrunsource, self.shell, self.shell.__class__
+        self._original_run_source = self.shell.run_source
+        self.shell.run_source = new.instancemethod(
+            self.pxrun_source, self.shell, self.shell.__class__
         )
         self.autopx = True
         print "%autopx enabled"
     
     def _disable_autopx(self):
-        """Disable %autopx by restoring the original InteractiveShell.runsource."""
+        """Disable %autopx by restoring the original InteractiveShell.run_source.
+        """
         if self.autopx:
-            self.shell.runsource = self._original_runsource
+            self.shell.run_source = self._original_run_source
             self.autopx = False
             print "%autopx disabled"
 
-    def pxrunsource(self, ipself, source, filename="<input>", symbol="single"):
-        """A parallel replacement for InteractiveShell.runsource."""
+    def pxrun_source(self, ipself, source, filename="<input>", symbol="single"):
+        """A parallel replacement for InteractiveShell.run_source."""
 
         try:
             code = ipself.compile(source, filename, symbol)

IPython/frontend/terminal/interactiveshell.py

@@ -191,8 +191,7 b' class TerminalInteractiveShell(InteractiveShell):'
 
             # if you run stuff with -c <cmd>, raw hist is not updated
             # ensure that it's in sync
-            if len(self.input_hist) != len (self.input_hist_raw):
-                self.input_hist_raw = InputList(self.input_hist)
+            self.history_manager.sync_inputs()
 
             while 1:
                 try:
@@ -218,7 +217,7 b' class TerminalInteractiveShell(InteractiveShell):'
         if display_banner:
             self.show_banner()
 
-        more = 0
+        more = False
         
         # Mark activity in the builtins
         __builtin__.__dict__['__IPYTHON__active'] += 1
@@ -227,7 +226,7 b' class TerminalInteractiveShell(InteractiveShell):'
             self.readline_startup_hook(self.pre_readline)
         # exit_now is set by a call to %Exit or %Quit, through the
         # ask_exit callback.
-        
+
         while not self.exit_now:
             self.hooks.pre_prompt_hook()
             if more:
@@ -244,7 +243,7 b' class TerminalInteractiveShell(InteractiveShell):'
                 except:
                     self.showtraceback()
             try:
-                line = self.raw_input(prompt, more)
+                line = self.raw_input(prompt)
                 if self.exit_now:
                     # quick exit on sys.std[in|out] close
                     break
@@ -256,12 +255,7 b' class TerminalInteractiveShell(InteractiveShell):'
                 try:
                     self.write('\nKeyboardInterrupt\n')
                     self.resetbuffer()
-                    # keep cache in sync with the prompt counter:
-                    self.displayhook.prompt_count -= 1
-    
-                    if self.autoindent:
-                        self.indent_current_nsp = 0
-                    more = 0
+                    more = False
                 except KeyboardInterrupt:
                     pass
             except EOFError:
@@ -281,18 +275,22 b' class TerminalInteractiveShell(InteractiveShell):'
                 # asynchronously by signal handlers, for example.
                 self.showtraceback()
             else:
-                more = self.push_line(line)
+                self.input_splitter.push(line)
+                more = self.input_splitter.push_accepts_more()
                 if (self.SyntaxTB.last_syntax_error and
                     self.autoedit_syntax):
                     self.edit_syntax_error()
-
+                if not more:
+                    source_raw = self.input_splitter.source_raw_reset()[1]
+                    self.run_cell(source_raw)
+                
         # We are off again...
         __builtin__.__dict__['__IPYTHON__active'] -= 1
 
         # Turn off the exit flag, so the mainloop can be restarted if desired
         self.exit_now = False
 
-    def raw_input(self,prompt='',continue_prompt=False):
+    def raw_input(self, prompt='', continue_prompt=False):
         """Write a prompt and read a line.
 
         The returned line does not include the trailing newline.
@@ -305,8 +303,6 b' class TerminalInteractiveShell(InteractiveShell):'
           - continue_prompt(False): whether this line is the first one or a
           continuation in a sequence of inputs.
         """
-        # growl.notify("raw_input: ", "prompt = %r\ncontinue_prompt = %s" % (prompt, continue_prompt))
-
         # Code run by the user may have modified the readline completer state.
         # We must ensure that our completer is back in place.
 
@@ -324,8 +320,6 b' class TerminalInteractiveShell(InteractiveShell):'
         # Try to be reasonably smart about not re-indenting pasted input more
         # than necessary.  We do this by trimming out the auto-indent initial
         # spaces, if the user's actual input started itself with whitespace.
-        #debugx('self.buffer[-1]')
-
         if self.autoindent:
             if num_ini_spaces(line) > self.indent_current_nsp:
                 line = line[self.indent_current_nsp:]
@@ -335,22 +329,15 b' class TerminalInteractiveShell(InteractiveShell):'
         # it.
         if line.strip():
             if continue_prompt:
-                self.input_hist_raw[-1] += '%s\n' % line
                 if self.has_readline and self.readline_use:
-                    try:
-                        histlen = self.readline.get_current_history_length()
-                        if histlen > 1:
-                            newhist = self.input_hist_raw[-1].rstrip()
-                            self.readline.remove_history_item(histlen-1)
-                            self.readline.replace_history_item(histlen-2,
-                                            newhist.encode(self.stdin_encoding))
-                    except AttributeError:
-                        pass # re{move,place}_history_item are new in 2.4.                
+                    histlen = self.readline.get_current_history_length()
+                    if histlen > 1:
+                        newhist = self.input_hist_raw[-1].rstrip()
+                        self.readline.remove_history_item(histlen-1)
+                        self.readline.replace_history_item(histlen-2,
+                                        newhist.encode(self.stdin_encoding))
             else:
                 self.input_hist_raw.append('%s\n' % line)                
-            # only entries starting at first column go to shadow history
-            if line.lstrip() == line:
-                self.shadowhist.add(line.strip())
         elif not continue_prompt:
             self.input_hist_raw.append('\n')
         try:
@@ -363,67 +350,43 b' class TerminalInteractiveShell(InteractiveShell):'
         else:
             return lineout
 
-    # TODO: The following three methods are an early attempt to refactor
-    # the main code execution logic. We don't use them, but they may be
-    # helpful when we refactor the code execution logic further.
-    # def interact_prompt(self):
-    #     """ Print the prompt (in read-eval-print loop) 
-    # 
-    #     Provided for those who want to implement their own read-eval-print loop (e.g. GUIs), not 
-    #     used in standard IPython flow.
-    #     """
-    #     if self.more:
-    #         try:
-    #             prompt = self.hooks.generate_prompt(True)
-    #         except:
-    #             self.showtraceback()
-    #         if self.autoindent:
-    #             self.rl_do_indent = True
-    # 
-    #     else:
-    #         try:
-    #             prompt = self.hooks.generate_prompt(False)
-    #         except:
-    #             self.showtraceback()
-    #     self.write(prompt)
-    # 
-    # def interact_handle_input(self,line):
-    #     """ Handle the input line (in read-eval-print loop)
-    #     
-    #     Provided for those who want to implement their own read-eval-print loop (e.g. GUIs), not 
-    #     used in standard IPython flow.        
-    #     """
-    #     if line.lstrip() == line:
-    #         self.shadowhist.add(line.strip())
-    #     lineout = self.prefilter_manager.prefilter_lines(line,self.more)
-    # 
-    #     if line.strip():
-    #         if self.more:
-    #             self.input_hist_raw[-1] += '%s\n' % line
-    #         else:
-    #             self.input_hist_raw.append('%s\n' % line)                
-    # 
-    #     
-    #     self.more = self.push_line(lineout)
-    #     if (self.SyntaxTB.last_syntax_error and
-    #         self.autoedit_syntax):
-    #         self.edit_syntax_error()
-    # 
-    # def interact_with_readline(self):
-    #     """ Demo of using interact_handle_input, interact_prompt
-    #     
-    #     This is the main read-eval-print loop. If you need to implement your own (e.g. for GUI),
-    #     it should work like this.
-    #     """ 
-    #     self.readline_startup_hook(self.pre_readline)
-    #     while not self.exit_now:
-    #         self.interact_prompt()
-    #         if self.more:
-    #             self.rl_do_indent = True
-    #         else:
-    #             self.rl_do_indent = False
-    #         line = raw_input_original().decode(self.stdin_encoding)
-    #         self.interact_handle_input(line)
+
+    def raw_input(self, prompt=''):
+        """Write a prompt and read a line.
+
+        The returned line does not include the trailing newline.
+        When the user enters the EOF key sequence, EOFError is raised.
+
+        Optional inputs:
+
+          - prompt(''): a string to be printed to prompt the user.
+
+          - continue_prompt(False): whether this line is the first one or a
+          continuation in a sequence of inputs.
+        """
+        # Code run by the user may have modified the readline completer state.
+        # We must ensure that our completer is back in place.
+
+        if self.has_readline:
+            self.set_readline_completer()
+        
+        try:
+            line = raw_input_original(prompt).decode(self.stdin_encoding)
+        except ValueError:
+            warn("\n********\nYou or a %run:ed script called sys.stdin.close()"
+                 " or sys.stdout.close()!\nExiting IPython!")
+            self.ask_exit()
+            return ""
+
+        # Try to be reasonably smart about not re-indenting pasted input more
+        # than necessary.  We do this by trimming out the auto-indent initial
+        # spaces, if the user's actual input started itself with whitespace.
+        if self.autoindent:
+            if num_ini_spaces(line) > self.indent_current_nsp:
+                line = line[self.indent_current_nsp:]
+                self.indent_current_nsp = 0
+        
+        return line
 
     #-------------------------------------------------------------------------
     # Methods to support auto-editing of SyntaxErrors.

IPython/frontend/terminal/ipapp.py

@@ -571,7 +571,7 b' class IPythonApp(Application):'
                     try:
                         self.log.info("Running code in user namespace: %s" %
                                       line)
-                        self.shell.runlines(line)
+                        self.shell.run_cell(line)
                     except:
                         self.log.warn("Error in executing line in user "
                                       "namespace: %s" % line)
@@ -616,7 +616,7 b' class IPythonApp(Application):'
             try:
                 self.log.info("Running code given at command line (-c): %s" %
                               line)
-                self.shell.runlines(line)
+                self.shell.run_cell(line)
             except:
                 self.log.warn("Error in executing line in user namespace: %s" %
                               line)

IPython/lib/demo.py

@@ -248,7 +248,7 b' class Demo(object):'
         self.ip_ns       = ip.user_ns
         self.ip_colorize = ip.pycolorize
         self.ip_showtb   = ip.showtraceback
-        self.ip_runlines = ip.runlines
+        self.ip_run_cell = ip.run_cell
         self.shell       = ip
 
         # load user data and initialize data structures
@@ -411,7 +411,7 b' class Demo(object):'
             print >>IPython.utils.io.Term.cout, block,
         sys.stdout.flush()
 
-    def runlines(self,source):
+    def run_cell(self,source):
         """Execute a string with one or more lines of code"""
 
         exec source in self.user_ns
@@ -449,7 +449,7 b' class Demo(object):'
             try:
                 save_argv = sys.argv
                 sys.argv = self.sys_argv
-                self.runlines(next_block)
+                self.run_cell(next_block)
                 self.post_cmd()
             finally:
                 sys.argv = save_argv
@@ -496,10 +496,10 b' class IPythonDemo(Demo):'
     class requires the input to be valid, pure Python code.
     """
 
-    def runlines(self,source):
+    def run_cell(self,source):
         """Execute a string with one or more lines of code"""
 
-        self.shell.runlines(source)
+        self.shell.run_cell(source)
         
 class LineDemo(Demo):
     """Demo where each line is executed as a separate block.

IPython/zmq/ipkernel.py

@@ -209,19 +209,13 b' class Kernel(Configurable):'
         reply_content = {}
         try:
             if silent:
-                # runcode uses 'exec' mode, so no displayhook will fire, and it
+                # run_code uses 'exec' mode, so no displayhook will fire, and it
                 # doesn't call logging or history manipulations.  Print
                 # statements in that code will obviously still execute.
-                shell.runcode(code)
+                shell.run_code(code)
             else:
-                # FIXME: runlines calls the exception handler itself.
+                # FIXME: the shell calls the exception handler itself.
                 shell._reply_content = None
-
-                # For now leave this here until we're sure we can stop using it
-                #shell.runlines(code)
-
-                # Experimental: cell mode!  Test more before turning into
-                # default and removing the hacks around runlines.
                 shell.run_cell(code)
         except:
             status = u'error'
@@ -238,8 +232,9 b' class Kernel(Configurable):'
             status = u'ok'
 
         reply_content[u'status'] = status
-        # Compute the execution counter so clients can display prompts
-        reply_content['execution_count'] = shell.displayhook.prompt_count
+        
+        # Return the execution counter so clients can display prompts
+        reply_content['execution_count'] = shell.execution_count -1
 
         # FIXME - fish exception info out of shell, possibly left there by
         # runlines.  We'll need to clean up this logic later.