Merge branch 'sqlite-history' of github.com:takluyver/ipython

IPython/config/default/ipython_config.py

@@ -148,3 +148,14 b' c = get_config()'
 # c.AliasManager.user_aliases = [
 #     ('foo', 'echo Hi')
 # ]
+
+#-----------------------------------------------------------------------------
+# HistoryManager options
+#-----------------------------------------------------------------------------
+
+# Enable logging output as well as input to the database.
+# c.HistoryManager.db_log_output = False
+
+# Only write to the database every n commands - this can save disk
+# access (and hence power) over the default of writing on every command.
+# c.HistoryManager.db_cache_size = 0

IPython/core/displayhook.py

@@ -277,10 +277,13 b' class DisplayHook(Configurable):'
                 self.shell.user_ns.update(to_main)
                 self.shell.user_ns['_oh'][self.prompt_count] = result
 
-    def log_output(self, result):
+    def log_output(self, format_dict):
         """Log the output."""
         if self.shell.logger.log_output:
-            self.shell.logger.log_write(repr(result), 'output')
+            self.shell.logger.log_write(format_dict['text/plain'], 'output')
+        # This is a defaultdict of lists, so we can always append
+        self.shell.history_manager.output_hist_reprs[self.prompt_count]\
+                                    .append(format_dict['text/plain'])
 
     def finish_displayhook(self):
         """Finish up all displayhook activities."""
@@ -300,7 +303,7 b' class DisplayHook(Configurable):'
             format_dict = self.compute_format_data(result)
             self.write_format_data(format_dict)
             self.update_user_ns(result)
-            self.log_output(result)
+            self.log_output(format_dict)
             self.finish_displayhook()
 
     def flush(self):

IPython/core/history.py

@@ -13,47 +13,61 b''
 from __future__ import print_function
 
 # Stdlib imports
-import atexit
-import fnmatch
+import datetime
 import json
 import os
-import sys
-import threading
-import time
+import re
+import sqlite3
+
+from collections import defaultdict
 
 # Our own packages
+from IPython.config.configurable import Configurable
 import IPython.utils.io
 
 from IPython.testing import decorators as testdec
-from IPython.utils.pickleshare import PickleShareDB
 from IPython.utils.io import ask_yes_no
+from IPython.utils.traitlets import Bool, Dict, Instance, Int, List, Unicode
 from IPython.utils.warn import warn
 
 #-----------------------------------------------------------------------------
 # Classes and functions
 #-----------------------------------------------------------------------------
 
-class HistoryManager(object):
+class HistoryManager(Configurable):
     """A class to organize all history-related functionality in one place.
     """
     # Public interface
 
     # An instance of the IPython shell we are attached to
-    shell = None
-    # A list to hold processed history
-    input_hist_parsed = None
-    # A list to hold raw history (as typed by user)
-    input_hist_raw = None
+    shell = Instance('IPython.core.interactiveshell.InteractiveShellABC')
+    # Lists to hold processed and raw history. These start with a blank entry
+    # so that we can index them starting from 1
+    input_hist_parsed = List([""])
+    input_hist_raw = List([""])
     # A list of directories visited during session
-    dir_hist = None
-    # A dict of output history, keyed with ints from the shell's execution count
-    output_hist = None
-    # String with path to the history file
-    hist_file = None
-    # PickleShareDB instance holding the raw data for the shadow history
-    shadow_db = None
-    # ShadowHist instance with the actual shadow history
-    shadow_hist = None
+    dir_hist = List()
+    # A dict of output history, keyed with ints from the shell's
+    # execution count. If there are several outputs from one command,
+    # only the last one is stored.
+    output_hist = Dict()
+    # Contains all outputs, in lists of reprs.
+    output_hist_reprs = Instance(defaultdict)
+    
+    # String holding the path to the history file
+    hist_file = Unicode()
+    # The SQLite database
+    db = Instance(sqlite3.Connection)
+    # The number of the current session in the history database
+    session_number = Int()
+    # Should we log output to the database? (default no)
+    db_log_output = Bool(False, config=True)
+    # Write to database every x commands (higher values save disk access & power)
+    #  Values of 1 or less effectively disable caching. 
+    db_cache_size = Int(0, config=True)
+    # The input and output caches
+    db_input_cache = List()
+    db_output_cache = List()
     
     # Private interface
     # Variables used to store the three last inputs from the user.  On each new
@@ -66,18 +80,11 b' class HistoryManager(object):'
     # call).
     _exit_commands = None
     
-    def __init__(self, shell):
+    def __init__(self, shell, config=None):
         """Create a new history manager associated with a shell instance.
         """
         # We need a pointer back to the shell for various tasks.
-        self.shell = shell
-        
-        # List of input with multi-line handling.
-        self.input_hist_parsed = []
-        # This one will hold the 'raw' input history, without any
-        # pre-processing.  This will allow users to retrieve the input just as
-        # it was exactly typed in by the user, with %hist -r.
-        self.input_hist_raw = []
+        super(HistoryManager, self).__init__(shell=shell, config=config)
 
         # list of visited directories
         try:
@@ -85,149 +92,260 b' class HistoryManager(object):'
         except OSError:
             self.dir_hist = []
 
-        # dict of output history
-        self.output_hist = {}
-
         # Now the history file
         if shell.profile:
             histfname = 'history-%s' % shell.profile
         else:
             histfname = 'history'
-        self.hist_file = os.path.join(shell.ipython_dir, histfname + '.json')
-
-        # Objects related to shadow history management
-        self._init_shadow_hist()
+        self.hist_file = os.path.join(shell.ipython_dir, histfname + '.sqlite')
+        try:
+            self.init_db()
+        except sqlite3.DatabaseError:
+            newpath = os.path.join(self.shell.ipython_dir, "hist-corrupt.sqlite")
+            os.rename(self.hist_file, newpath)
+            print("ERROR! History file wasn't a valid SQLite database.",
+            "It was moved to %s" % newpath, "and a new file created.")
+            self.init_db()
+        
+        self.new_session()
     
         self._i00, self._i, self._ii, self._iii = '','','',''
+        self.output_hist_reprs = defaultdict(list)
 
         self._exit_commands = set(['Quit', 'quit', 'Exit', 'exit', '%Quit',
                                    '%quit', '%Exit', '%exit'])
-
-        # Object is fully initialized, we can now call methods on it.
         
-        # Fill the history zero entry, user counter starts at 1
-        self.store_inputs('\n', '\n')
+    def init_db(self):
+        """Connect to the database, and create tables if necessary."""
+        self.db = sqlite3.connect(self.hist_file)
+        self.db.execute("""CREATE TABLE IF NOT EXISTS sessions (session integer
+                        primary key autoincrement, start timestamp,
+                        end timestamp, num_cmds integer, remark text)""")
+        self.db.execute("""CREATE TABLE IF NOT EXISTS history 
+                (session integer, line integer, source text, source_raw text,
+                PRIMARY KEY (session, line))""")
+        # Output history is optional, but ensure the table's there so it can be
+        # enabled later.
+        self.db.execute("""CREATE TABLE IF NOT EXISTS output_history
+                        (session integer, line integer, output text,
+                        PRIMARY KEY (session, line))""")
+        self.db.commit()
+    
+    def new_session(self):
+        """Get a new session number."""
+        with self.db:
+            cur = self.db.execute("""INSERT INTO sessions VALUES (NULL, ?, NULL,
+                            NULL, "") """, (datetime.datetime.now(),))
+            self.session_number = cur.lastrowid
+            
+    def end_session(self):
+        """Close the database session, filling in the end time and line count."""
+        self.writeout_cache()
+        with self.db:
+            self.db.execute("""UPDATE sessions SET end=?, num_cmds=? WHERE
+                            session==?""", (datetime.datetime.now(),
+                            len(self.input_hist_parsed)-1, self.session_number))
+        self.session_number = 0
+                            
+    def name_session(self, name):
+        """Give the current session a name in the history database."""
+        with self.db:
+            self.db.execute("UPDATE sessions SET remark=? WHERE session==?",
+                            (name, self.session_number))
+                            
+    def reset(self, new_session=True):
+        """Clear the session history, releasing all object references, and
+        optionally open a new session."""
+        if self.session_number:
+            self.end_session()
+        self.input_hist_parsed[:] = [""]
+        self.input_hist_raw[:] = [""]
+        self.output_hist.clear()
+        # The directory history can't be completely empty
+        self.dir_hist[:] = [os.getcwd()]
         
-        # Create and start the autosaver.
-        self.autosave_flag = threading.Event()
-        self.autosave_timer = HistorySaveThread(self.autosave_flag, 60)
-        self.autosave_timer.start()
-        # Register the autosave handler to be triggered as a post execute
-        # callback.
-        self.shell.register_post_execute(self.autosave_if_due)
-
-    def _init_shadow_hist(self):
-        try:
-            self.shadow_db = PickleShareDB(os.path.join(
-                                           self.shell.ipython_dir, 'db'))
-        except UnicodeDecodeError:
-            print("Your ipython_dir can't be decoded to unicode!")
-            print("Please set HOME environment variable to something that")
-            print(r"only has ASCII characters, e.g. c:\home")
-            print("Now it is", self.ipython_dir)
-            sys.exit()
-        self.shadow_hist = ShadowHist(self.shadow_db, self.shell)
+        if new_session:
+            self.new_session()
+    
+    ## -------------------------------
+    ## Methods for retrieving history:
+    ## -------------------------------
+    def _run_sql(self, sql, params, raw=True, output=False):
+        """Prepares and runs an SQL query for the history database.
         
-    def populate_readline_history(self):
-        """Populate the readline history from the raw history.
-
-        We only store one copy of the raw history, which is persisted to a json
-        file on disk.  The readline history is repopulated from the contents of
-        this file."""
-
-        try:
-            self.shell.readline.clear_history()
-        except AttributeError:
-            pass
-        else:
-            for h in self.input_hist_raw:
-                if not h.isspace():
-                    for line in h.splitlines():
-                        self.shell.readline.add_history(line)
-
-    def save_history(self):
-        """Save input history to a file (via readline library)."""
-        hist = dict(raw=self.input_hist_raw, #[-self.shell.history_length:],
-                    parsed=self.input_hist_parsed) #[-self.shell.history_length:])
-        with open(self.hist_file,'wt') as hfile:
-            json.dump(hist, hfile,
-                      sort_keys=True, indent=4)
-                      
-    def autosave_if_due(self):
-        """Check if the autosave event is set; if so, save history. We do it 
-        this way so that the save takes place in the main thread."""
-        if self.autosave_flag.is_set():
-            self.save_history()
-            self.autosave_flag.clear()
+        Parameters
+        ----------
+        sql : str
+          Any filtering expressions to go after SELECT ... FROM ...
+        params : tuple
+          Parameters passed to the SQL query (to replace "?")
+        raw, output : bool
+          See :meth:`get_range`
         
-    def reload_history(self):
-        """Reload the input history from disk file."""
-
-        with open(self.hist_file,'rt') as hfile:
-            try:
-                hist = json.load(hfile)
-            except ValueError:    # Ignore it if JSON is corrupt.
-                return
-            self.input_hist_parsed = hist['parsed']
-            self.input_hist_raw = hist['raw']
-            if self.shell.has_readline:
-                self.populate_readline_history()
+        Returns
+        -------
+        Tuples as :meth:`get_range`
+        """
+        toget = 'source_raw' if raw else 'source'
+        sqlfrom = "history"
+        if output:
+            sqlfrom = "history LEFT JOIN output_history USING (session, line)"
+            toget = "history.%s, output_history.output" % toget
+        cur = self.db.execute("SELECT session, line, %s FROM %s " %\
+                                (toget, sqlfrom) + sql, params)
+        if output:    # Regroup into 3-tuples, and parse JSON
+            loads = lambda out: json.loads(out) if out else None
+            return ((ses, lin, (inp, loads(out))) \
+                                        for ses, lin, inp, out in cur)
+        return cur
+        
+    
+    def get_tail(self, n=10, raw=True, output=False, include_latest=False):
+        """Get the last n lines from the history database.
         
-    def get_history(self, index=None, raw=False, output=True):
-        """Get the history list.
-
-        Get the input and output history.
-
         Parameters
         ----------
-        index : n or (n1, n2) or None
-            If n, then the last entries. If a tuple, then all in
-            range(n1, n2). If None, then all entries. Raises IndexError if
-            the format of index is incorrect.
-        raw : bool
-            If True, return the raw input.
-        output : bool
-            If True, then return the output as well.
-
+        n : int
+          The number of lines to get
+        raw, output : bool
+          See :meth:`get_range`
+        include_latest : bool
+          If False (default), n+1 lines are fetched, and the latest one
+          is discarded. This is intended to be used where the function
+          is called by a user command, which it should not return.
+        
         Returns
         -------
-        If output is True, then return a dict of tuples, keyed by the prompt
-        numbers and with values of (input, output). If output is False, then
-        a dict, keyed by the prompt number with the values of input. Raises
-        IndexError if no history is found.
+        Tuples as :meth:`get_range`
         """
-        if raw:
-            input_hist = self.input_hist_raw
-        else:
-            input_hist = self.input_hist_parsed
+        self.writeout_cache()
+        if not include_latest:
+            n += 1
+        cur = self._run_sql("ORDER BY session DESC, line DESC LIMIT ?",
+                                (n,), raw=raw, output=output)
+        if not include_latest:
+            return reversed(list(cur)[1:])
+        return reversed(list(cur))
+        
+    def search(self, pattern="*", raw=True, search_raw=True,
+                                                        output=False):
+        """Search the database using unix glob-style matching (wildcards
+        * and ?).
+        
+        Parameters
+        ----------
+        pattern : str
+          The wildcarded pattern to match when searching
+        search_raw : bool
+          If True, search the raw input, otherwise, the parsed input
+        raw, output : bool
+          See :meth:`get_range`
+        
+        Returns
+        -------
+        Tuples as :meth:`get_range`
+        """
+        tosearch = "source_raw" if search_raw else "source"
         if output:
-            output_hist = self.output_hist
+            tosearch = "history." + tosearch
+        self.writeout_cache()
+        return self._run_sql("WHERE %s GLOB ?" % tosearch, (pattern,),
+                                    raw=raw, output=output)
+                                
+    def _get_range_session(self, start=1, stop=None, raw=True, output=False):
+        """Get input and output history from the current session. Called by
+        get_range, and takes similar parameters."""
+        input_hist = self.input_hist_raw if raw else self.input_hist_parsed
+            
         n = len(input_hist)
-        if index is None:
-            start=0; stop=n
-        elif isinstance(index, int):
-            start=n-index; stop=n
-        elif isinstance(index, tuple) and len(index) == 2:
-            start=index[0]; stop=index[1]
-        else:
-            raise IndexError('Not a valid index for the input history: %r'
-                             % index)
-        hist = {}
+        if start < 0:
+            start += n
+        if not stop:
+            stop = n
+        elif stop < 0:
+            stop += n
+        
         for i in range(start, stop):
             if output:
-                hist[i] = (input_hist[i], output_hist.get(i))
+                line = (input_hist[i], self.output_hist_reprs.get(i))
             else:
-                hist[i] = input_hist[i]
-        if not hist:
-            raise IndexError('No history for range of indices: %r' % index)
-        return hist
-
-    def store_inputs(self, source, source_raw=None):
+                line = input_hist[i]
+            yield (0, i, line)
+            
+    def get_range(self, session=0, start=1, stop=None, raw=True,output=False):
+        """Retrieve input by session.
+        
+        Parameters
+        ----------
+        session : int
+            Session number to retrieve. The current session is 0, and negative
+            numbers count back from current session, so -1 is previous session.
+        start : int
+            First line to retrieve.
+        stop : int
+            End of line range (excluded from output itself). If None, retrieve
+            to the end of the session.
+        raw : bool
+            If True, return untranslated input
+        output : bool
+            If True, attempt to include output. This will be 'real' Python
+            objects for the current session, or text reprs from previous
+            sessions if db_log_output was enabled at the time. Where no output
+            is found, None is used.
+            
+        Returns
+        -------
+        An iterator over the desired lines. Each line is a 3-tuple, either
+        (session, line, input) if output is False, or
+        (session, line, (input, output)) if output is True.
+        """
+        if session == 0 or session==self.session_number:   # Current session
+            return self._get_range_session(start, stop, raw, output)
+        if session < 0:
+            session += self.session_number
+            
+        if stop:
+            lineclause = "line >= ? AND line < ?"
+            params = (session, start, stop)
+        else:
+            lineclause = "line>=?"
+            params = (session, start)
+        
+        return self._run_sql("WHERE session==? AND %s""" % lineclause,
+                                    params, raw=raw, output=output)
+        
+    def get_range_by_str(self, rangestr, raw=True, output=False):
+        """Get lines of history from a string of ranges, as used by magic
+        commands %hist, %save, %macro, etc.
+        
+        Parameters
+        ----------
+        rangestr : str
+          A string specifying ranges, e.g. "5 ~2/1-4". See
+          :func:`magic_history` for full details.
+        raw, output : bool
+          As :meth:`get_range`
+          
+        Returns
+        -------
+        Tuples as :meth:`get_range`
+        """
+        for sess, s, e in extract_hist_ranges(rangestr):
+            for line in self.get_range(sess, s, e, raw=raw, output=output):
+                yield line
+    
+    ## ----------------------------
+    ## Methods for storing history:
+    ## ----------------------------
+    def store_inputs(self, line_num, source, source_raw=None):
         """Store source and raw input in history and create input cache
         variables _i*.
         
         Parameters
         ----------
+        line_num : int
+          The prompt number of this input.
+        
         source : str
           Python input.
 
@@ -237,14 +355,20 b' class HistoryManager(object):'
         """
         if source_raw is None:
             source_raw = source
+        source = source.rstrip('\n')
+        source_raw = source_raw.rstrip('\n')
             
         # do not store exit/quit commands
         if source_raw.strip() in self._exit_commands:
             return
         
-        self.input_hist_parsed.append(source.rstrip())
-        self.input_hist_raw.append(source_raw.rstrip())
-        self.shadow_hist.add(source)
+        self.input_hist_parsed.append(source)
+        self.input_hist_raw.append(source_raw)
+        
+        self.db_input_cache.append((line_num, source, source_raw))
+        # Trigger to flush cache and write to DB.
+        if len(self.db_input_cache) >= self.db_cache_size:
+            self.writeout_cache()
 
         # update the auto _i variables
         self._iii = self._ii
@@ -253,58 +377,114 b' class HistoryManager(object):'
         self._i00 = source_raw
 
         # hackish access to user namespace to create _i1,_i2... dynamically
-        new_i = '_i%s' % self.shell.execution_count
+        new_i = '_i%s' % line_num
         to_main = {'_i': self._i,
                    '_ii': self._ii,
                    '_iii': self._iii,
                    new_i : self._i00 }
         self.shell.user_ns.update(to_main)
-
-    def sync_inputs(self):
-        """Ensure raw and translated histories have same length."""
-        if len(self.input_hist_parsed) != len (self.input_hist_raw):
-            self.input_hist_raw[:] = self.input_hist_parsed
-
-    def reset(self):
-        """Clear all histories managed by this object."""
-        self.input_hist_parsed[:] = []
-        self.input_hist_raw[:] = []
-        self.output_hist.clear()
-        # The directory history can't be completely empty
-        self.dir_hist[:] = [os.getcwd()]
-
-class HistorySaveThread(threading.Thread):
-    """This thread makes IPython save history periodically.
-
-    Without this class, IPython would only save the history on a clean exit.
-    This saves the history periodically (the current default is once per
-    minute), so that it is not lost in the event of a crash.
+        
+    def store_output(self, line_num):
+        """If database output logging is enabled, this saves all the
+        outputs from the indicated prompt number to the database. It's
+        called by run_cell after code has been executed.
+        
+        Parameters
+        ----------
+        line_num : int
+          The line number from which to save outputs
+        """
+        if (not self.db_log_output) or not self.output_hist_reprs[line_num]:
+            return
+        output = json.dumps(self.output_hist_reprs[line_num])
+        
+        self.db_output_cache.append((line_num, output))
+        if self.db_cache_size <= 1:
+            self.writeout_cache()
+        
+    def _writeout_input_cache(self):
+        for line in self.db_input_cache:
+            with self.db:
+                self.db.execute("INSERT INTO history VALUES (?, ?, ?, ?)",
+                                (self.session_number,)+line)
     
-    The implementation sets an event to indicate that history should be saved.
-    The actual save is carried out after executing a user command, to avoid
-    thread issues.
-    """
-    daemon = True
+    def _writeout_output_cache(self):
+        for line in self.db_output_cache:
+            with self.db:
+                self.db.execute("INSERT INTO output_history VALUES (?, ?, ?)",
+                                (self.session_number,)+line)
     
-    def __init__(self, autosave_flag, time_interval=60):
-        threading.Thread.__init__(self)
-        self.time_interval = time_interval
-        self.autosave_flag = autosave_flag
-        self.exit_now = threading.Event()
-        # Ensure the thread is stopped tidily when exiting normally
-        atexit.register(self.stop)
-
-    def run(self):
-        while True:
-            self.exit_now.wait(self.time_interval)
-            if self.exit_now.is_set():
-                break
-            self.autosave_flag.set()
+    def writeout_cache(self):
+        """Write any entries in the cache to the database."""
+        try:
+            self._writeout_input_cache()
+        except sqlite3.IntegrityError:
+            self.new_session()
+            print("ERROR! Session/line number was not unique in",
+                  "database. History logging moved to new session",
+                                            self.session_number)
+            try: # Try writing to the new session. If this fails, don't recurse
+                self.writeout_cache()
+            except sqlite3.IntegrityError:
+                pass
+        finally:
+            self.db_input_cache = []
             
-    def stop(self):
-        """Safely and quickly stop the autosave timer thread."""
-        self.exit_now.set()
-        self.join()
+        try:
+            self._writeout_output_cache()
+        except sqlite3.IntegrityError:
+            print("!! Session/line number for output was not unique",
+                  "in database. Output will not be stored.")
+        finally:
+            self.db_output_cache = []
+
+        
+# To match, e.g. ~5/8-~2/3
+range_re = re.compile(r"""
+((?P<startsess>~?\d+)/)?
+(?P<start>\d+)                    # Only the start line num is compulsory
+((?P<sep>[\-:])
+ ((?P<endsess>~?\d+)/)?
+ (?P<end>\d+))?
+""", re.VERBOSE)
+
+def extract_hist_ranges(ranges_str):
+    """Turn a string of history ranges into 3-tuples of (session, start, stop).
+    
+    Examples
+    --------
+    list(extract_input_ranges("~8/5-~7/4 2"))
+    [(-8, 5, None), (-7, 1, 4), (0, 2, 3)]
+    """
+    for range_str in ranges_str.split():
+        rmatch = range_re.match(range_str)
+        if not rmatch:
+            continue
+        start = int(rmatch.group("start"))
+        end = rmatch.group("end")
+        end = int(end) if end else start+1   # If no end specified, get (a, a+1)
+        if rmatch.group("sep") == "-":       # 1-3 == 1:4 --> [1, 2, 3]
+            end += 1
+        startsess = rmatch.group("startsess") or "0"
+        endsess = rmatch.group("endsess") or startsess
+        startsess = int(startsess.replace("~","-"))
+        endsess = int(endsess.replace("~","-"))
+        assert endsess >= startsess
+
+        if endsess == startsess:
+            yield (startsess, start, end)
+            continue
+        # Multiple sessions in one range:
+        yield (startsess, start, None)
+        for sess in range(startsess+1, endsess):
+            yield (sess, 1, None)
+        yield (endsess, 1, end)
+
+def _format_lineno(session, line):
+    """Helper function to format line numbers properly."""
+    if session == 0:
+        return str(line)
+    return "%s#%s" % (session, line)
 
 @testdec.skip_doctest
 def magic_history(self, parameter_s = ''):
@@ -315,11 +495,18 b" def magic_history(self, parameter_s = ''):"
     %history n1 n2 -> print inputs between n1 and n2 (n2 not included)\\
 
     By default, input history is printed without line numbers so it can be
-    directly pasted into an editor.
-
-    With -n, each input's number <n> is shown, and is accessible as the
-    automatically generated variable _i<n> as well as In[<n>].  Multi-line
-    statements are printed starting at a new line for easy copy/paste.
+    directly pasted into an editor. Use -n to show them.
+
+    Ranges of history can be indicated using the syntax:
+    4      : Line 4, current session
+    4-6    : Lines 4-6, current session
+    243/1-5: Lines 1-5, session 243
+    ~2/7   : Line 7, session 2 before current
+    ~8/1-~6/5 : From the first line of 8 sessions ago, to the fifth line
+                of 6 sessions ago.
+    Multiple ranges can be entered, separated by spaces
+    
+    The same syntax is used by %macro, %save, %edit, %rerun
 
     Options:
 
@@ -342,9 +529,11 b" def magic_history(self, parameter_s = ''):"
       'get_ipython().magic("%cd /")' instead of '%cd /'.
       
       -g: treat the arg as a pattern to grep for in (full) history.
-      This includes the "shadow history" (almost all commands ever written).
-      Use '%hist -g' to show full shadow history (may be very long).
-      In shadow history, every index nuwber starts with 0.
+      This includes the saved history (almost all commands ever written).
+      Use '%hist -g' to show full saved history (may be very long).
+      
+      -l: get the last n lines from all sessions. Specify n as a single arg, or
+      the default is the last 10 lines.
 
       -f FILENAME: instead of printing the output to the screen, redirect it to
        the given file.  The file is always overwritten, though IPython asks for
@@ -363,7 +552,16 b" def magic_history(self, parameter_s = ''):"
     if not self.shell.displayhook.do_full_cache:
         print('This feature is only available if numbered prompts are in use.')
         return
-    opts,args = self.parse_options(parameter_s,'gnoptsrf:',mode='list')
+    opts,args = self.parse_options(parameter_s,'noprtglf:',mode='string')
+    
+    # For brevity
+    history_manager = self.shell.history_manager
+    
+    def _format_lineno(session, line):
+        """Helper function to format line numbers properly."""
+        if session in (0, history_manager.session_number):
+            return str(line)
+        return "%s/%s" % (session, line)
 
     # Check if output to specific file was requested.
     try:
@@ -380,96 +578,61 b" def magic_history(self, parameter_s = ''):"
 
         outfile = open(outfname,'w')
         close_at_end = True
-
-    if 't' in opts:
-        input_hist = self.shell.history_manager.input_hist_parsed
-    elif 'r' in opts:
-        input_hist = self.shell.history_manager.input_hist_raw
-    else:
-        # Raw history is the default
-        input_hist = self.shell.history_manager.input_hist_raw
-            
-    default_length = 40
-    pattern = None
-    if 'g' in opts:
-        init = 1
-        final = len(input_hist)
-        parts = parameter_s.split(None, 1)
-        if len(parts) == 1:
-            parts += '*'
-        head, pattern = parts
-        pattern = "*" + pattern + "*"
-    elif len(args) == 0:
-        final = len(input_hist)-1
-        init = max(1,final-default_length)
-    elif len(args) == 1:
-        final = len(input_hist)
-        init = max(1, final-int(args[0]))
-    elif len(args) == 2:
-        init, final = map(int, args)
-    else:
-        warn('%hist takes 0, 1 or 2 arguments separated by spaces.')
-        print(self.magic_hist.__doc__, file=IPython.utils.io.Term.cout)
-        return
     
-    width = len(str(final))
-    line_sep = ['','\n']
     print_nums = 'n' in opts
-    print_outputs = 'o' in opts
+    get_output = 'o' in opts
     pyprompts = 'p' in opts
+    # Raw history is the default
+    raw = not('t' in opts)
+            
+    default_length = 40
+    pattern = None
     
-    found = False
-    if pattern is not None:
-        sh = self.shell.history_manager.shadowhist.all()
-        for idx, s in sh:
-            if fnmatch.fnmatch(s, pattern):
-                print("0%d: %s" %(idx, s.expandtabs(4)), file=outfile)
-                found = True
+    if 'g' in opts:         # Glob search
+        pattern = "*" + args + "*" if args else "*"
+        hist = history_manager.search(pattern, raw=raw, output=get_output)
+    elif 'l' in opts:       # Get 'tail'
+        try:
+            n = int(args)
+        except ValueError, IndexError:
+            n = 10
+        hist = history_manager.get_tail(n, raw=raw, output=get_output)
+    else:
+        if args:            # Get history by ranges
+            hist = history_manager.get_range_by_str(args, raw, get_output)
+        else:               # Just get history for the current session
+            hist = history_manager.get_range(raw=raw, output=get_output)
     
-    if found:
-        print("===", file=outfile)
-        print("shadow history ends, fetch by %rep <number> (must start with 0)",
-              file=outfile)
-        print("=== start of normal history ===", file=outfile)
+    # We could be displaying the entire history, so let's not try to pull it 
+    # into a list in memory. Anything that needs more space will just misalign.
+    width = 4
         
-    for in_num in range(init, final):
+    for session, lineno, inline in hist:
         # Print user history with tabs expanded to 4 spaces.  The GUI clients
         # use hard tabs for easier usability in auto-indented code, but we want
         # to produce PEP-8 compliant history for safe pasting into an editor.
-        inline = input_hist[in_num].expandtabs(4).rstrip()+'\n'
-
-        if pattern is not None and not fnmatch.fnmatch(inline, pattern):
-            continue
+        if get_output:
+            inline, output = inline
+        inline = inline.expandtabs(4).rstrip()
             
-        multiline = int(inline.count('\n') > 1)
+        multiline = "\n" in inline
+        line_sep = '\n' if multiline else ' '
         if print_nums:
-            print('%s:%s' % (str(in_num).ljust(width), line_sep[multiline]),
-                  file=outfile)
+            print('%s:%s' % (_format_lineno(session, lineno).rjust(width),
+                    line_sep),  file=outfile, end='')
         if pyprompts:
-            print('>>>', file=outfile)
+            print(">>> ", end="", file=outfile)
             if multiline:
-                lines = inline.splitlines()
-                print('\n... '.join(lines), file=outfile)
-                print('... ', file=outfile)
-            else:
-                print(inline, end='', file=outfile)
-        else:
-            print(inline, end='', file=outfile)
-        if print_outputs:
-            output = self.shell.history_manager.output_hist.get(in_num)
-            if output is not None:
-                print(repr(output), file=outfile)
+                inline = "\n... ".join(inline.splitlines()) + "\n..."
+        print(inline, file=outfile)
+        if get_output and output:
+            print("\n".join(output), file=outfile)
 
     if close_at_end:
         outfile.close()
 
 
-def magic_hist(self, parameter_s=''):
-    """Alternate name for %history."""
-    return self.magic_history(parameter_s)
-
-
-def rep_f(self, arg):
+def magic_rep(self, arg):
     r""" Repeat a command, or get command to input line for editing
 
     - %rep (no arguments):
@@ -478,110 +641,98 b' def rep_f(self, arg):'
     variable) to the next input prompt. Allows you to create elaborate command
     lines without using copy-paste::
     
-        $ l = ["hei", "vaan"]       
-        $ "".join(l)        
-        ==> heivaan        
-        $ %rep        
-        $ heivaan_ <== cursor blinking    
+         In[1]: l = ["hei", "vaan"]
+         In[2]: "".join(l)
+        Out[2]: heivaan
+         In[3]: %rep
+         In[4]: heivaan_ <== cursor blinking
     
     %rep 45
     
-    Place history line 45 to next input prompt. Use %hist to find out the
-    number.
+    Place history line 45 on the next input prompt. Use %hist to find
+    out the number.
     
-    %rep 1-4 6-7 3
+    %rep 1-4
     
-    Repeat the specified lines immediately. Input slice syntax is the same as
-    in %macro and %save.
+    Combine the specified lines into one cell, and place it on the next
+    input prompt. See %history for the slice syntax.
     
-    %rep foo
+    %rep foo+bar
     
-    Place the most recent line that has the substring "foo" to next input.
-    (e.g. 'svn ci -m foobar').    
+    If foo+bar can be evaluated in the user namespace, the result is
+    placed at the next input prompt. Otherwise, the history is searched
+    for lines which contain that substring, and the most recent one is
+    placed at the next input prompt.
     """
-    
-    opts,args = self.parse_options(arg,'',mode='list')
-    if not args:
+    if not arg:                 # Last output
         self.set_next_input(str(self.shell.user_ns["_"]))
         return
+                                # Get history range
+    histlines = self.history_manager.get_range_by_str(arg)
+    cmd = "\n".join(x[2] for x in histlines)
+    if cmd:
+        self.set_next_input(cmd.rstrip())
+        return
 
-    if len(args) == 1 and not '-' in args[0]:
-        arg = args[0]
-        if len(arg) > 1 and arg.startswith('0'):
-            # get from shadow hist
-            num = int(arg[1:])
-            line = self.shell.shadowhist.get(num)
-            self.set_next_input(str(line))
-            return
-        try:
-            num = int(args[0])
-            self.set_next_input(str(self.shell.input_hist_raw[num]).rstrip())
-            return
-        except ValueError:
-            pass
-        
-        for h in reversed(self.shell.input_hist_raw):
+    try:                        # Variable in user namespace
+        cmd = str(eval(arg, self.shell.user_ns))
+    except Exception:           # Search for term in history
+        histlines = self.history_manager.search("*"+arg+"*")
+        for h in reversed([x[2] for x in histlines]):
             if 'rep' in h:
                 continue
-            if fnmatch.fnmatch(h,'*' + arg + '*'):
-                self.set_next_input(str(h).rstrip())
-                return
-        
-    try:
-        lines = self.extract_input_slices(args, True)
-        print("lines", lines)
-        self.run_cell(lines)
-    except ValueError:
-        print("Not found in recent history:", args)
+            self.set_next_input(h.rstrip())
+            return
+    else:
+        self.set_next_input(cmd.rstrip())
+    print("Couldn't evaluate or find in history:", arg)
         
-
-_sentinel = object()
-
-class ShadowHist(object):
-    def __init__(self, db, shell):
-        # cmd => idx mapping
-        self.curidx = 0
-        self.db = db
-        self.disabled = False
-        self.shell = shell
+def magic_rerun(self, parameter_s=''):
+    """Re-run previous input
     
-    def inc_idx(self):
-        idx = self.db.get('shadowhist_idx', 1)
-        self.db['shadowhist_idx'] = idx + 1
-        return idx
-        
-    def add(self, ent):
-        if self.disabled:
-            return
-        try:
-            old = self.db.hget('shadowhist', ent, _sentinel)
-            if old is not _sentinel:
-                return
-            newidx = self.inc_idx()
-            #print("new", newidx) # dbg
-            self.db.hset('shadowhist',ent, newidx)
-        except:
-            self.shell.showtraceback()
-            print("WARNING: disabling shadow history")
-            self.disabled = True
+    By default, you can specify ranges of input history to be repeated
+    (as with %history). With no arguments, it will repeat the last line.
     
-    def all(self):
-        d = self.db.hdict('shadowhist')
-        items = [(i,s) for (s,i) in d.iteritems()]
-        items.sort()
-        return items
-
-    def get(self, idx):
-        all = self.all()
-        
-        for k, v in all:
-            if k == idx:
-                return v
+    Options:
+    
+      -l <n> : Repeat the last n lines of input, not including the
+      current command.
+      
+      -g foo : Repeat the most recent line which contains foo
+    """
+    opts, args = self.parse_options(parameter_s, 'l:g:', mode='string')
+    if "l" in opts:         # Last n lines
+        n = int(opts['l'])
+        hist = self.history_manager.get_tail(n)
+    elif "g" in opts:       # Search
+        p = "*"+opts['g']+"*"
+        hist = list(self.history_manager.search(p))
+        for l in reversed(hist):
+            if "rerun" not in l[2]:
+                hist = [l]     # The last match which isn't a %rerun
+                break
+        else:
+            hist = []          # No matches except %rerun
+    elif args:              # Specify history ranges
+        hist = self.history_manager.get_range_by_str(args)
+    else:                   # Last line
+        hist = self.history_manager.get_tail(1)
+    hist = [x[2] for x in hist]
+    if not hist:
+        print("No lines in history match specification")
+        return
+    histlines = "\n".join(hist)
+    print("=== Executing: ===")
+    print(histlines)
+    print("=== Output: ===")
+    self.run_cell("\n".join(hist), store_history=False)
 
 
 def init_ipython(ip):
-    ip.define_magic("rep",rep_f)        
-    ip.define_magic("hist",magic_hist)            
+    ip.define_magic("rep", magic_rep) 
+    ip.define_magic("recall", magic_rep)
+    ip.define_magic("rerun", magic_rerun)
+    ip.define_magic("hist",magic_history)    # Alternative name
     ip.define_magic("history",magic_history)
 
     # XXX - ipy_completers are in quarantine, need to be updated to new apis

IPython/core/interactiveshell.py

@@ -56,11 +56,11 b' from IPython.core.prefilter import PrefilterManager, ESC_MAGIC'
 from IPython.external.Itpl import ItplNS
 from IPython.utils import PyColorize
 from IPython.utils import io
-from IPython.utils import pickleshare
 from IPython.utils.doctestreload import doctest_reload
 from IPython.utils.io import ask_yes_no, rprint
 from IPython.utils.ipstruct import Struct
 from IPython.utils.path import get_home_dir, get_ipython_dir, HomeDirError
+from IPython.utils.pickleshare import PickleShareDB
 from IPython.utils.process import system, getoutput
 from IPython.utils.strdispatch import StrDispatch
 from IPython.utils.syspathcontext import prepended_to_syspath
@@ -251,6 +251,11 b' class InteractiveShell(Configurable, Magic):'
         # is what we want to do.
         self.save_sys_module_state()
         self.init_sys_modules()
+        
+        # While we're trying to have each part of the code directly access what
+        # it needs without keeping redundant references to objects, we have too
+        # much legacy code that expects ip.db to exist.
+        self.db = PickleShareDB(os.path.join(self.ipython_dir, 'db'))
 
         self.init_history()
         self.init_encoding()
@@ -301,14 +306,6 b' class InteractiveShell(Configurable, Magic):'
         self.hooks.late_startup_hook()
         atexit.register(self.atexit_operations)
 
-    # While we're trying to have each part of the code directly access what it
-    # needs without keeping redundant references to objects, we have too much
-    # legacy code that expects ip.db to exist, so let's make it a property that
-    # retrieves the underlying object from our new history manager.
-    @property
-    def db(self):
-        return self.history_manager.shadow_db
-
     @classmethod
     def instance(cls, *args, **kwargs):
         """Returns a global InteractiveShell instance."""
@@ -994,14 +991,16 b' class InteractiveShell(Configurable, Magic):'
         # Finally, update the real user's namespace
         self.user_ns.update(ns)
 
-    def reset(self):
+    def reset(self, new_session=True):
         """Clear all internal namespaces.
 
         Note that this is much more aggressive than %reset, since it clears
         fully all namespaces, as well as all input/output lists.
+        
+        If new_session is True, a new history session will be opened.
         """
         # Clear histories
-        self.history_manager.reset()
+        self.history_manager.reset(new_session)
 
         # Reset counter used to index all histories
         self.execution_count = 0
@@ -1249,15 +1248,7 b' class InteractiveShell(Configurable, Magic):'
 
     def init_history(self):
         """Sets up the command history, and starts regular autosaves."""
-        self.history_manager = HistoryManager(shell=self)
-
-    def save_history(self):
-        """Save input history to a file (via readline library)."""
-        self.history_manager.save_history()
-
-    def reload_history(self):
-        """Reload the input history from disk file."""
-        self.history_manager.reload_history()
+        self.history_manager = HistoryManager(shell=self, config=self.config)
 
     def history_saving_wrapper(self, func):
         """ Wrap func for readline history saving
@@ -1278,9 +1269,6 b' class InteractiveShell(Configurable, Magic):'
                 self.reload_history()
         return wrapper
     
-    def get_history(self, index=None, raw=False, output=True):
-        return self.history_manager.get_history(index, raw, output)
-    
 
     #-------------------------------------------------------------------------
     # Things related to exception handling and tracebacks (not debugging)
@@ -1561,11 +1549,13 b' class InteractiveShell(Configurable, Magic):'
             readline.set_completer_delims(delims)
             # otherwise we end up with a monster history after a while:
             readline.set_history_length(self.history_length)
-            try:
-                #print '*** Reading readline history'  # dbg
-                self.reload_history() 
-            except IOError:
-                pass  # It doesn't exist yet.
+            
+            # Load the last 1000 lines from history
+            for _, _, cell in self.history_manager.get_tail(1000,
+                                                include_latest=True):
+                if cell.strip(): # Ignore blank lines
+                    for line in cell.splitlines():
+                        readline.add_history(line)
 
         # Configure auto-indent for all platforms
         self.set_autoindent(self.autoindent)
@@ -2071,14 +2061,16 b' class InteractiveShell(Configurable, Magic):'
                 self.showtraceback()
                 warn('Unknown failure executing file: <%s>' % fname)
 
-    def run_cell(self, cell):
-        """Run the contents of an entire multiline 'cell' of code.
+    def run_cell(self, cell, store_history=True):
+        """Run the contents of an entire multiline 'cell' of code, and store it
+        in the history.
 
         The cell is split into separate blocks which can be executed
         individually.  Then, based on how many blocks there are, they are
         executed as follows:
 
-        - A single block: 'single' mode.
+        - A single block: 'single' mode. If it is also a single line, dynamic
+        transformations, including automagic and macros, will be applied.
 
         If there's more than one block, it depends:
 
@@ -2097,6 +2089,15 b' class InteractiveShell(Configurable, Magic):'
         cell : str
           A single or multiline string.
         """
+        # Store the untransformed code
+        raw_cell = cell
+        
+        # We only do dynamic transforms on a single line. We need to do this
+        # first, because a macro can be expanded to several lines, which then
+        # need to be split into blocks again.
+        if len(cell.splitlines()) <= 1:
+            temp = self.input_splitter.split_blocks(cell)
+            cell = self.prefilter_manager.prefilter_line(temp[0])
         
         # We need to break up the input into executable blocks that can be run
         # in 'single' mode, to provide comfortable user behavior.
@@ -2108,32 +2109,36 b' class InteractiveShell(Configurable, Magic):'
         # Store the 'ipython' version of the cell as well, since that's what
         # needs to go into the translated history and get executed (the
         # original cell may contain non-python syntax).
-        ipy_cell = ''.join(blocks)
+        cell = ''.join(blocks)
 
         # Store raw and processed history
-        self.history_manager.store_inputs(ipy_cell, cell)
+        if store_history:
+            self.history_manager.store_inputs(self.execution_count, 
+                                                        cell, raw_cell)
 
-        self.logger.log(ipy_cell, cell)
+        self.logger.log(cell, raw_cell)
 
         # All user code execution must happen with our context managers active
         with nested(self.builtin_trap, self.display_trap):
 
             # Single-block input should behave like an interactive prompt
             if len(blocks) == 1:
-                # since we return here, we need to update the execution count
-                out = self.run_one_block(blocks[0])
-                self.execution_count += 1
+                out = self.run_source(blocks[0])
+                # Write output to the database. Does nothing unless
+                # history output logging is enabled.
+                if store_history:
+                    self.history_manager.store_output(self.execution_count)
+                    # since we return here, we need to update the execution count
+                    self.execution_count += 1
                 return out
 
             # In multi-block input, if the last block is a simple (one-two
             # lines) expression, run it in single mode so it produces output.
-            # Otherwise just feed the whole thing to run_code.  This seems like
-            # a reasonable usability design.
+            # Otherwise just run it all in 'exec' mode.  This seems like a
+            # reasonable usability design.
             last = blocks[-1]
             last_nlines = len(last.splitlines())
-
-            # Note: below, whenever we call run_code, we must sync history
-            # ourselves, because run_code is NOT meant to manage history at all.
+            
             if last_nlines < 2:
                 # Here we consider the cell split between 'body' and 'last',
                 # store all history and execute 'body', and if successful, then
@@ -2144,55 +2149,19 b' class InteractiveShell(Configurable, Magic):'
                 retcode = self.run_source(ipy_body, symbol='exec',
                                           post_execute=False)
                 if retcode==0:
-                    # And the last expression via runlines so it produces output
-                    self.run_one_block(last)
+                    # Last expression compiled as 'single' so it produces output
+                    self.run_source(last)
             else:
                 # Run the whole cell as one entity, storing both raw and
                 # processed input in history
                 self.run_source(ipy_cell, symbol='exec')
 
-        # Each cell is a *single* input, regardless of how many lines it has
-        self.execution_count += 1
-
-    def run_one_block(self, block):
-        """Run a single interactive block of source code.
-
-        If the block is single-line, dynamic transformations are applied to it
-        (like automagics, autocall and alias recognition).
-
-        If the block is multi-line, it must consist of valid Python code only.
-
-        Parameters
-        ----------
-        block : string
-           A (possibly multiline) string of code to be executed.
-
-        Returns
-        -------
-        The output of the underlying execution method used, be it
-        :meth:`run_source` or :meth:`run_single_line`.
-        """
-        if len(block.splitlines()) <= 1:
-            out = self.run_single_line(block)
-        else:
-            # Call run_source, which correctly compiles the input cell.
-            # run_code must only be called when we know we have a code object,
-            # as it does a naked exec and the compilation mode may not be what
-            # we wanted.
-            out = self.run_source(block)
-        return out
-
-    def run_single_line(self, line):
-        """Run a single-line interactive statement.
-
-        This assumes the input has been transformed to IPython syntax by
-        applying all static transformations (those with an explicit prefix like
-        % or !), but it will further try to apply the dynamic ones.
-
-        It does not update history.
-        """
-        tline = self.prefilter_manager.prefilter_line(line)
-        return self.run_source(tline)
+        # Write output to the database. Does nothing unless
+        # history output logging is enabled.
+        if store_history:
+            self.history_manager.store_output(self.execution_count)
+            # Each cell is a *single* input, regardless of how many lines it has
+            self.execution_count += 1
 
     # PENDING REMOVAL: this method is slated for deletion, once our new
     # input logic has been 100% moved to frontends and is stable.
@@ -2205,8 +2174,8 b' class InteractiveShell(Configurable, Magic):'
         magic calls (%magic), special shell access (!cmd), etc.
         """
         
-        if isinstance(lines, (list, tuple)):
-            lines = '\n'.join(lines)
+        if not isinstance(lines, (list, tuple)):
+            lines = lines.splitlines()
 
         if clean:
             lines = self._cleanup_ipy_script(lines)
@@ -2214,7 +2183,6 b' class InteractiveShell(Configurable, Magic):'
         # We must start with a clean buffer, in case this is run from an
         # interactive IPython session (via a magic, for example).
         self.reset_buffer()
-        lines = lines.splitlines()
 
         # Since we will prefilter all lines, store the user's raw input too
         # before we apply any transformations
@@ -2401,8 +2369,8 b' class InteractiveShell(Configurable, Magic):'
         full_source = '\n'.join(self.buffer)
         more = self.run_source(full_source, self.filename)
         if not more:
-            self.history_manager.store_inputs('\n'.join(self.buffer_raw),
-                                              full_source)
+            self.history_manager.store_inputs(self.execution_count,
+                                        '\n'.join(self.buffer_raw), full_source)
             self.reset_buffer()
             self.execution_count += 1
         return more
@@ -2539,11 +2507,12 b' class InteractiveShell(Configurable, Magic):'
                 os.unlink(tfile)
             except OSError:
                 pass
-
-        self.save_history()
-
+        
+        # Close the history session (this stores the end time and line count)
+        self.history_manager.end_session()
+        
         # Clear all user namespaces to release all references cleanly.
-        self.reset()
+        self.reset(new_session=False)
 
         # Run user hooks
         self.hooks.shutdown_hook()

IPython/core/macro.py

@@ -8,9 +8,8 b''
 #*****************************************************************************
 
 import IPython.utils.io
-from IPython.core.autocall import IPyAutocall
 
-class Macro(IPyAutocall):
+class Macro(object):
     """Simple class to store the value of macros as strings.
 
     Macro is just a callable that executes a string of IPython
@@ -19,9 +18,9 b' class Macro(IPyAutocall):'
     Args to macro are available in _margv list if you need them.
     """
 
-    def __init__(self,data):
+    def __init__(self,code):
         """store the macro value, as a single string which can be executed"""
-        self.value = ''.join(data).rstrip()+'\n'
+        self.value = code.rstrip()+'\n'
         
     def __str__(self):
         return self.value
@@ -29,11 +28,6 b' class Macro(IPyAutocall):'
     def __repr__(self):
         return 'IPython.macro.Macro(%s)' % repr(self.value)
     
-    def __call__(self,*args):
-        IPython.utils.io.Term.cout.flush()
-        self._ip.user_ns['_margv'] = args
-        self._ip.run_cell(self.value)
-    
     def __getstate__(self):
         """ needed for safe pickling via %store """
         return {'value': self.value}

IPython/core/magic.py

@@ -57,7 +57,7 b' import IPython.utils.io'
 from IPython.utils.path import get_py_filename
 from IPython.utils.process import arg_split, abbrev_cwd
 from IPython.utils.terminal import set_term_title
-from IPython.utils.text import LSString, SList, StringTypes, format_screen
+from IPython.utils.text import LSString, SList, format_screen
 from IPython.utils.timing import clock, clock2
 from IPython.utils.warn import warn, error
 from IPython.utils.ipstruct import Struct
@@ -165,14 +165,15 b' python-profiler package from non-free.""")'
         out.sort()
         return out
     
-    def extract_input_slices(self,slices,raw=False):
+    def extract_input_lines(self, range_str, raw=False):
         """Return as a string a set of input history slices.
 
         Inputs:
 
-          - slices: the set of slices is given as a list of strings (like
-          ['1','4:8','9'], since this function is for use by magic functions
-          which get their arguments as strings.
+          - range_str: the set of slices is given as a string, like 
+          "~5/6-~4/2 4:8 9", since this function is for use by magic functions
+          which get their arguments as strings. The number before the / is the
+          session number: ~n goes n back from the current session.
 
         Optional inputs:
 
@@ -184,24 +185,9 b' python-profiler package from non-free.""")'
         N:M -> standard python form, means including items N...(M-1).
 
         N-M -> include items N..M (closed endpoint)."""
-
-        if raw:
-            hist = self.shell.history_manager.input_hist_raw
-        else:
-            hist = self.shell.history_manager.input_hist_parsed
-
-        cmds = []
-        for chunk in slices:
-            if ':' in chunk:
-                ini,fin = map(int,chunk.split(':'))
-            elif '-' in chunk:
-                ini,fin = map(int,chunk.split('-'))
-                fin += 1
-            else:
-                ini = int(chunk)
-                fin = ini+1
-            cmds.append(''.join(hist[ini:fin]))
-        return cmds
+        lines = self.shell.history_manager.\
+                                    get_range_by_str(range_str, raw=raw)
+        return "\n".join(x for _, _, x in lines)
             
     def arg_err(self,func):
         """Print docstring if incorrect arguments were passed"""
@@ -1617,7 +1603,7 b' Currently the magic system has the following functions:\\n"""'
         
         stats = None
         try:
-            self.shell.save_history()
+            #self.shell.save_history()
 
             if opts.has_key('p'):
                 stats = self.magic_prun('',0,opts,arg_lst,prog_ns)
@@ -1736,7 +1722,7 b' Currently the magic system has the following functions:\\n"""'
                 # contained therein.
                 del sys.modules[main_mod_name]
 
-            self.shell.reload_history()
+            #self.shell.reload_history()
                 
         return stats
 
@@ -1990,9 +1976,7 b' Currently the magic system has the following functions:\\n"""'
         you had typed them. You just type 'name' at the prompt and the code
         executes.
 
-        The notation for indicating number ranges is: n1-n2 means 'use line
-        numbers n1,...n2' (the endpoint is included).  That is, '5-7' means
-        using the lines numbered 5,6 and 7.
+        The syntax for indicating input ranges is described in %history.
 
         Note: as a 'hidden' feature, you can also use traditional python slice
         notation, where N:M means numbers N through M-1.
@@ -2033,17 +2017,16 b' Currently the magic system has the following functions:\\n"""'
           In [60]: exec In[44:48]+In[49]"""
 
         opts,args = self.parse_options(parameter_s,'r',mode='list')
-        if not args:
-            macs = [k for k,v in self.shell.user_ns.items() if isinstance(v, Macro)]
-            macs.sort()
-            return macs
+        if not args:   # List existing macros
+            return sorted(k for k,v in self.shell.user_ns.iteritems() if\
+                                                        isinstance(v, Macro))
         if len(args) == 1:
             raise UsageError(
                 "%macro insufficient args; usage '%macro name n1-n2 n3-4...")
-        name,ranges = args[0], args[1:]
+        name, ranges = args[0], " ".join(args[1:])
         
         #print 'rng',ranges  # dbg
-        lines = self.extract_input_slices(ranges,opts.has_key('r'))
+        lines = self.extract_input_lines(ranges,'r' in opts)
         macro = Macro(lines)
         self.shell.define_macro(name, macro)
         print 'Macro `%s` created. To execute, type its name (without quotes).' % name
@@ -2063,15 +2046,14 b' Currently the magic system has the following functions:\\n"""'
           Python.  If this option is given, the raw input as typed as the
           command line is used instead.
 
-        This function uses the same syntax as %macro for line extraction, but
-        instead of creating a macro it saves the resulting string to the
-        filename you specify.
+        This function uses the same syntax as %history for input ranges, 
+        then saves the lines to the filename you specify.
 
         It adds a '.py' extension to the file if you don't do so yourself, and
         it asks for confirmation before overwriting existing files."""
 
         opts,args = self.parse_options(parameter_s,'r',mode='list')
-        fname,ranges = args[0], args[1:]
+        fname,ranges = args[0], " ".join(args[1:])
         if not fname.endswith('.py'):
             fname += '.py'
         if os.path.isfile(fname):
@@ -2079,10 +2061,9 b' Currently the magic system has the following functions:\\n"""'
             if ans.lower() not in ['y','yes']:
                 print 'Operation cancelled.'
                 return
-        cmds = ''.join(self.extract_input_slices(ranges,opts.has_key('r')))
-        f = file(fname,'w')
-        f.write(cmds)
-        f.close()
+        cmds = self.extract_input_lines(ranges, 'r' in opts)
+        with open(fname,'w') as f:
+            f.write(cmds)
         print 'The following commands were written to file `%s`:' % fname
         print cmds
 
@@ -2154,15 +2135,17 b' Currently the magic system has the following functions:\\n"""'
         Arguments:
 
         If arguments are given, the following possibilites exist:
+        
+        - If the argument is a filename, IPython will load that into the
+        editor. It will execute its contents with execfile() when you exit,
+        loading any code in the file into your interactive namespace.
 
-        - The arguments are numbers or pairs of colon-separated numbers (like
-        1 4:8 9). These are interpreted as lines of previous input to be
-        loaded into the editor. The syntax is the same of the %macro command.
+        - The arguments are ranges of input history,  e.g. "7 ~1/4-6".
+        The syntax is the same as in the %history magic.
 
-        - If the argument doesn't start with a number, it is evaluated as a
-        variable and its contents loaded into the editor. You can thus edit
-        any string which contains python code (including the result of
-        previous edits).
+        - If the argument is a string variable, its contents are loaded
+        into the editor. You can thus edit any string which contains
+        python code (including the result of previous edits).
 
         - If the argument is the name of an object (other than a string),
         IPython will try to locate the file where it was defined and open the
@@ -2179,11 +2162,6 b' Currently the magic system has the following functions:\\n"""'
         '+NUMBER' parameter necessary for this feature. Good editors like
         (X)Emacs, vi, jed, pico and joe all do.
 
-        - If the argument is not found as a variable, IPython will look for a
-        file with that name (adding .py if necessary) and load it into the
-        editor. It will execute its contents with execfile() when you exit,
-        loading any code in the file into your interactive namespace.
-
         After executing your code, %edit will return as output the code you
         typed in the editor (except when it was an existing file). This way
         you can reload the code in further invocations of %edit as a variable,
@@ -2266,13 +2244,13 b' Currently the magic system has the following functions:\\n"""'
 
         opts,args = self.parse_options(parameter_s,'prxn:')
         # Set a few locals from the options for convenience:
-        opts_p = opts.has_key('p')
-        opts_r = opts.has_key('r')
+        opts_prev = 'p' in opts
+        opts_raw = 'r' in opts
         
         # Default line number value
         lineno = opts.get('n',None)
 
-        if opts_p:
+        if opts_prev:
             args = '_%s' % last_call[0]
             if not self.shell.user_ns.has_key(args):
                 args = last_call[1]
@@ -2281,90 +2259,83 b' Currently the magic system has the following functions:\\n"""'
         # let it be clobbered by successive '-p' calls.
         try:
             last_call[0] = self.shell.displayhook.prompt_count
-            if not opts_p:
+            if not opts_prev:
                 last_call[1] = parameter_s
         except:
             pass
 
         # by default this is done with temp files, except when the given
         # arg is a filename
-        use_temp = 1
+        use_temp = True
 
-        if re.match(r'\d',args):
-            # Mode where user specifies ranges of lines, like in %macro.
-            # This means that you can't edit files whose names begin with
-            # numbers this way. Tough.
-            ranges = args.split()
-            data = ''.join(self.extract_input_slices(ranges,opts_r))
-        elif args.endswith('.py'):
+        data = ''
+        if args.endswith('.py'):
             filename = make_filename(args)
-            data = ''
-            use_temp = 0
+            use_temp = False
         elif args:
-            try:
-                # Load the parameter given as a variable. If not a string,
-                # process it as an object instead (below)
-
-                #print '*** args',args,'type',type(args)  # dbg
-                data = eval(args,self.shell.user_ns)
-                if not type(data) in StringTypes:
-                    raise DataIsObject
-
-            except (NameError,SyntaxError):
-                # given argument is not a variable, try as a filename
-                filename = make_filename(args)
-                if filename is None:
-                    warn("Argument given (%s) can't be found as a variable "
-                         "or as a filename." % args)
-                    return
+            # Mode where user specifies ranges of lines, like in %macro.
+            data = self.extract_input_lines(args, opts_raw)
+            if not data:
+                try:
+                    # Load the parameter given as a variable. If not a string,
+                    # process it as an object instead (below)
 
-                data = ''
-                use_temp = 0
-            except DataIsObject:
+                    #print '*** args',args,'type',type(args)  # dbg
+                    data = eval(args, self.shell.user_ns)
+                    if not isinstance(data, basestring):
+                        raise DataIsObject
 
-                # macros have a special edit function
-                if isinstance(data,Macro):
-                    self._edit_macro(args,data)
-                    return
-                                
-                # For objects, try to edit the file where they are defined
-                try:
-                    filename = inspect.getabsfile(data)
-                    if 'fakemodule' in filename.lower() and inspect.isclass(data):                     
-                        # class created by %edit? Try to find source
-                        # by looking for method definitions instead, the
-                        # __module__ in those classes is FakeModule.
-                        attrs = [getattr(data, aname) for aname in dir(data)]
-                        for attr in attrs:
-                            if not inspect.ismethod(attr):
-                                continue
-                            filename = inspect.getabsfile(attr)
-                            if filename and 'fakemodule' not in filename.lower():
-                                # change the attribute to be the edit target instead
-                                data = attr 
-                                break
-                    
-                    datafile = 1
-                except TypeError:
+                except (NameError,SyntaxError):
+                    # given argument is not a variable, try as a filename
                     filename = make_filename(args)
-                    datafile = 1
-                    warn('Could not find file where `%s` is defined.\n'
-                         'Opening a file named `%s`' % (args,filename))
-                # Now, make sure we can actually read the source (if it was in
-                # a temp file it's gone by now).
-                if datafile:
+                    if filename is None:
+                        warn("Argument given (%s) can't be found as a variable "
+                             "or as a filename." % args)
+                        return
+                    use_temp = False
+                
+                except DataIsObject:
+                    # macros have a special edit function
+                    if isinstance(data, Macro):
+                        self._edit_macro(args,data)
+                        return
+                                    
+                    # For objects, try to edit the file where they are defined
                     try:
-                        if lineno is None:
-                            lineno = inspect.getsourcelines(data)[1]
-                    except IOError:
+                        filename = inspect.getabsfile(data)
+                        if 'fakemodule' in filename.lower() and inspect.isclass(data):                     
+                            # class created by %edit? Try to find source
+                            # by looking for method definitions instead, the
+                            # __module__ in those classes is FakeModule.
+                            attrs = [getattr(data, aname) for aname in dir(data)]
+                            for attr in attrs:
+                                if not inspect.ismethod(attr):
+                                    continue
+                                filename = inspect.getabsfile(attr)
+                                if filename and 'fakemodule' not in filename.lower():
+                                    # change the attribute to be the edit target instead
+                                    data = attr 
+                                    break
+                        
+                        datafile = 1
+                    except TypeError:
                         filename = make_filename(args)
-                        if filename is None:
-                            warn('The file `%s` where `%s` was defined cannot '
-                                 'be read.' % (filename,data))
-                            return
-                use_temp = 0
-        else:
-            data = ''
+                        datafile = 1
+                        warn('Could not find file where `%s` is defined.\n'
+                             'Opening a file named `%s`' % (args,filename))
+                    # Now, make sure we can actually read the source (if it was in
+                    # a temp file it's gone by now).
+                    if datafile:
+                        try:
+                            if lineno is None:
+                                lineno = inspect.getsourcelines(data)[1]
+                        except IOError:
+                            filename = make_filename(args)
+                            if filename is None:
+                                warn('The file `%s` where `%s` was defined cannot '
+                                     'be read.' % (filename,data))
+                                return
+                    use_temp = False
 
         if use_temp:
             filename = self.shell.mktempfile(data)
@@ -2387,12 +2358,13 b' Currently the magic system has the following functions:\\n"""'
         if args.strip() == 'pasted_block':
             self.shell.user_ns['pasted_block'] = file_read(filename)
         
-        if opts.has_key('x'):  # -x prevents actual execution
+        if 'x' in opts:  # -x prevents actual execution
             print
         else:
             print 'done. Executing edited code...'
-            if opts_r:
-                self.shell.run_cell(file_read(filename))
+            if opts_raw:
+                self.shell.run_cell(file_read(filename),
+                                                    store_history=False)
             else:
                 self.shell.safe_execfile(filename,self.shell.user_ns,
                                          self.shell.user_ns)
@@ -3050,38 +3022,6 b' Defaulting color scheme to \'NoColor\'"""'
         if parameter_s:
             return self.shell.getoutput(parameter_s)
 
-    def magic_r(self, parameter_s=''):
-        """Repeat previous input.
-
-        Note: Consider using the more powerfull %rep instead!
-        
-        If given an argument, repeats the previous command which starts with
-        the same string, otherwise it just repeats the previous input.
-
-        Shell escaped commands (with ! as first character) are not recognized
-        by this system, only pure python code and magic commands.
-        """
-
-        start = parameter_s.strip()
-        esc_magic = ESC_MAGIC
-        # Identify magic commands even if automagic is on (which means
-        # the in-memory version is different from that typed by the user).
-        if self.shell.automagic:
-            start_magic = esc_magic+start
-        else:
-            start_magic = start
-        # Look through the input history in reverse
-        for n in range(len(self.shell.history_manager.input_hist_parsed)-2,0,-1):
-            input = self.shell.history_manager.input_hist_parsed[n]
-            # skip plain 'r' lines so we don't recurse to infinity
-            if input != '_ip.magic("r")\n' and \
-                   (input.startswith(start) or input.startswith(start_magic)):
-                #print 'match',`input`  # dbg
-                print 'Executing:',input,
-                self.shell.run_cell(input)
-                return
-        print 'No previous input matching `%s` found.' % start
-
     
     def magic_bookmark(self, parameter_s=''):
         """Manage IPython's bookmark system.

IPython/core/prefilter.py

@@ -32,6 +32,7 b' import re'
 from IPython.core.alias import AliasManager
 from IPython.core.autocall import IPyAutocall
 from IPython.config.configurable import Configurable
+from IPython.core.macro import Macro
 from IPython.core.splitinput import split_user_input
 from IPython.core import page
 
@@ -598,6 +599,18 b' class ShellEscapeChecker(PrefilterChecker):'
             return self.prefilter_manager.get_handler_by_name('shell')
 
 
+class MacroChecker(PrefilterChecker):
+    
+    priority = Int(250, config=True)
+    
+    def check(self, line_info):
+        obj = self.shell.user_ns.get(line_info.ifun)
+        if isinstance(obj, Macro):
+            return self.prefilter_manager.get_handler_by_name('macro')
+        else:
+            return None
+
+
 class IPyAutocallChecker(PrefilterChecker):
 
     priority = Int(300, config=True)
@@ -837,6 +850,16 b' class ShellEscapeHandler(PrefilterHandler):'
         return line_out
 
 
+class MacroHandler(PrefilterHandler):
+    handler_name = Str("macro")
+    
+    def handle(self, line_info):
+        obj = self.shell.user_ns.get(line_info.ifun)
+        pre_space = line_info.pre_whitespace
+        line_sep = "\n" + pre_space
+        return pre_space + line_sep.join(obj.value.splitlines())
+
+
 class MagicHandler(PrefilterHandler):
 
     handler_name = Str('magic')
@@ -979,6 +1002,7 b' _default_transformers = ['
 _default_checkers = [
     EmacsChecker,
     ShellEscapeChecker,
+    MacroChecker,
     IPyAutocallChecker,
     MultiLineMagicChecker,
     EscCharsChecker,
@@ -993,6 +1017,7 b' _default_handlers = ['
     PrefilterHandler,
     AliasHandler,
     ShellEscapeHandler,
+    MacroHandler,
     MagicHandler,
     AutoHandler,
     HelpHandler,

IPython/core/tests/test_history.py

@@ -14,35 +14,101 b' import nose.tools as nt'
 
 # our own packages
 from IPython.utils.tempdir import TemporaryDirectory
-from IPython.core.history import HistoryManager
+from IPython.core.history import HistoryManager, extract_hist_ranges
 
 def test_history():
 
     ip = get_ipython()
     with TemporaryDirectory() as tmpdir:
         #tmpdir = '/software/temp'
-        histfile = os.path.realpath(os.path.join(tmpdir, 'history.json'))
+        histfile = os.path.realpath(os.path.join(tmpdir, 'history.sqlite'))
         # Ensure that we restore the history management that we mess with in
         # this test doesn't affect the IPython instance used by the test suite
         # beyond this test.
         hist_manager_ori = ip.history_manager
         try:
-            ip.history_manager = HistoryManager(ip)
+            ip.history_manager = HistoryManager(shell=ip)
             ip.history_manager.hist_file = histfile
+            ip.history_manager.init_db()  # Has to be called after changing file
+            ip.history_manager.reset()
             print 'test',histfile
-            hist = ['a=1\n', 'def f():\n    test = 1\n    return test\n', 'b=2\n']
-            # test save and load
-            ip.history_manager.input_hist_raw[:] = []
-            for h in hist:
-                ip.history_manager.input_hist_raw.append(h)
-            ip.save_history()
-            ip.history_manager.input_hist_raw[:] = []
-            ip.reload_history()
-            print type(ip.history_manager.input_hist_raw)
-            print ip.history_manager.input_hist_raw
-            nt.assert_equal(len(ip.history_manager.input_hist_raw), len(hist))
-            for i,h in enumerate(hist):
-                nt.assert_equal(hist[i], ip.history_manager.input_hist_raw[i])
+            hist = ['a=1', 'def f():\n    test = 1\n    return test', 'b=2']
+            for i, h in enumerate(hist, start=1):
+                ip.history_manager.store_inputs(i, h)
+            
+            ip.history_manager.db_log_output = True
+            # Doesn't match the input, but we'll just check it's stored.
+            ip.history_manager.output_hist_reprs[3].append("spam")
+            ip.history_manager.store_output(3)
+            
+            nt.assert_equal(ip.history_manager.input_hist_raw, [''] + hist)
+            
+            # Check lines were written to DB
+            c = ip.history_manager.db.execute("SELECT source_raw FROM history")
+            nt.assert_equal([x for x, in c], hist)
+              
+            # New session
+            ip.history_manager.reset()
+            newcmds = ["z=5","class X(object):\n    pass", "k='p'"]
+            for i, cmd in enumerate(newcmds, start=1):
+                ip.history_manager.store_inputs(i, cmd)
+            gothist = ip.history_manager.get_range(start=1, stop=4)
+            nt.assert_equal(list(gothist), zip([0,0,0],[1,2,3], newcmds))
+            # Previous session:
+            gothist = ip.history_manager.get_range(-1, 1, 4)
+            nt.assert_equal(list(gothist), zip([1,1,1],[1,2,3], hist))
+            
+            # Check get_hist_tail
+            gothist = ip.history_manager.get_tail(4, output=True,
+                                                    include_latest=True)
+            expected = [(1, 3, (hist[-1], ["spam"])),
+                        (2, 1, (newcmds[0], None)),
+                        (2, 2, (newcmds[1], None)),
+                        (2, 3, (newcmds[2], None)),]
+            nt.assert_equal(list(gothist), expected)
+            
+            gothist = ip.history_manager.get_tail(2)
+            expected = [(2, 1, newcmds[0]),
+                        (2, 2, newcmds[1])]
+            nt.assert_equal(list(gothist), expected)
+            
+            # Check get_hist_search
+            gothist = ip.history_manager.search("*test*")
+            nt.assert_equal(list(gothist), [(1,2,hist[1])] )
+            gothist = ip.history_manager.search("b*", output=True)
+            nt.assert_equal(list(gothist), [(1,3,(hist[2],["spam"]))] )
+            
+            # Cross testing: check that magic %save can get previous session.
+            testfilename = os.path.realpath(os.path.join(tmpdir, "test.py"))
+            ip.magic_save(testfilename + " ~1/1-3")
+            testfile = open(testfilename, "r")
+            nt.assert_equal(testfile.read(), "\n".join(hist))
+            
+            # Duplicate line numbers - check that it doesn't crash, and
+            # gets a new session
+            ip.history_manager.store_inputs(1, "rogue")
+            nt.assert_equal(ip.history_manager.session_number, 3)
         finally:
             # Restore history manager
             ip.history_manager = hist_manager_ori
+
+def test_extract_hist_ranges():
+    instr = "1 2/3 ~4/5-6 ~4/7-~4/9 ~9/2-~7/5"
+    expected = [(0, 1, 2),  # 0 == current session
+                (2, 3, 4),
+                (-4, 5, 7),
+                (-4, 7, 10),
+                (-9, 2, None),  # None == to end
+                (-8, 1, None),
+                (-7, 1, 6)]
+    actual = list(extract_hist_ranges(instr))
+    nt.assert_equal(actual, expected)
+    
+def test_magic_rerun():
+    """Simple test for %rerun (no args -> rerun last line)"""
+    ip = get_ipython()
+    ip.run_cell("a = 10")
+    ip.run_cell("a += 1")
+    nt.assert_equal(ip.user_ns["a"], 11)
+    ip.run_cell("%rerun")
+    nt.assert_equal(ip.user_ns["a"], 12)

IPython/core/tests/test_magic.py

@@ -62,7 +62,7 b' def doctest_hist_f():'
 
     In [10]: tfile = tempfile.mktemp('.py','tmp-ipython-')
 
-    In [11]: %hist -n -f $tfile 3
+    In [11]: %hist -nl -f $tfile 3
 
     In [13]: import os; os.unlink(tfile)
     """
@@ -80,7 +80,7 b' def doctest_hist_r():'
 
     In [2]: x=1
 
-    In [3]: %hist -r 2
+    In [3]: %hist -rl 2
     x=1 # random
     %hist -r 2
     """
@@ -150,29 +150,38 b' def doctest_hist_op():'
     <...s instance at ...>
     >>> 
     """
-
-def test_shist():
-    # Simple tests of ShadowHist class - test generator.
-    import os, shutil, tempfile
-
-    from IPython.utils import pickleshare
-    from IPython.core.history import ShadowHist
-    
-    tfile = tempfile.mktemp('','tmp-ipython-')
-    
-    db = pickleshare.PickleShareDB(tfile)
-    s = ShadowHist(db, get_ipython())
-    s.add('hello')
-    s.add('world')
-    s.add('hello')
-    s.add('hello')
-    s.add('karhu')
-
-    yield nt.assert_equals,s.all(),[(1, 'hello'), (2, 'world'), (3, 'karhu')]
-    
-    yield nt.assert_equal,s.get(2),'world'
+  
+def test_macro():
+    ip = get_ipython()
+    ip.history_manager.reset()   # Clear any existing history.
+    cmds = ["a=1", "def b():\n  return a**2", "print(a,b())"]
+    for i, cmd in enumerate(cmds, start=1):
+        ip.history_manager.store_inputs(i, cmd)
+    ip.magic("macro test 1-3")
+    nt.assert_equal(ip.user_ns["test"].value, "\n".join(cmds)+"\n")
     
-    shutil.rmtree(tfile)
+    # List macros.
+    assert "test" in ip.magic("macro")
+
+def test_macro_run():
+    """Test that we can run a multi-line macro successfully."""
+    ip = get_ipython()
+    ip.history_manager.reset()
+    cmds = ["a=10", "a+=1", "print a", "%macro test 2-3"]
+    for cmd in cmds:
+        ip.run_cell(cmd)
+    nt.assert_equal(ip.user_ns["test"].value, "a+=1\nprint a\n")
+    original_stdout = sys.stdout
+    new_stdout = StringIO()
+    sys.stdout = new_stdout
+    try:
+        ip.run_cell("test")
+        nt.assert_true("12" in new_stdout.getvalue())
+        ip.run_cell("test")
+        nt.assert_true("13" in new_stdout.getvalue())
+    finally:
+        sys.stdout = original_stdout
+        new_stdout.close()
 
     
 # XXX failing for now, until we get clearcmd out of quarantine.  But we should

IPython/frontend/qt/console/ipython_widget.py

@@ -158,15 +158,12 b' class IPythonWidget(FrontendWidget):'
             else:
                 super(IPythonWidget, self)._handle_execute_reply(msg)
 
-    def _handle_history_reply(self, msg):
-        """ Implemented to handle history replies, which are only supported by
-            the IPython kernel.
+    def _handle_history_tail_reply(self, msg):
+        """ Implemented to handle history tail replies, which are only supported
+            by the IPython kernel.
         """
-        history_dict = msg['content']['history']
-        input_history_dict = {}
-        for key,val in history_dict.items():
-            input_history_dict[int(key)] = val
-        items = [ val.rstrip() for _, val in sorted(input_history_dict.items()) ]
+        history_items = msg['content']['history']
+        items = [ line.rstrip() for _, _, line in history_items ]
         self._set_history(items)
 
     def _handle_pyout(self, msg):
@@ -213,7 +210,7 b' class IPythonWidget(FrontendWidget):'
         """ Reimplemented to make a history request.
         """
         super(IPythonWidget, self)._started_channels()
-        self.kernel_manager.xreq_channel.history(raw=True, output=False)
+        self.kernel_manager.xreq_channel.history_tail(1000)
 
     #---------------------------------------------------------------------------
     # 'ConsoleWidget' public interface

IPython/frontend/terminal/interactiveshell.py

@@ -185,10 +185,6 b' class TerminalInteractiveShell(InteractiveShell):'
         
         with nested(self.builtin_trap, self.display_trap):
 
-            # if you run stuff with -c <cmd>, raw hist is not updated
-            # ensure that it's in sync
-            self.history_manager.sync_inputs()
-
             while 1:
                 try:
                     self.interact(display_banner=display_banner)
@@ -286,67 +282,6 b' class TerminalInteractiveShell(InteractiveShell):'
         # Turn off the exit flag, so the mainloop can be restarted if desired
         self.exit_now = False
 
-    def raw_input(self, prompt='', continue_prompt=False):
-        """Write a prompt and read a line.
-
-        The returned line does not include the trailing newline.
-        When the user enters the EOF key sequence, EOFError is raised.
-
-        Optional inputs:
-
-          - prompt(''): a string to be printed to prompt the user.
-
-          - continue_prompt(False): whether this line is the first one or a
-          continuation in a sequence of inputs.
-        """
-        # Code run by the user may have modified the readline completer state.
-        # We must ensure that our completer is back in place.
-
-        if self.has_readline:
-            self.set_readline_completer()
-        
-        try:
-            line = raw_input_original(prompt).decode(self.stdin_encoding)
-        except ValueError:
-            warn("\n********\nYou or a %run:ed script called sys.stdin.close()"
-                 " or sys.stdout.close()!\nExiting IPython!")
-            self.ask_exit()
-            return ""
-
-        # Try to be reasonably smart about not re-indenting pasted input more
-        # than necessary.  We do this by trimming out the auto-indent initial
-        # spaces, if the user's actual input started itself with whitespace.
-        if self.autoindent:
-            if num_ini_spaces(line) > self.indent_current_nsp:
-                line = line[self.indent_current_nsp:]
-                self.indent_current_nsp = 0
-            
-        # store the unfiltered input before the user has any chance to modify
-        # it.
-        if line.strip():
-            if continue_prompt:
-                if self.has_readline and self.readline_use:
-                    histlen = self.readline.get_current_history_length()
-                    if histlen > 1:
-                        newhist = self.history_manager.input_hist_raw[-1].rstrip()
-                        self.readline.remove_history_item(histlen-1)
-                        self.readline.replace_history_item(histlen-2,
-                                        newhist.encode(self.stdin_encoding))
-            else:
-                self.history_manager.input_hist_raw.append('%s\n' % line)                
-        elif not continue_prompt:
-            self.history_manager.input_hist_raw.append('\n')
-        try:
-            lineout = self.prefilter_manager.prefilter_lines(line,continue_prompt)
-        except:
-            # blanket except, in case a user-defined prefilter crashes, so it
-            # can't take all of ipython with it.
-            self.showtraceback()
-            return ''
-        else:
-            return lineout
-
-
     def raw_input(self, prompt=''):
         """Write a prompt and read a line.
 

IPython/utils/text.py

@@ -19,7 +19,6 b' import __main__'
 import os
 import re
 import shutil
-import types
 
 from IPython.external.path import path
 
@@ -30,8 +29,6 b' from IPython.utils.data import flatten'
 # Code
 #-----------------------------------------------------------------------------
 
-StringTypes = types.StringTypes
-
 
 def unquote_ends(istr):
     """Remove a single pair of quotes from the endpoints of a string."""
@@ -325,7 +322,7 b' def qw(words,flat=0,sep=None,maxsplit=-1):'
     ['a', 'b', '1', '2', 'm', 'n', 'p', 'q']
     """
 
-    if type(words) in StringTypes:
+    if isinstance(words, basestring):
         return [word.strip() for word in words.split(sep,maxsplit)
                 if word and not word.isspace() ]
     if flat:
@@ -345,7 +342,7 b' def qw_lol(indata):'
     We need this to make sure the modules_some keys *always* end up as a
     list of lists."""
 
-    if type(indata) in StringTypes:
+    if isinstance(indata, basestring):
         return [qw(indata)]
     else:
         return qw(indata)

IPython/zmq/ipkernel.py

@@ -122,7 +122,7 b' class Kernel(Configurable):'
 
         # Build dict of handlers for message types
         msg_types = [ 'execute_request', 'complete_request', 
-                      'object_info_request', 'history_request',
+                      'object_info_request', 'history_tail_request',
                       'connect_request', 'shutdown_request']
         self.handlers = {}
         for msg_type in msg_types:
@@ -323,13 +323,15 b' class Kernel(Configurable):'
                                 oinfo, parent, ident)
         logger.debug(msg)
 
-    def history_request(self, ident, parent):
-        output = parent['content']['output']
-        index = parent['content']['index']
+    def history_tail_request(self, ident, parent):
+        # We need to pull these out, as passing **kwargs doesn't work with
+        # unicode keys before Python 2.6.5.
+        n = parent['content']['n']
         raw = parent['content']['raw']
-        hist = self.shell.get_history(index=index, raw=raw, output=output)
-        content = {'history' : hist}
-        msg = self.session.send(self.reply_socket, 'history_reply',
+        output = parent['content']['output']
+        hist = self.shell.history_manager.get_tail(n, raw=raw, output=output)
+        content = {'history' : list(hist)}
+        msg = self.session.send(self.reply_socket, 'history_tail_reply',
                                 content, parent, ident)
         logger.debug(str(msg))
 

IPython/zmq/kernelmanager.py

@@ -282,15 +282,13 b' class XReqSocketChannel(ZmqSocketChannel):'
         self._queue_request(msg)
         return msg['header']['msg_id']
 
-    def history(self, index=None, raw=False, output=True):
+    def history_tail(self, n=10, raw=True, output=False):
         """Get the history list.
 
         Parameters
         ----------
-        index : n or (n1, n2) or None
-            If n, then the last entries. If a tuple, then all in
-            range(n1, n2). If None, then all entries. Raises IndexError if
-            the format of index is incorrect.
+        n : int
+            The number of lines of history to get.
         raw : bool
             If True, return the raw input.
         output : bool
@@ -300,8 +298,8 b' class XReqSocketChannel(ZmqSocketChannel):'
         -------
         The msg_id of the message sent.
         """
-        content = dict(index=index, raw=raw, output=output)
-        msg = self.session.msg('history_request', content)
+        content = dict(n=n, raw=raw, output=output)
+        msg = self.session.msg('history_tail_request', content)
         self._queue_request(msg)
         return msg['header']['msg_id']
 

IPython/zmq/zmqshell.py

@@ -18,7 +18,6 b' from __future__ import print_function'
 # Stdlib
 import inspect
 import os
-import re
 
 # Our own
 from IPython.core.interactiveshell import (
@@ -31,7 +30,6 b' from IPython.core.macro import Macro'
 from IPython.core.payloadpage import install_payload_page
 from IPython.utils import io
 from IPython.utils.path import get_py_filename
-from IPython.utils.text import StringTypes
 from IPython.utils.traitlets import Instance, Type, Dict
 from IPython.utils.warn import warn
 from IPython.zmq.session import extract_header
@@ -433,9 +431,10 b' class ZMQInteractiveShell(InteractiveShell):'
 
         # by default this is done with temp files, except when the given
         # arg is a filename
-        use_temp = 1
+        use_temp = True
 
-        if re.match(r'\d',args):
+        data = ''
+        if args[0].isdigit():
             # Mode where user specifies ranges of lines, like in %macro.
             # This means that you can't edit files whose names begin with
             # numbers this way. Tough.
@@ -443,16 +442,15 b' class ZMQInteractiveShell(InteractiveShell):'
             data = ''.join(self.extract_input_slices(ranges,opts_r))
         elif args.endswith('.py'):
             filename = make_filename(args)
-            data = ''
-            use_temp = 0
+            use_temp = False
         elif args:
             try:
                 # Load the parameter given as a variable. If not a string,
                 # process it as an object instead (below)
 
                 #print '*** args',args,'type',type(args)  # dbg
-                data = eval(args,self.shell.user_ns)
-                if not type(data) in StringTypes:
+                data = eval(args, self.shell.user_ns)
+                if not isinstance(data, basestring):
                     raise DataIsObject
 
             except (NameError,SyntaxError):
@@ -462,13 +460,11 b' class ZMQInteractiveShell(InteractiveShell):'
                     warn("Argument given (%s) can't be found as a variable "
                          "or as a filename." % args)
                     return
-
-                data = ''
-                use_temp = 0
+                use_temp = False
+                
             except DataIsObject:
-
                 # macros have a special edit function
-                if isinstance(data,Macro):
+                if isinstance(data, Macro):
                     self._edit_macro(args,data)
                     return
                                 
@@ -507,9 +503,7 b' class ZMQInteractiveShell(InteractiveShell):'
                             warn('The file `%s` where `%s` was defined cannot '
                                  'be read.' % (filename,data))
                             return
-                use_temp = 0
-        else:
-            data = ''
+                use_temp = False
 
         if use_temp:
             filename = self.shell.mktempfile(data)