Merge branch 'history-access'

docs/examples/core/ipython-get-history.py

@@ -0,0 +1,38 b''
+#!/usr/bin/env python
+"""Extract a session from the IPython input history.
+
+Usage:
+  ipython-get-history.py sessionnumber [outputfile]
+
+If outputfile is not given, the relevant history is written to stdout. If
+outputfile has a .py extension, the translated history (without IPython's
+special syntax) will be extracted.
+
+Example:
+  ./ipython-get-history.py 57 record.ipy
+
+
+This script is a simple demonstration of HistoryAccessor. It should be possible
+to build much more flexible and powerful tools to browse and pull from the
+history database.
+"""
+import sys
+import codecs
+
+from IPython.core.history import HistoryAccessor
+
+session_number = int(sys.argv[1])
+if len(sys.argv) > 2:
+    dest = open(sys.argv[2], "w")
+    raw = not sys.argv[2].endswith('.py')
+else:
+    dest = sys.stdout
+    raw = True
+dest.write("# coding: utf-8\n")
+
+# Profiles other than 'default' can be specified here with a profile= argument:
+hist = HistoryAccessor()
+
+for session, lineno, cell in hist.get_range(session=session_number, raw=raw):
+    # To use this in Python 3, remove the .encode() here:
+    dest.write(cell.encode('utf-8') + '\n')

IPython/core/history.py

@@ -25,6 +25,7 b' from IPython.config.configurable import Configurable'
 
 from IPython.testing.skipdoctest import skip_doctest
 from IPython.utils import io
+from IPython.utils.path import locate_profile
 from IPython.utils.traitlets import Bool, Dict, Instance, Int, CInt, List, Unicode
 from IPython.utils.warn import warn
 
@@ -32,79 +33,39 b' from IPython.utils.warn import warn'
 # Classes and functions
 #-----------------------------------------------------------------------------
 
-class HistoryManager(Configurable):
-    """A class to organize all history-related functionality in one place.
-    """
-    # Public interface
-
-    # An instance of the IPython shell we are attached to
-    shell = Instance('IPython.core.interactiveshell.InteractiveShellABC')
-    # Lists to hold processed and raw history. These start with a blank entry
-    # so that we can index them starting from 1
-    input_hist_parsed = List([""])
-    input_hist_raw = List([""])
-    # A list of directories visited during session
-    dir_hist = List()
-    def _dir_hist_default(self):
-        try:
-            return [os.getcwdu()]
-        except OSError:
-            return []
-
-    # A dict of output history, keyed with ints from the shell's
-    # execution count.
-    output_hist = Dict()
-    # The text/plain repr of outputs.
-    output_hist_reprs = Dict()
+class HistoryAccessor(Configurable):
+    """Access the history database without adding to it.
 
+    This is intended for use by standalone history tools. IPython shells use
+    HistoryManager, below, which is a subclass of this."""
     # String holding the path to the history file
     hist_file = Unicode(config=True)
 
     # The SQLite database
     db = Instance(sqlite3.Connection)
-    # The number of the current session in the history database
-    session_number = CInt()
-    # Should we log output to the database? (default no)
-    db_log_output = Bool(False, config=True)
-    # Write to database every x commands (higher values save disk access & power)
-    #  Values of 1 or less effectively disable caching.
-    db_cache_size = Int(0, config=True)
-    # The input and output caches
-    db_input_cache = List()
-    db_output_cache = List()
-
-    # History saving in separate thread
-    save_thread = Instance('IPython.core.history.HistorySavingThread')
-    try:               # Event is a function returning an instance of _Event...
-        save_flag = Instance(threading._Event)
-    except AttributeError:         # ...until Python 3.3, when it's a class.
-        save_flag = Instance(threading.Event)
 
-    # Private interface
-    # Variables used to store the three last inputs from the user.  On each new
-    # history update, we populate the user's namespace with these, shifted as
-    # necessary.
-    _i00 = Unicode(u'')
-    _i = Unicode(u'')
-    _ii = Unicode(u'')
-    _iii = Unicode(u'')
+    def __init__(self, profile='default', hist_file=u'', shell=None, config=None, **traits):
+        """Create a new history accessor.
 
-    # A regex matching all forms of the exit command, so that we don't store
-    # them in the history (it's annoying to rewind the first entry and land on
-    # an exit call).
-    _exit_re = re.compile(r"(exit|quit)(\s*\(.*\))?$")
-
-    def __init__(self, shell, config=None, **traits):
-        """Create a new history manager associated with a shell instance.
+        Parameters
+        ----------
+        profile : str
+          The name of the profile from which to open history.
+        hist_file : str
+          Path to an SQLite history database stored by IPython. If specified,
+          hist_file overrides profile.
+        shell :
+          InteractiveShell object, for use by HistoryManager subclass
+        config :
+          Config object. hist_file can also be set through this.
         """
         # We need a pointer back to the shell for various tasks.
-        super(HistoryManager, self).__init__(shell=shell, config=config,
-            **traits)
+        super(HistoryAccessor, self).__init__(shell=shell, config=config,
+                                              hist_file=hist_file, **traits)
 
         if self.hist_file == u'':
             # No one has set the hist_file, yet.
-            histfname = 'history'
-            self.hist_file = os.path.join(shell.profile_dir.location, histfname + '.sqlite')
+            self.hist_file = self._get_hist_file_name(profile)
 
         try:
             self.init_db()
@@ -120,14 +81,18 b' class HistoryManager(Configurable):'
                 # The hist_file is probably :memory: or something else.
                 raise
 
-        self.save_flag = threading.Event()
-        self.db_input_cache_lock = threading.Lock()
-        self.db_output_cache_lock = threading.Lock()
-        self.save_thread = HistorySavingThread(self)
-        self.save_thread.start()
+    def _get_hist_file_name(self, profile='default'):
+        """Find the history file for the given profile name.
 
-        self.new_session()
+        This is overridden by the HistoryManager subclass, to use the shell's
+        active profile.
 
+        Parameters
+        ----------
+        profile : str
+          The name of a profile which has a history file.
+        """
+        return os.path.join(locate_profile(profile), 'history.sqlite')
 
     def init_db(self):
         """Connect to the database, and create tables if necessary."""
@@ -146,48 +111,10 b' class HistoryManager(Configurable):'
                         PRIMARY KEY (session, line))""")
         self.db.commit()
 
-    def new_session(self, conn=None):
-        """Get a new session number."""
-        if conn is None:
-            conn = self.db
-
-        with conn:
-            # N.B. 'insert into' here is lower case because of a bug in the
-            # sqlite3 module that affects the Turkish locale. This should be
-            # fixed for Python 2.7.3 and 3.2.3, as well as 3.3 onwards.
-            # http://bugs.python.org/issue13099
-            cur = conn.execute("""insert into sessions VALUES (NULL, ?, NULL,
-                            NULL, "") """, (datetime.datetime.now(),))
-            self.session_number = cur.lastrowid
-
-    def end_session(self):
-        """Close the database session, filling in the end time and line count."""
-        self.writeout_cache()
-        with self.db:
-            self.db.execute("""UPDATE sessions SET end=?, num_cmds=? WHERE
-                            session==?""", (datetime.datetime.now(),
-                            len(self.input_hist_parsed)-1, self.session_number))
-        self.session_number = 0
-
-    def name_session(self, name):
-        """Give the current session a name in the history database."""
-        with self.db:
-            self.db.execute("UPDATE sessions SET remark=? WHERE session==?",
-                            (name, self.session_number))
-
-    def reset(self, new_session=True):
-        """Clear the session history, releasing all object references, and
-        optionally open a new session."""
-        self.output_hist.clear()
-        # The directory history can't be completely empty
-        self.dir_hist[:] = [os.getcwdu()]
-
-        if new_session:
-            if self.session_number:
-                self.end_session()
-            self.input_hist_parsed[:] = [""]
-            self.input_hist_raw[:] = [""]
-            self.new_session()
+    def writeout_cache(self):
+        """Overridden by HistoryManager to dump the cache before certain
+        database lookups."""
+        pass
 
     ## -------------------------------
     ## Methods for retrieving history:
@@ -219,7 +146,6 b' class HistoryManager(Configurable):'
             return ((ses, lin, (inp, out)) for ses, lin, inp, out in cur)
         return cur
 
-
     def get_session_info(self, session=0):
         """get info about a session
 
@@ -246,7 +172,6 b' class HistoryManager(Configurable):'
         query = "SELECT * from sessions where session == ?"
         return self.db.execute(query, (session,)).fetchone()
 
-
     def get_tail(self, n=10, raw=True, output=False, include_latest=False):
         """Get the last n lines from the history database.
 
@@ -299,34 +224,13 b' class HistoryManager(Configurable):'
         return self._run_sql("WHERE %s GLOB ?" % tosearch, (pattern,),
                                     raw=raw, output=output)
 
-    def _get_range_session(self, start=1, stop=None, raw=True, output=False):
-        """Get input and output history from the current session. Called by
-        get_range, and takes similar parameters."""
-        input_hist = self.input_hist_raw if raw else self.input_hist_parsed
-
-        n = len(input_hist)
-        if start < 0:
-            start += n
-        if not stop or (stop > n):
-            stop = n
-        elif stop < 0:
-            stop += n
-
-        for i in range(start, stop):
-            if output:
-                line = (input_hist[i], self.output_hist_reprs.get(i))
-            else:
-                line = input_hist[i]
-            yield (0, i, line)
-
-    def get_range(self, session=0, start=1, stop=None, raw=True,output=False):
+    def get_range(self, session, start=1, stop=None, raw=True,output=False):
         """Retrieve input by session.
 
         Parameters
         ----------
         session : int
-            Session number to retrieve. The current session is 0, and negative
-            numbers count back from current session, so -1 is previous session.
+            Session number to retrieve.
         start : int
             First line to retrieve.
         stop : int
@@ -346,11 +250,6 b' class HistoryManager(Configurable):'
         (session, line, input) if output is False, or
         (session, line, (input, output)) if output is True.
         """
-        if session == 0 or session==self.session_number:   # Current session
-            return self._get_range_session(start, stop, raw, output)
-        if session < 0:
-            session += self.session_number
-
         if stop:
             lineclause = "line >= ? AND line < ?"
             params = (session, start, stop)
@@ -381,6 +280,181 b' class HistoryManager(Configurable):'
             for line in self.get_range(sess, s, e, raw=raw, output=output):
                 yield line
 
+
+class HistoryManager(HistoryAccessor):
+    """A class to organize all history-related functionality in one place.
+    """
+    # Public interface
+
+    # An instance of the IPython shell we are attached to
+    shell = Instance('IPython.core.interactiveshell.InteractiveShellABC')
+    # Lists to hold processed and raw history. These start with a blank entry
+    # so that we can index them starting from 1
+    input_hist_parsed = List([""])
+    input_hist_raw = List([""])
+    # A list of directories visited during session
+    dir_hist = List()
+    def _dir_hist_default(self):
+        try:
+            return [os.getcwdu()]
+        except OSError:
+            return []
+
+    # A dict of output history, keyed with ints from the shell's
+    # execution count.
+    output_hist = Dict()
+    # The text/plain repr of outputs.
+    output_hist_reprs = Dict()
+
+    # The number of the current session in the history database
+    session_number = CInt()
+    # Should we log output to the database? (default no)
+    db_log_output = Bool(False, config=True)
+    # Write to database every x commands (higher values save disk access & power)
+    #  Values of 1 or less effectively disable caching. 
+    db_cache_size = Int(0, config=True)
+    # The input and output caches
+    db_input_cache = List()
+    db_output_cache = List()
+    
+    # History saving in separate thread
+    save_thread = Instance('IPython.core.history.HistorySavingThread')
+    try:               # Event is a function returning an instance of _Event...
+        save_flag = Instance(threading._Event)
+    except AttributeError:         # ...until Python 3.3, when it's a class.
+        save_flag = Instance(threading.Event)
+    
+    # Private interface
+    # Variables used to store the three last inputs from the user.  On each new
+    # history update, we populate the user's namespace with these, shifted as
+    # necessary.
+    _i00 = Unicode(u'')
+    _i = Unicode(u'')
+    _ii = Unicode(u'')
+    _iii = Unicode(u'')
+
+    # A regex matching all forms of the exit command, so that we don't store
+    # them in the history (it's annoying to rewind the first entry and land on
+    # an exit call).
+    _exit_re = re.compile(r"(exit|quit)(\s*\(.*\))?$")
+
+    def __init__(self, shell=None, config=None, **traits):
+        """Create a new history manager associated with a shell instance.
+        """
+        # We need a pointer back to the shell for various tasks.
+        super(HistoryManager, self).__init__(shell=shell, config=config,
+            **traits)
+        self.save_flag = threading.Event()
+        self.db_input_cache_lock = threading.Lock()
+        self.db_output_cache_lock = threading.Lock()
+        self.save_thread = HistorySavingThread(self)
+        self.save_thread.start()
+
+        self.new_session()
+    
+    def _get_hist_file_name(self, profile=None):
+        """Get default history file name based on the Shell's profile.
+        
+        The profile parameter is ignored, but must exist for compatibility with
+        the parent class."""
+        profile_dir = self.shell.profile_dir.location
+        return os.path.join(profile_dir, 'history.sqlite')
+    
+    def new_session(self, conn=None):
+        """Get a new session number."""
+        if conn is None:
+            conn = self.db
+        
+        with conn:
+            cur = conn.execute("""INSERT INTO sessions VALUES (NULL, ?, NULL,
+                            NULL, "") """, (datetime.datetime.now(),))
+            self.session_number = cur.lastrowid
+            
+    def end_session(self):
+        """Close the database session, filling in the end time and line count."""
+        self.writeout_cache()
+        with self.db:
+            self.db.execute("""UPDATE sessions SET end=?, num_cmds=? WHERE
+                            session==?""", (datetime.datetime.now(),
+                            len(self.input_hist_parsed)-1, self.session_number))
+        self.session_number = 0
+                            
+    def name_session(self, name):
+        """Give the current session a name in the history database."""
+        with self.db:
+            self.db.execute("UPDATE sessions SET remark=? WHERE session==?",
+                            (name, self.session_number))
+                            
+    def reset(self, new_session=True):
+        """Clear the session history, releasing all object references, and
+        optionally open a new session."""
+        self.output_hist.clear()
+        # The directory history can't be completely empty
+        self.dir_hist[:] = [os.getcwdu()]
+        
+        if new_session:
+            if self.session_number:
+                self.end_session()
+            self.input_hist_parsed[:] = [""]
+            self.input_hist_raw[:] = [""]
+            self.new_session()
+    
+    # ------------------------------
+    # Methods for retrieving history
+    # ------------------------------
+    def _get_range_session(self, start=1, stop=None, raw=True, output=False):
+        """Get input and output history from the current session. Called by
+        get_range, and takes similar parameters."""
+        input_hist = self.input_hist_raw if raw else self.input_hist_parsed
+            
+        n = len(input_hist)
+        if start < 0:
+            start += n
+        if not stop or (stop > n):
+            stop = n
+        elif stop < 0:
+            stop += n
+        
+        for i in range(start, stop):
+            if output:
+                line = (input_hist[i], self.output_hist_reprs.get(i))
+            else:
+                line = input_hist[i]
+            yield (0, i, line)
+    
+    def get_range(self, session=0, start=1, stop=None, raw=True,output=False):
+        """Retrieve input by session.
+        
+        Parameters
+        ----------
+        session : int
+            Session number to retrieve. The current session is 0, and negative
+            numbers count back from current session, so -1 is previous session.
+        start : int
+            First line to retrieve.
+        stop : int
+            End of line range (excluded from output itself). If None, retrieve
+            to the end of the session.
+        raw : bool
+            If True, return untranslated input
+        output : bool
+            If True, attempt to include output. This will be 'real' Python
+            objects for the current session, or text reprs from previous
+            sessions if db_log_output was enabled at the time. Where no output
+            is found, None is used.
+            
+        Returns
+        -------
+        An iterator over the desired lines. Each line is a 3-tuple, either
+        (session, line, input) if output is False, or
+        (session, line, (input, output)) if output is True.
+        """
+        if session <= 0:
+            session += self.session_number
+        if session==self.session_number:          # Current session
+            return self._get_range_session(start, stop, raw, output)
+        return super(HistoryManager, self).get_range(session, start, stop, raw, output)
+
     ## ----------------------------
     ## Methods for storing history:
     ## ----------------------------

IPython/core/tests/test_history.py

@@ -39,6 +39,12 b' def test_history():'
 
             nt.assert_equal(ip.history_manager.input_hist_raw, [''] + hist)
 
+            # Detailed tests for _get_range_session
+            grs = ip.history_manager._get_range_session
+            nt.assert_equal(list(grs(start=2,stop=-1)), zip([0], [2], hist[1:-1]))
+            nt.assert_equal(list(grs(start=-2)), zip([0,0], [2,3], hist[-2:]))
+            nt.assert_equal(list(grs(output=True)), zip([0,0,0], [1,2,3], zip(hist, [None,None,'spam'])))
+
             # Check whether specifying a range beyond the end of the current
             # session results in an error (gh-804)
             ip.magic('%hist 2-500')

IPython/utils/path.py

@@ -372,6 +372,18 b' def get_ipython_module_path(module_str):'
     the_path = the_path.replace('.pyo', '.py')
     return py3compat.cast_unicode(the_path, fs_encoding)
 
+def locate_profile(profile='default'):
+    """Find the path to the folder associated with a given profile.
+    
+    I.e. find $IPYTHON_DIR/profile_whatever.
+    """
+    from IPython.core.profiledir import ProfileDir, ProfileDirError
+    try:
+        pd = ProfileDir.find_profile_dir_by_name(get_ipython_dir(), profile)
+    except ProfileDirError:
+        # IOError makes more sense when people are expecting a path
+        raise IOError("Couldn't find profile %r" % profile)
+    return pd.location
 
 def expand_path(s):
     """Expand $VARS and ~names in a string, like a shell