From 401962b8b5b28fd9916255c27a0e26e30d70712c 2011-10-14 20:44:19 From: Thomas Kluyver Date: 2011-10-14 20:44:19 Subject: [PATCH] Merge branch 'history-access' --- diff --git a/IPython/core/history.py b/IPython/core/history.py index 3a8049d..9ae70f6 100644 --- a/IPython/core/history.py +++ b/IPython/core/history.py @@ -25,6 +25,7 @@ from IPython.config.configurable import Configurable from IPython.testing.skipdoctest import skip_doctest from IPython.utils import io +from IPython.utils.path import locate_profile from IPython.utils.traitlets import Bool, Dict, Instance, Int, CInt, List, Unicode from IPython.utils.warn import warn @@ -32,79 +33,39 @@ from IPython.utils.warn import warn # Classes and functions #----------------------------------------------------------------------------- -class HistoryManager(Configurable): - """A class to organize all history-related functionality in one place. - """ - # Public interface - - # An instance of the IPython shell we are attached to - shell = Instance('IPython.core.interactiveshell.InteractiveShellABC') - # Lists to hold processed and raw history. These start with a blank entry - # so that we can index them starting from 1 - input_hist_parsed = List([""]) - input_hist_raw = List([""]) - # A list of directories visited during session - dir_hist = List() - def _dir_hist_default(self): - try: - return [os.getcwdu()] - except OSError: - return [] - - # A dict of output history, keyed with ints from the shell's - # execution count. - output_hist = Dict() - # The text/plain repr of outputs. - output_hist_reprs = Dict() - +class HistoryAccessor(Configurable): + """Access the history database without adding to it. + + This is intended for use by standalone history tools. IPython shells use + HistoryManager, below, which is a subclass of this.""" # String holding the path to the history file hist_file = Unicode(config=True) # The SQLite database db = Instance(sqlite3.Connection) - # The number of the current session in the history database - session_number = CInt() - # Should we log output to the database? (default no) - db_log_output = Bool(False, config=True) - # Write to database every x commands (higher values save disk access & power) - # Values of 1 or less effectively disable caching. - db_cache_size = Int(0, config=True) - # The input and output caches - db_input_cache = List() - db_output_cache = List() - - # History saving in separate thread - save_thread = Instance('IPython.core.history.HistorySavingThread') - try: # Event is a function returning an instance of _Event... - save_flag = Instance(threading._Event) - except AttributeError: # ...until Python 3.3, when it's a class. - save_flag = Instance(threading.Event) - - # Private interface - # Variables used to store the three last inputs from the user. On each new - # history update, we populate the user's namespace with these, shifted as - # necessary. - _i00 = Unicode(u'') - _i = Unicode(u'') - _ii = Unicode(u'') - _iii = Unicode(u'') - - # A regex matching all forms of the exit command, so that we don't store - # them in the history (it's annoying to rewind the first entry and land on - # an exit call). - _exit_re = re.compile(r"(exit|quit)(\s*\(.*\))?$") - - def __init__(self, shell, config=None, **traits): - """Create a new history manager associated with a shell instance. + + def __init__(self, profile='default', hist_file=u'', shell=None, config=None, **traits): + """Create a new history accessor. + + Parameters + ---------- + profile : str + The name of the profile from which to open history. + hist_file : str + Path to an SQLite history database stored by IPython. If specified, + hist_file overrides profile. + shell : + InteractiveShell object, for use by HistoryManager subclass + config : + Config object. hist_file can also be set through this. """ # We need a pointer back to the shell for various tasks. - super(HistoryManager, self).__init__(shell=shell, config=config, - **traits) + super(HistoryAccessor, self).__init__(shell=shell, config=config, + hist_file=hist_file, **traits) if self.hist_file == u'': # No one has set the hist_file, yet. - histfname = 'history' - self.hist_file = os.path.join(shell.profile_dir.location, histfname + '.sqlite') + self.hist_file = self._get_hist_file_name(profile) try: self.init_db() @@ -119,16 +80,20 @@ class HistoryManager(Configurable): else: # The hist_file is probably :memory: or something else. raise - - self.save_flag = threading.Event() - self.db_input_cache_lock = threading.Lock() - self.db_output_cache_lock = threading.Lock() - self.save_thread = HistorySavingThread(self) - self.save_thread.start() - - self.new_session() - - + + def _get_hist_file_name(self, profile='default'): + """Find the history file for the given profile name. + + This is overridden by the HistoryManager subclass, to use the shell's + active profile. + + Parameters + ---------- + profile : str + The name of a profile which has a history file. + """ + return os.path.join(locate_profile(profile), 'history.sqlite') + def init_db(self): """Connect to the database, and create tables if necessary.""" # use detect_types so that timestamps return datetime objects @@ -146,48 +111,10 @@ class HistoryManager(Configurable): PRIMARY KEY (session, line))""") self.db.commit() - def new_session(self, conn=None): - """Get a new session number.""" - if conn is None: - conn = self.db - - with conn: - # N.B. 'insert into' here is lower case because of a bug in the - # sqlite3 module that affects the Turkish locale. This should be - # fixed for Python 2.7.3 and 3.2.3, as well as 3.3 onwards. - # http://bugs.python.org/issue13099 - cur = conn.execute("""insert into sessions VALUES (NULL, ?, NULL, - NULL, "") """, (datetime.datetime.now(),)) - self.session_number = cur.lastrowid - - def end_session(self): - """Close the database session, filling in the end time and line count.""" - self.writeout_cache() - with self.db: - self.db.execute("""UPDATE sessions SET end=?, num_cmds=? WHERE - session==?""", (datetime.datetime.now(), - len(self.input_hist_parsed)-1, self.session_number)) - self.session_number = 0 - - def name_session(self, name): - """Give the current session a name in the history database.""" - with self.db: - self.db.execute("UPDATE sessions SET remark=? WHERE session==?", - (name, self.session_number)) - - def reset(self, new_session=True): - """Clear the session history, releasing all object references, and - optionally open a new session.""" - self.output_hist.clear() - # The directory history can't be completely empty - self.dir_hist[:] = [os.getcwdu()] - - if new_session: - if self.session_number: - self.end_session() - self.input_hist_parsed[:] = [""] - self.input_hist_raw[:] = [""] - self.new_session() + def writeout_cache(self): + """Overridden by HistoryManager to dump the cache before certain + database lookups.""" + pass ## ------------------------------- ## Methods for retrieving history: @@ -219,7 +146,6 @@ class HistoryManager(Configurable): return ((ses, lin, (inp, out)) for ses, lin, inp, out in cur) return cur - def get_session_info(self, session=0): """get info about a session @@ -246,7 +172,6 @@ class HistoryManager(Configurable): query = "SELECT * from sessions where session == ?" return self.db.execute(query, (session,)).fetchone() - def get_tail(self, n=10, raw=True, output=False, include_latest=False): """Get the last n lines from the history database. @@ -298,35 +223,14 @@ class HistoryManager(Configurable): self.writeout_cache() return self._run_sql("WHERE %s GLOB ?" % tosearch, (pattern,), raw=raw, output=output) - - def _get_range_session(self, start=1, stop=None, raw=True, output=False): - """Get input and output history from the current session. Called by - get_range, and takes similar parameters.""" - input_hist = self.input_hist_raw if raw else self.input_hist_parsed - - n = len(input_hist) - if start < 0: - start += n - if not stop or (stop > n): - stop = n - elif stop < 0: - stop += n - - for i in range(start, stop): - if output: - line = (input_hist[i], self.output_hist_reprs.get(i)) - else: - line = input_hist[i] - yield (0, i, line) - - def get_range(self, session=0, start=1, stop=None, raw=True,output=False): + + def get_range(self, session, start=1, stop=None, raw=True,output=False): """Retrieve input by session. Parameters ---------- session : int - Session number to retrieve. The current session is 0, and negative - numbers count back from current session, so -1 is previous session. + Session number to retrieve. start : int First line to retrieve. stop : int @@ -346,11 +250,6 @@ class HistoryManager(Configurable): (session, line, input) if output is False, or (session, line, (input, output)) if output is True. """ - if session == 0 or session==self.session_number: # Current session - return self._get_range_session(start, stop, raw, output) - if session < 0: - session += self.session_number - if stop: lineclause = "line >= ? AND line < ?" params = (session, start, stop) @@ -381,6 +280,181 @@ class HistoryManager(Configurable): for line in self.get_range(sess, s, e, raw=raw, output=output): yield line + +class HistoryManager(HistoryAccessor): + """A class to organize all history-related functionality in one place. + """ + # Public interface + + # An instance of the IPython shell we are attached to + shell = Instance('IPython.core.interactiveshell.InteractiveShellABC') + # Lists to hold processed and raw history. These start with a blank entry + # so that we can index them starting from 1 + input_hist_parsed = List([""]) + input_hist_raw = List([""]) + # A list of directories visited during session + dir_hist = List() + def _dir_hist_default(self): + try: + return [os.getcwdu()] + except OSError: + return [] + + # A dict of output history, keyed with ints from the shell's + # execution count. + output_hist = Dict() + # The text/plain repr of outputs. + output_hist_reprs = Dict() + + # The number of the current session in the history database + session_number = CInt() + # Should we log output to the database? (default no) + db_log_output = Bool(False, config=True) + # Write to database every x commands (higher values save disk access & power) + # Values of 1 or less effectively disable caching. + db_cache_size = Int(0, config=True) + # The input and output caches + db_input_cache = List() + db_output_cache = List() + + # History saving in separate thread + save_thread = Instance('IPython.core.history.HistorySavingThread') + try: # Event is a function returning an instance of _Event... + save_flag = Instance(threading._Event) + except AttributeError: # ...until Python 3.3, when it's a class. + save_flag = Instance(threading.Event) + + # Private interface + # Variables used to store the three last inputs from the user. On each new + # history update, we populate the user's namespace with these, shifted as + # necessary. + _i00 = Unicode(u'') + _i = Unicode(u'') + _ii = Unicode(u'') + _iii = Unicode(u'') + + # A regex matching all forms of the exit command, so that we don't store + # them in the history (it's annoying to rewind the first entry and land on + # an exit call). + _exit_re = re.compile(r"(exit|quit)(\s*\(.*\))?$") + + def __init__(self, shell=None, config=None, **traits): + """Create a new history manager associated with a shell instance. + """ + # We need a pointer back to the shell for various tasks. + super(HistoryManager, self).__init__(shell=shell, config=config, + **traits) + self.save_flag = threading.Event() + self.db_input_cache_lock = threading.Lock() + self.db_output_cache_lock = threading.Lock() + self.save_thread = HistorySavingThread(self) + self.save_thread.start() + + self.new_session() + + def _get_hist_file_name(self, profile=None): + """Get default history file name based on the Shell's profile. + + The profile parameter is ignored, but must exist for compatibility with + the parent class.""" + profile_dir = self.shell.profile_dir.location + return os.path.join(profile_dir, 'history.sqlite') + + def new_session(self, conn=None): + """Get a new session number.""" + if conn is None: + conn = self.db + + with conn: + cur = conn.execute("""INSERT INTO sessions VALUES (NULL, ?, NULL, + NULL, "") """, (datetime.datetime.now(),)) + self.session_number = cur.lastrowid + + def end_session(self): + """Close the database session, filling in the end time and line count.""" + self.writeout_cache() + with self.db: + self.db.execute("""UPDATE sessions SET end=?, num_cmds=? WHERE + session==?""", (datetime.datetime.now(), + len(self.input_hist_parsed)-1, self.session_number)) + self.session_number = 0 + + def name_session(self, name): + """Give the current session a name in the history database.""" + with self.db: + self.db.execute("UPDATE sessions SET remark=? WHERE session==?", + (name, self.session_number)) + + def reset(self, new_session=True): + """Clear the session history, releasing all object references, and + optionally open a new session.""" + self.output_hist.clear() + # The directory history can't be completely empty + self.dir_hist[:] = [os.getcwdu()] + + if new_session: + if self.session_number: + self.end_session() + self.input_hist_parsed[:] = [""] + self.input_hist_raw[:] = [""] + self.new_session() + + # ------------------------------ + # Methods for retrieving history + # ------------------------------ + def _get_range_session(self, start=1, stop=None, raw=True, output=False): + """Get input and output history from the current session. Called by + get_range, and takes similar parameters.""" + input_hist = self.input_hist_raw if raw else self.input_hist_parsed + + n = len(input_hist) + if start < 0: + start += n + if not stop or (stop > n): + stop = n + elif stop < 0: + stop += n + + for i in range(start, stop): + if output: + line = (input_hist[i], self.output_hist_reprs.get(i)) + else: + line = input_hist[i] + yield (0, i, line) + + def get_range(self, session=0, start=1, stop=None, raw=True,output=False): + """Retrieve input by session. + + Parameters + ---------- + session : int + Session number to retrieve. The current session is 0, and negative + numbers count back from current session, so -1 is previous session. + start : int + First line to retrieve. + stop : int + End of line range (excluded from output itself). If None, retrieve + to the end of the session. + raw : bool + If True, return untranslated input + output : bool + If True, attempt to include output. This will be 'real' Python + objects for the current session, or text reprs from previous + sessions if db_log_output was enabled at the time. Where no output + is found, None is used. + + Returns + ------- + An iterator over the desired lines. Each line is a 3-tuple, either + (session, line, input) if output is False, or + (session, line, (input, output)) if output is True. + """ + if session <= 0: + session += self.session_number + if session==self.session_number: # Current session + return self._get_range_session(start, stop, raw, output) + return super(HistoryManager, self).get_range(session, start, stop, raw, output) + ## ---------------------------- ## Methods for storing history: ## ---------------------------- diff --git a/IPython/core/tests/test_history.py b/IPython/core/tests/test_history.py index 64dff11..112665a 100644 --- a/IPython/core/tests/test_history.py +++ b/IPython/core/tests/test_history.py @@ -38,6 +38,12 @@ def test_history(): ip.history_manager.store_output(3) nt.assert_equal(ip.history_manager.input_hist_raw, [''] + hist) + + # Detailed tests for _get_range_session + grs = ip.history_manager._get_range_session + nt.assert_equal(list(grs(start=2,stop=-1)), zip([0], [2], hist[1:-1])) + nt.assert_equal(list(grs(start=-2)), zip([0,0], [2,3], hist[-2:])) + nt.assert_equal(list(grs(output=True)), zip([0,0,0], [1,2,3], zip(hist, [None,None,'spam']))) # Check whether specifying a range beyond the end of the current # session results in an error (gh-804) diff --git a/IPython/utils/path.py b/IPython/utils/path.py index 5b4a09c..96e96a2 100644 --- a/IPython/utils/path.py +++ b/IPython/utils/path.py @@ -372,6 +372,18 @@ def get_ipython_module_path(module_str): the_path = the_path.replace('.pyo', '.py') return py3compat.cast_unicode(the_path, fs_encoding) +def locate_profile(profile='default'): + """Find the path to the folder associated with a given profile. + + I.e. find $IPYTHON_DIR/profile_whatever. + """ + from IPython.core.profiledir import ProfileDir, ProfileDirError + try: + pd = ProfileDir.find_profile_dir_by_name(get_ipython_dir(), profile) + except ProfileDirError: + # IOError makes more sense when people are expecting a path + raise IOError("Couldn't find profile %r" % profile) + return pd.location def expand_path(s): """Expand $VARS and ~names in a string, like a shell diff --git a/docs/examples/core/ipython-get-history.py b/docs/examples/core/ipython-get-history.py new file mode 100755 index 0000000..d6d51ec --- /dev/null +++ b/docs/examples/core/ipython-get-history.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python +"""Extract a session from the IPython input history. + +Usage: + ipython-get-history.py sessionnumber [outputfile] + +If outputfile is not given, the relevant history is written to stdout. If +outputfile has a .py extension, the translated history (without IPython's +special syntax) will be extracted. + +Example: + ./ipython-get-history.py 57 record.ipy + + +This script is a simple demonstration of HistoryAccessor. It should be possible +to build much more flexible and powerful tools to browse and pull from the +history database. +""" +import sys +import codecs + +from IPython.core.history import HistoryAccessor + +session_number = int(sys.argv[1]) +if len(sys.argv) > 2: + dest = open(sys.argv[2], "w") + raw = not sys.argv[2].endswith('.py') +else: + dest = sys.stdout + raw = True +dest.write("# coding: utf-8\n") + +# Profiles other than 'default' can be specified here with a profile= argument: +hist = HistoryAccessor() + +for session, lineno, cell in hist.get_range(session=session_number, raw=raw): + # To use this in Python 3, remove the .encode() here: + dest.write(cell.encode('utf-8') + '\n')